From c0aac13cd36d9e10338dd6d9ff259c8ad3f05453 Mon Sep 17 00:00:00 2001 From: Zihan Dai <99155080+PDGGK@users.noreply.github.com> Date: Tue, 21 Jul 2026 03:56:56 +1000 Subject: [PATCH 1/3] Add table-model query support to the Grafana data source plugin The Grafana data source plugin only understood the tree model: both QueryEditor modes build root.* paths and the backend calls /grafana/v1/query/expression, which has no table-model equivalent, so table-model users could not visualize their data through the plugin. This adds a third QueryEditor mode, "SQL: Table Model", that runs standard SQL against a database and renders the result. It reuses IoTDB's existing table REST endpoint (POST /rest/table/v1/query) rather than adding a new server API: the backend sends {database, sql} and turns the row-major QueryDataSet into a Grafana frame with one typed, nullable field per column, driven by the response's data_types so column types are exact instead of guessed. The body is decoded with UseNumber so INT64 values beyond 2^53 keep their precision. In the default Time series format, rows are sorted by the first TIMESTAMP column and a long-shaped result (time + tag columns + value columns) is pivoted into one labeled series per tag combination, so a multi-device query renders as separate lines; the Table format preserves the server's row order. A $__timeFilter(col) / $__timeFrom / $__timeTo macro set expands to the panel's range, accepting the $__timeFrom() / $__timeTo() function forms as well. Because the table REST endpoint returns TIMESTAMP values (and compares integer time literals) in the server's configured timestamp_precision -- unlike the tree-model /grafana/v1 endpoints, which normalize to milliseconds server-side -- a new data source "time precision" option (ms default / us / ns) scales both the macro bounds and the TIMESTAMP rendering, so the mode also works on us/ns servers. Follow-up work for the same issue: a database -> table -> column browser, a visual query builder, a default row limit to bound large SELECTs, and the Grafana plugin SDK / Go backend modernization. Signed-off-by: Zihan Dai <99155080+PDGGK@users.noreply.github.com> --- connectors/grafana-plugin/README.md | 31 +- .../grafana-plugin/pkg/plugin/plugin.go | 23 +- .../grafana-plugin/pkg/plugin/table_query.go | 396 ++++++++++++++++++ .../pkg/plugin/table_query_test.go | 368 ++++++++++++++++ .../grafana-plugin/src/ConfigEditor.tsx | 31 +- connectors/grafana-plugin/src/QueryEditor.tsx | 93 +++- connectors/grafana-plugin/src/datasource.ts | 7 + connectors/grafana-plugin/src/types.ts | 11 + 8 files changed, 947 insertions(+), 13 deletions(-) create mode 100644 connectors/grafana-plugin/pkg/plugin/table_query.go create mode 100644 connectors/grafana-plugin/pkg/plugin/table_query_test.go diff --git a/connectors/grafana-plugin/README.md b/connectors/grafana-plugin/README.md index 974916bc..08d35d81 100644 --- a/connectors/grafana-plugin/README.md +++ b/connectors/grafana-plugin/README.md @@ -69,7 +69,7 @@ Click the `New Dashboard` icon on the top right, and select `Add an empty panel` -Grafana plugin supports SQL: Full Customized mode and SQL: Drop-down List mode, and the default mode is SQL: Full Customized mode. +Grafana plugin supports SQL: Full Customized mode and SQL: Drop-down List mode for the tree model, and SQL: Table Model mode for the table model. The default mode is SQL: Full Customized mode. @@ -119,6 +119,35 @@ Select a time series in the TIME-SERIES selection box, select a function in the +##### SQL: Table Model + +Queries IoTDB's table model (IoTDB 2.x) through the REST interface `/rest/table/v1/query`. Enter a standard SQL statement in the SQL input box, for example: + +```sql +SELECT time, device_id, temperature FROM table1 WHERE $__timeFilter(time) +``` + +DATABASE input box: the database to run the statement against. Optional when every table reference in the statement is fully qualified (`database.table`). + +SQL input box: a single table-model SELECT statement. The following macros are expanded by the plugin before the statement is sent: + +| Macro | Expands to | +| ----- | ---------- | +| `$__timeFilter(col)` | `(col >= AND col <= )`; `col` defaults to `time` when omitted | +| `$__timeFrom`, `$__timeFrom()` | the panel range start as an epoch integer | +| `$__timeTo`, `$__timeTo()` | the panel range end as an epoch integer | + +FORMAT option: + +* `Time series` (default): rows are sorted by the first TIMESTAMP column, and a result that contains tag/string columns next to numeric columns is pivoted into one series per tag combination — so a query returning several devices renders as separate lines in a time-series panel. Note that the pivot requires timestamps without nulls; results are sorted by the plugin, so an `ORDER BY` clause is not needed. +* `Table`: rows are returned exactly as the server sent them (use this with the table panel, or when your `ORDER BY` should be preserved). + +Result columns are typed from the response's `data_types`: TIMESTAMP columns become Grafana time fields, INT32/INT64 become integers, FLOAT/DOUBLE become floats, BOOLEAN becomes booleans, and TEXT/STRING/DATE/BLOB render as strings. + +If the server runs with a non-default `timestamp_precision` (`us` or `ns`), set the same value in the data source's `time precision` option — it controls both how TIMESTAMP values are interpreted and what the time macros expand to. + +The REST service caps query results at `rest_query_default_row_size_limit` rows (10000 by default, configurable on the server); a query exceeding it returns an error message rather than truncated data. + #### Support for variables and template functions Both SQL: Full Customized and SQL: Drop-down List input methods support the variable and template functions of grafana. In the following example, raw input method is used, and aggregation is similar. diff --git a/connectors/grafana-plugin/pkg/plugin/plugin.go b/connectors/grafana-plugin/pkg/plugin/plugin.go index a65e987b..dab92aa2 100644 --- a/connectors/grafana-plugin/pkg/plugin/plugin.go +++ b/connectors/grafana-plugin/pkg/plugin/plugin.go @@ -70,16 +70,17 @@ func ApacheIoTDBDatasource(ctx context.Context, d backend.DataSourceInstanceSett if password, exists := d.DecryptedSecureJSONData["password"]; exists { authorization = "Basic " + base64.StdEncoding.EncodeToString([]byte(dm.Username+":"+password)) } - return &IoTDBDataSource{CallResourceHandler: iotdbResourceHandler(authorization, httpClient), Username: dm.Username, Ulr: dm.Url, httpClient: httpClient}, nil + return &IoTDBDataSource{CallResourceHandler: iotdbResourceHandler(authorization, httpClient), Username: dm.Username, Ulr: dm.Url, TimestampPrecision: dm.TimestampPrecision, httpClient: httpClient}, nil } // SampleDatasource is an example datasource which can respond to data queries, reports // its health and has streaming skills. type IoTDBDataSource struct { backend.CallResourceHandler - Username string - Ulr string - httpClient *http.Client + Username string + Ulr string + TimestampPrecision string + httpClient *http.Client } // Dispose here tells plugin SDK that plugin wants to clean up resources when a new instance @@ -113,6 +114,9 @@ func (d *IoTDBDataSource) QueryData(ctx context.Context, req *backend.QueryDataR type dataSourceModel struct { Username string `json:"username"` Url string `json:"url"` + // TimestampPrecision mirrors the server's timestamp_precision property + // (ms, us or ns; ms when unset). Used by the table-model mode. + TimestampPrecision string `json:"timestampPrecision"` } type groupBy struct { @@ -134,6 +138,9 @@ type queryParam struct { FillClauses string `json:"fillClauses"` GroupBy groupBy `json:"groupBy"` Hide bool `json:"hide"` + Database string `json:"database"` + Sql string `json:"sql"` + Format string `json:"format"` } type QueryDataReq struct { @@ -200,6 +207,10 @@ func verifyQuery(query backend.DataQuery) (qp *queryParam, errMsg string) { return nil, "Input error, FROM is required" } } + } else if qp.SqlType == TableModelSqlType { + if strings.TrimSpace(qp.Sql) == "" { + return nil, "Input error, SQL is required" + } } else { return nil, "none" } @@ -231,6 +242,10 @@ func (d *IoTDBDataSource) query(cxt context.Context, pCtx backend.PluginContext, qp.StartTime = query.TimeRange.From.UnixNano() / 1000000 qp.EndTime = query.TimeRange.To.UnixNano() / 1000000 + if qp.SqlType == TableModelSqlType { + return d.queryTableModel(cxt, qp, authorization) + } + if qp.SqlType == "SQL: Drop-down List" { qp.Control = "" var expressions []string = qp.Paths[len(qp.Paths)-1:] diff --git a/connectors/grafana-plugin/pkg/plugin/table_query.go b/connectors/grafana-plugin/pkg/plugin/table_query.go new file mode 100644 index 00000000..199f2000 --- /dev/null +++ b/connectors/grafana-plugin/pkg/plugin/table_query.go @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package plugin + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/backend/log" + "github.com/grafana/grafana-plugin-sdk-go/data" +) + +// TableModelSqlType is the QueryEditor mode that sends standard table-model SQL +// straight to IoTDB's table REST interface, as opposed to the tree-model modes +// ("SQL: Full Customized" / "SQL: Drop-down List") that build root.* paths. +const TableModelSqlType = "SQL: Table Model" + +// Result formats for the table-model mode. Time series (the default) sorts +// rows ascending by the first TIMESTAMP column and pivots a long-shaped result +// (time + tag columns + value columns) into one labeled series per tag +// combination; Table returns the rows exactly as the server sent them. +const ( + tableFormatTimeSeries = "Time series" + tableFormatTable = "Table" +) + +// tableQueryPath is IoTDB's table-model query endpoint (see the /rest/table/v1 +// RestApi). It accepts a standard SQL statement plus an optional database +// context and returns a column-major QueryDataSet. +const tableQueryPath = "/rest/table/v1/query" + +// tableQueryReq is the request body for tableQueryPath. Field names match the +// generated table/v1 SQL model (database / sql / row_limit). +type tableQueryReq struct { + Database string `json:"database,omitempty"` + Sql string `json:"sql"` + RowLimit *int `json:"row_limit,omitempty"` +} + +// tableQueryDataSet mirrors the table/v1 QueryDataSet response. Its values are +// ROW-major (values[row][col]) because the table endpoint transposes before +// serializing, unlike the tree-model endpoints. snake_case JSON. On failure +// IoTDB returns an ExecutionStatus instead, whose code/message are captured +// here so a non-zero code surfaces as an error. +type tableQueryDataSet struct { + ColumnNames []string `json:"column_names"` + DataTypes []string `json:"data_types"` + Values [][]interface{} `json:"values"` + Code int32 `json:"code"` + Message string `json:"message"` +} + +// timeFilterRe matches Grafana's $__timeFilter(column) macro; the column is +// optional and defaults to "time". One level of nested parentheses is allowed +// so expressions like $__timeFilter(cast(x)) survive intact. +var timeFilterRe = regexp.MustCompile(`\$__timeFilter\(\s*((?:[^()]|\([^()]*\))*?)\s*\)`) + +// timeFromRe / timeToRe match $__timeFrom / $__timeTo, with or without the +// trailing () that Grafana's SQL data sources use ($__timeFrom()). +var ( + timeFromRe = regexp.MustCompile(`\$__timeFrom\b(?:\s*\(\s*\))?`) + timeToRe = regexp.MustCompile(`\$__timeTo\b(?:\s*\(\s*\))?`) +) + +// timestampUnits maps the datasource's timestampPrecision option (which must +// match the server's timestamp_precision property) to conversion factors: +// unitsPerMs scales the panel's epoch-ms range into server units for the time +// macros, nsPerUnit scales raw TIMESTAMP values into nanoseconds for Grafana. +func timestampUnits(precision string) (unitsPerMs int64, nsPerUnit int64) { + switch strings.TrimSpace(strings.ToLower(precision)) { + case "us": + return 1000, int64(time.Microsecond) + case "ns": + return 1000000, 1 + default: // ms is IoTDB's default timestamp_precision + return 1, int64(time.Millisecond) + } +} + +// expandTableMacros rewrites the Grafana time macros a dashboard author can put +// in table-model SQL into concrete bounds for the panel's range: +// +// $__timeFilter(col) -> (col >= AND col <= ) +// $__timeFrom[()] -> +// $__timeTo[()] -> +// +// Bounds are epoch values in the server's timestamp precision (milliseconds +// unless the datasource says otherwise), matching how IoTDB compares integer +// literals against TIMESTAMP columns. +func expandTableMacros(sql string, start int64, end int64) string { + from := strconv.FormatInt(start, 10) + to := strconv.FormatInt(end, 10) + sql = timeFilterRe.ReplaceAllStringFunc(sql, func(m string) string { + col := strings.TrimSpace(timeFilterRe.FindStringSubmatch(m)[1]) + if col == "" { + col = "time" + } + return "(" + col + " >= " + from + " AND " + col + " <= " + to + ")" + }) + sql = timeFromRe.ReplaceAllString(sql, from) + sql = timeToRe.ReplaceAllString(sql, to) + return sql +} + +// queryTableModel runs a table-model SQL query against IoTDB's table REST +// endpoint and turns the column-major QueryDataSet into a Grafana data frame. +func (d *IoTDBDataSource) queryTableModel(ctx context.Context, qp *queryParam, authorization string) backend.DataResponse { + response := backend.DataResponse{} + + // The panel range arrives in epoch ms; the server compares integer time + // literals (and returns TIMESTAMP values) in its own configured precision. + unitsPerMs, nsPerUnit := timestampUnits(d.TimestampPrecision) + + sql := expandTableMacros(qp.Sql, qp.StartTime*unitsPerMs, qp.EndTime*unitsPerMs) + reqBody := tableQueryReq{Database: qp.Database, Sql: sql} + qpJson, err := json.Marshal(reqBody) + if err != nil { + response.Error = err + return response + } + + dataSourceUrl := DataSourceUrlHandler(d.Ulr) + request, err := http.NewRequestWithContext(ctx, http.MethodPost, dataSourceUrl+tableQueryPath, bytes.NewReader(qpJson)) + if err != nil { + response.Error = err + return response + } + request.Header.Set("Content-Type", "application/json") + request.Header.Add("Authorization", authorization) + + rsp, err := d.httpClient.Do(request) + if err != nil { + response.Error = errors.New("Data source is not working properly") + log.DefaultLogger.Error("Data source is not working properly", "err", err) + return response + } + defer rsp.Body.Close() + + body, err := io.ReadAll(rsp.Body) + if err != nil { + response.Error = errors.New("Data source is not working properly") + log.DefaultLogger.Error("Failed to read response body", "err", err) + return response + } + + dataSet, err := parseTableQueryResponse(body) + if err != nil { + response.Error = err + log.DefaultLogger.Error("Parsing JSON error", "err", err) + return response + } + if dataSet.Code > 0 { + response.Error = errors.New(dataSet.Message) + log.DefaultLogger.Error(dataSet.Message) + return response + } + + response.Frames = append(response.Frames, buildTableResponseFrame(dataSet, qp.Format, nsPerUnit)) + return response +} + +// buildTableResponseFrame turns a decoded dataset into the response frame, +// honoring the query's format. In the default Time series format the rows are +// sorted ascending by the first TIMESTAMP column and a long-shaped result +// (time + string tag columns + value columns) is pivoted into one labeled +// series per tag combination, so a multi-device query renders as separate +// lines instead of one interleaved series; when the pivot does not apply (or +// fails, e.g. on a null timestamp) the plain frame is returned. The Table +// format preserves the server's row order untouched. +func buildTableResponseFrame(dataSet *tableQueryDataSet, format string, nsPerUnit int64) *data.Frame { + // Anything that is not explicitly the Table format gets the default + // time-series treatment, including queries saved before FORMAT existed. + isTimeSeries := !strings.EqualFold(format, tableFormatTable) + if isTimeSeries { + sortRowsByFirstTimestamp(dataSet) + } + frame := buildTableFrame(dataSet, nsPerUnit) + if isTimeSeries && frame.TimeSeriesSchema().Type == data.TimeSeriesTypeLong { + if wide, err := data.LongToWide(frame, nil); err == nil { + frame = wide + } + } + return frame +} + +// sortRowsByFirstTimestamp stably sorts the row-major values ascending by the +// first TIMESTAMP column, which both time-series rendering and the long-to-wide +// pivot require. Rows whose time cell is null sort last. +func sortRowsByFirstTimestamp(dataSet *tableQueryDataSet) { + timeCol := -1 + for i, t := range dataSet.DataTypes { + if strings.EqualFold(t, "TIMESTAMP") { + timeCol = i + break + } + } + if timeCol < 0 { + return + } + sort.SliceStable(dataSet.Values, func(a, b int) bool { + va, okA := rowTimeAt(dataSet.Values[a], timeCol) + vb, okB := rowTimeAt(dataSet.Values[b], timeCol) + if okA != okB { + return okA + } + return okA && va < vb + }) +} + +func rowTimeAt(row []interface{}, col int) (int64, bool) { + if col >= len(row) { + return 0, false + } + return toInt64(row[col]) +} + +// parseTableQueryResponse unmarshals a table-model query response, surfacing an +// error object (code/message) as a Go error when the request did not succeed. +// It decodes with UseNumber so that INT64/TIMESTAMP values beyond 2^53 keep +// their precision (a plain interface{} decode would coerce them to float64). +func parseTableQueryResponse(body []byte) (*tableQueryDataSet, error) { + var dataSet tableQueryDataSet + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + if err := decoder.Decode(&dataSet); err != nil { + return nil, errors.New("Parsing JSON error") + } + return &dataSet, nil +} + +// buildTableFrame converts a row-major QueryDataSet into a Grafana frame, one +// typed field per column driven by the response's data_types (so we no longer +// have to guess types the way the tree-model path does). The endpoint returns +// values row-major (values[row][col]); each output field gathers a single +// column across all rows, so every field ends up the same length (the row +// count) that Grafana requires. +func buildTableFrame(dataSet *tableQueryDataSet, nsPerUnit int64) *data.Frame { + frame := data.NewFrame("response") + rowCount := len(dataSet.Values) + for col := 0; col < len(dataSet.ColumnNames); col++ { + name := dataSet.ColumnNames[col] + dataType := "" + if col < len(dataSet.DataTypes) { + dataType = dataSet.DataTypes[col] + } + columnValues := make([]interface{}, rowCount) + for row := 0; row < rowCount; row++ { + if col < len(dataSet.Values[row]) { + columnValues[row] = dataSet.Values[row][col] + } + } + frame.Fields = append(frame.Fields, buildTableField(name, dataType, columnValues, nsPerUnit)) + } + return frame +} + +// buildTableField turns one column's raw JSON values into a typed, nullable +// Grafana field selected by the IoTDB data type. Numbers arrive as json.Number +// (the body is decoded with UseNumber), so integer, timestamp and float columns +// are coerced through toInt64 / toFloat64. TIMESTAMP values are raw epoch +// ticks in the server's precision; nsPerUnit scales them to nanoseconds. +func buildTableField(name string, dataType string, values []interface{}, nsPerUnit int64) *data.Field { + switch strings.ToUpper(dataType) { + case "TIMESTAMP": + out := make([]*time.Time, len(values)) + for i, v := range values { + if ticks, ok := toInt64(v); ok { + t := time.Unix(0, ticks*nsPerUnit) + out[i] = &t + } + } + return data.NewField(name, nil, out) + case "INT32", "INT64": + out := make([]*int64, len(values)) + for i, v := range values { + if n, ok := toInt64(v); ok { + value := n + out[i] = &value + } + } + return data.NewField(name, nil, out) + case "FLOAT", "DOUBLE": + out := make([]*float64, len(values)) + for i, v := range values { + if f, ok := toFloat64(v); ok { + value := f + out[i] = &value + } + } + return data.NewField(name, nil, out) + case "BOOLEAN": + out := make([]*bool, len(values)) + for i, v := range values { + if b, ok := v.(bool); ok { + value := b + out[i] = &value + } + } + return data.NewField(name, nil, out) + default: + // TEXT, STRING, BLOB and anything unrecognised render as strings. + out := make([]*string, len(values)) + for i, v := range values { + if v == nil { + continue + } + if s, ok := v.(string); ok { + value := s + out[i] = &value + } else { + value := toString(v) + out[i] = &value + } + } + return data.NewField(name, nil, out) + } +} + +// toInt64 coerces a JSON-decoded numeric value to int64. encoding/json decodes +// numbers into float64 by default; json.Number is handled too for callers that +// opt into it. +func toInt64(v interface{}) (int64, bool) { + switch n := v.(type) { + case float64: + return int64(n), true + case json.Number: + if i, err := n.Int64(); err == nil { + return i, true + } + if f, err := n.Float64(); err == nil { + return int64(f), true + } + case int64: + return n, true + } + return 0, false +} + +// toFloat64 coerces a JSON-decoded numeric value to float64, handling both the +// float64 (plain decode) and json.Number (UseNumber decode) representations. +func toFloat64(v interface{}) (float64, bool) { + switch n := v.(type) { + case float64: + return n, true + case json.Number: + if f, err := n.Float64(); err == nil { + return f, true + } + } + return 0, false +} + +// toString renders a non-string scalar for a text column without losing it. +func toString(v interface{}) string { + switch value := v.(type) { + case string: + return value + case float64: + return strconv.FormatFloat(value, 'f', -1, 64) + case bool: + return strconv.FormatBool(value) + case json.Number: + return value.String() + default: + b, err := json.Marshal(v) + if err != nil { + return "" + } + return string(b) + } +} diff --git a/connectors/grafana-plugin/pkg/plugin/table_query_test.go b/connectors/grafana-plugin/pkg/plugin/table_query_test.go new file mode 100644 index 00000000..0f521750 --- /dev/null +++ b/connectors/grafana-plugin/pkg/plugin/table_query_test.go @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package plugin + +import ( + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" +) + +// msNs is the TIMESTAMP tick-to-nanosecond factor for a default (ms) server. +const msNs = int64(time.Millisecond) + +func TestExpandTableMacros(t *testing.T) { + const from int64 = 1000 + const to int64 = 2000 + + cases := []struct { + name string + in string + want string + }{ + { + name: "timeFilter with explicit column", + in: "SELECT time, s0 FROM db.t WHERE $__timeFilter(time)", + want: "SELECT time, s0 FROM db.t WHERE (time >= 1000 AND time <= 2000)", + }, + { + name: "timeFilter defaults to time column when empty", + in: "SELECT * FROM db.t WHERE $__timeFilter()", + want: "SELECT * FROM db.t WHERE (time >= 1000 AND time <= 2000)", + }, + { + name: "timeFrom and timeTo", + in: "SELECT * FROM db.t WHERE time >= $__timeFrom AND time <= $__timeTo", + want: "SELECT * FROM db.t WHERE time >= 1000 AND time <= 2000", + }, + { + name: "no macros is unchanged", + in: "SELECT * FROM db.t", + want: "SELECT * FROM db.t", + }, + { + name: "function form timeFrom and timeTo", + in: "SELECT * FROM db.t WHERE time >= $__timeFrom() AND time <= $__timeTo( )", + want: "SELECT * FROM db.t WHERE time >= 1000 AND time <= 2000", + }, + { + name: "timeFilter with one nested paren level", + in: "SELECT * FROM db.t WHERE $__timeFilter(cast(x))", + want: "SELECT * FROM db.t WHERE (cast(x) >= 1000 AND cast(x) <= 2000)", + }, + { + name: "longer identifier is not mangled", + in: "SELECT $__timeFromage FROM db.t", + want: "SELECT $__timeFromage FROM db.t", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := expandTableMacros(c.in, from, to) + if got != c.want { + t.Fatalf("expandTableMacros() = %q, want %q", got, c.want) + } + }) + } +} + +func TestParseTableQueryResponseError(t *testing.T) { + body := []byte(`{"code":500,"message":"boom"}`) + ds, err := parseTableQueryResponse(body) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + if ds.Code != 500 || ds.Message != "boom" { + t.Fatalf("error status not parsed: code=%d message=%q", ds.Code, ds.Message) + } +} + +func TestParseTableQueryResponseInvalidJSON(t *testing.T) { + if _, err := parseTableQueryResponse([]byte("not json")); err == nil { + t.Fatalf("expected an error for malformed JSON") + } +} + +// TestBuildTableFrameRowMajor pins the response orientation: the table endpoint +// serialises values ROW-major (values[row][col]), so a field must gather a +// single column across every row. The two-row fixture below mirrors the shape +// asserted by IoTDB's own IoTDBRestServiceIT.testQuery and would fail if the +// values were read column-major. +func TestBuildTableFrameRowMajor(t *testing.T) { + ds := &tableQueryDataSet{ + ColumnNames: []string{"time", "i", "d", "b", "s"}, + DataTypes: []string{"TIMESTAMP", "INT64", "DOUBLE", "BOOLEAN", "TEXT"}, + Values: [][]interface{}{ + {float64(1600000000000), float64(42), float64(3.5), true, "hello"}, // row 0 + {float64(1600000001000), float64(43), float64(4.5), false, "world"}, // row 1 + }, + } + + frame := buildTableFrame(ds, msNs) + if len(frame.Fields) != 5 { + t.Fatalf("expected 5 fields, got %d", len(frame.Fields)) + } + for _, f := range frame.Fields { + if f.Len() != 2 { + t.Fatalf("field %q has len %d, want 2 (row count)", f.Name, f.Len()) + } + } + + // time column across both rows -> *time.Time + if v, ok := frame.Fields[0].At(0).(*time.Time); !ok || v == nil || + !v.Equal(time.Unix(0, 1600000000000*int64(time.Millisecond))) { + t.Fatalf("time[0] = %#v", frame.Fields[0].At(0)) + } + if v, ok := frame.Fields[0].At(1).(*time.Time); !ok || v == nil || + !v.Equal(time.Unix(0, 1600000001000*int64(time.Millisecond))) { + t.Fatalf("time[1] = %#v", frame.Fields[0].At(1)) + } + // int column + if v := frame.Fields[1].At(0).(*int64); *v != 42 { + t.Fatalf("i[0] = %d, want 42", *v) + } + if v := frame.Fields[1].At(1).(*int64); *v != 43 { + t.Fatalf("i[1] = %d, want 43", *v) + } + // double column + if v := frame.Fields[2].At(1).(*float64); *v != 4.5 { + t.Fatalf("d[1] = %v, want 4.5", *v) + } + // boolean column + if v := frame.Fields[3].At(1).(*bool); *v != false { + t.Fatalf("b[1] = %v, want false", *v) + } + // text column + if v := frame.Fields[4].At(1).(*string); *v != "world" { + t.Fatalf("s[1] = %q, want world", *v) + } +} + +// TestParseAndBuildPreservesInt64Precision decodes a real JSON body end-to-end +// and checks that an INT64 above 2^53 is not corrupted (which a plain float64 +// decode would do). 9007199254740993 == 2^53 + 1 is not representable in +// float64, so this fails unless the decoder preserves integer precision. +func TestParseAndBuildPreservesInt64Precision(t *testing.T) { + body := []byte(`{"column_names":["v"],"data_types":["INT64"],"values":[[9007199254740993]]}`) + + ds, err := parseTableQueryResponse(body) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + frame := buildTableFrame(ds, msNs) + if len(frame.Fields) != 1 || frame.Fields[0].Len() != 1 { + t.Fatalf("unexpected frame shape: %d fields", len(frame.Fields)) + } + got := frame.Fields[0].At(0).(*int64) + if got == nil || *got != 9007199254740993 { + t.Fatalf("int64 precision lost: got %v, want 9007199254740993", got) + } +} + +func TestBuildTableFieldNullsBecomeNilPointers(t *testing.T) { + f := buildTableField("i", "INT64", []interface{}{float64(1), nil, float64(3)}, msNs) + if f.Len() != 3 { + t.Fatalf("expected 3 values, got %d", f.Len()) + } + if v, ok := f.At(1).(*int64); !ok || v != nil { + t.Fatalf("null value should be a nil *int64, got %#v", f.At(1)) + } + if v, ok := f.At(0).(*int64); !ok || v == nil || *v != 1 { + t.Fatalf("first value = %#v, want *int64(1)", f.At(0)) + } +} + +func TestBuildTableFieldUnknownTypeRendersString(t *testing.T) { + f := buildTableField("x", "SOMETHING_NEW", []interface{}{float64(7), "raw", nil}, msNs) + if f.Len() != 3 { + t.Fatalf("expected 3 values, got %d", f.Len()) + } + if v, ok := f.At(0).(*string); !ok || v == nil || *v != "7" { + t.Fatalf("numeric coerced to string = %#v, want *string(7)", f.At(0)) + } + if v, ok := f.At(1).(*string); !ok || v == nil || *v != "raw" { + t.Fatalf("string value = %#v, want *string(raw)", f.At(1)) + } + if v, ok := f.At(2).(*string); !ok || v != nil { + t.Fatalf("null value should be nil *string, got %#v", f.At(2)) + } +} + +// TestBuildTableFrameRaggedRowIsSafe checks that a row shorter than the column +// header (a missing trailing cell) does not panic and yields equal-length, +// null-padded fields. +func TestBuildTableFrameRaggedRowIsSafe(t *testing.T) { + ds := &tableQueryDataSet{ + ColumnNames: []string{"time", "s0"}, + DataTypes: []string{"TIMESTAMP", "TEXT"}, + Values: [][]interface{}{ + {float64(1), "a"}, // full row + {float64(2)}, // ragged: missing s0 + }, + } + frame := buildTableFrame(ds, msNs) + if len(frame.Fields) != 2 { + t.Fatalf("expected 2 fields, got %d", len(frame.Fields)) + } + if frame.Fields[0].Len() != 2 || frame.Fields[1].Len() != 2 { + t.Fatalf("fields must be equal length (2), got %d and %d", frame.Fields[0].Len(), frame.Fields[1].Len()) + } + if v, ok := frame.Fields[1].At(1).(*string); !ok || v != nil { + t.Fatalf("missing ragged cell should be nil *string, got %#v", frame.Fields[1].At(1)) + } +} + +// TestTimestampUnits pins the precision option's conversion factors: the +// panel's ms range must be scaled INTO server units for the macros, and raw +// TIMESTAMP ticks must be scaled into nanoseconds for rendering. +func TestTimestampUnits(t *testing.T) { + cases := []struct { + precision string + unitsPerMs int64 + nsPerUnit int64 + }{ + {"", 1, int64(time.Millisecond)}, + {"ms", 1, int64(time.Millisecond)}, + {"us", 1000, int64(time.Microsecond)}, + {"ns", 1000000, 1}, + {" NS ", 1000000, 1}, + {"garbage", 1, int64(time.Millisecond)}, + } + for _, c := range cases { + unitsPerMs, nsPerUnit := timestampUnits(c.precision) + if unitsPerMs != c.unitsPerMs || nsPerUnit != c.nsPerUnit { + t.Fatalf("timestampUnits(%q) = (%d, %d), want (%d, %d)", + c.precision, unitsPerMs, nsPerUnit, c.unitsPerMs, c.nsPerUnit) + } + } +} + +// TestBuildTableFieldTimestampPrecision renders the same instant sent by a +// us- and an ns-precision server. With the old hard-coded ms conversion the +// microsecond value would land in January 1970, off by 1000x. +func TestBuildTableFieldTimestampPrecision(t *testing.T) { + want := time.Unix(0, 1600000000000*int64(time.Millisecond)) // 2020-09-13T12:26:40Z + + _, usNs := timestampUnits("us") + f := buildTableField("time", "TIMESTAMP", []interface{}{float64(1600000000000000)}, usNs) + if v, ok := f.At(0).(*time.Time); !ok || v == nil || !v.Equal(want) { + t.Fatalf("us tick rendered as %#v, want %v", f.At(0), want) + } + + _, nsNs := timestampUnits("ns") + f = buildTableField("time", "TIMESTAMP", []interface{}{float64(1600000000000000000)}, nsNs) + if v, ok := f.At(0).(*time.Time); !ok || v == nil || !v.Equal(want) { + t.Fatalf("ns tick rendered as %#v, want %v", f.At(0), want) + } +} + +// TestBuildTableResponseFrameLongToWide pins the multi-device path: a long +// result (time + tag + value), even arriving unsorted, must come back as one +// labeled series per tag value so a time-series panel draws separate lines. +func TestBuildTableResponseFrameLongToWide(t *testing.T) { + ds := &tableQueryDataSet{ + ColumnNames: []string{"time", "device", "temperature"}, + DataTypes: []string{"TIMESTAMP", "STRING", "DOUBLE"}, + Values: [][]interface{}{ + {float64(2000), "d2", float64(22.5)}, + {float64(1000), "d1", float64(11.0)}, + {float64(1000), "d2", float64(21.0)}, + {float64(2000), "d1", float64(12.0)}, + }, + } + + frame := buildTableResponseFrame(ds, "", msNs) + if len(frame.Fields) != 3 { + t.Fatalf("expected time + one series per device (3 fields), got %d", len(frame.Fields)) + } + if frame.Fields[0].Len() != 2 { + t.Fatalf("expected 2 wide rows, got %d", frame.Fields[0].Len()) + } + byDevice := map[string][]float64{} + for _, f := range frame.Fields[1:] { + dev := f.Labels["device"] + if dev == "" { + t.Fatalf("value field %q has no device label: %v", f.Name, f.Labels) + } + var vals []float64 + for i := 0; i < f.Len(); i++ { + v, ok := f.At(i).(*float64) + if !ok || v == nil { + t.Fatalf("field %q row %d = %#v, want *float64", f.Name, i, f.At(i)) + } + vals = append(vals, *v) + } + byDevice[dev] = vals + } + if v := byDevice["d1"]; len(v) != 2 || v[0] != 11.0 || v[1] != 12.0 { + t.Fatalf("d1 series = %v, want [11 12]", v) + } + if v := byDevice["d2"]; len(v) != 2 || v[0] != 21.0 || v[1] != 22.5 { + t.Fatalf("d2 series = %v, want [21 22.5]", v) + } +} + +// TestBuildTableResponseFrameTableFormatPreservesOrder pins that the Table +// format neither re-sorts rows (the user's ORDER BY wins) nor pivots tags +// into labels. +func TestBuildTableResponseFrameTableFormatPreservesOrder(t *testing.T) { + ds := &tableQueryDataSet{ + ColumnNames: []string{"time", "device", "temperature"}, + DataTypes: []string{"TIMESTAMP", "STRING", "DOUBLE"}, + Values: [][]interface{}{ + {float64(2000), "d2", float64(22.5)}, + {float64(1000), "d1", float64(11.0)}, + }, + } + + frame := buildTableResponseFrame(ds, tableFormatTable, msNs) + if len(frame.Fields) != 3 { + t.Fatalf("expected 3 plain fields, got %d", len(frame.Fields)) + } + first, ok := frame.Fields[0].At(0).(*time.Time) + if !ok || first == nil || !first.Equal(time.Unix(0, 2000*msNs)) { + t.Fatalf("row order changed: first time = %#v, want t=2000ms", frame.Fields[0].At(0)) + } + if _, ok := frame.Fields[1].At(0).(*string); !ok { + t.Fatalf("tag column should stay a plain string field in Table format") + } +} + +// TestBuildTableResponseFrameNullTimeFallsBack pins the safety net: when the +// long-to-wide pivot cannot apply (here: a null timestamp), the plain frame is +// returned instead of an error or a panic. +func TestBuildTableResponseFrameNullTimeFallsBack(t *testing.T) { + ds := &tableQueryDataSet{ + ColumnNames: []string{"time", "device", "temperature"}, + DataTypes: []string{"TIMESTAMP", "STRING", "DOUBLE"}, + Values: [][]interface{}{ + {float64(1000), "d1", float64(11.0)}, + {nil, "d2", float64(21.0)}, + }, + } + + frame := buildTableResponseFrame(ds, tableFormatTimeSeries, msNs) + if len(frame.Fields) != 3 { + t.Fatalf("expected plain 3-field fallback frame, got %d fields", len(frame.Fields)) + } + if frame.TimeSeriesSchema().Type != data.TimeSeriesTypeLong { + t.Fatalf("fallback frame should still be the long-shaped original") + } +} diff --git a/connectors/grafana-plugin/src/ConfigEditor.tsx b/connectors/grafana-plugin/src/ConfigEditor.tsx index fed474dd..917b422e 100644 --- a/connectors/grafana-plugin/src/ConfigEditor.tsx +++ b/connectors/grafana-plugin/src/ConfigEditor.tsx @@ -15,14 +15,20 @@ * limitations under the License. */ import React, { ChangeEvent, PureComponent } from 'react'; -import { InlineField, Input, SecretInput } from '@grafana/ui'; -import { DataSourcePluginOptionsEditorProps } from '@grafana/data'; +import { InlineField, Input, SecretInput, Select } from '@grafana/ui'; +import { DataSourcePluginOptionsEditorProps, SelectableValue } from '@grafana/data'; import { IoTDBOptions, IoTDBSecureJsonData } from './types'; interface Props extends DataSourcePluginOptionsEditorProps {} interface State {} +const timestampPrecisions: Array> = [ + { label: 'ms', value: 'ms' }, + { label: 'us', value: 'us' }, + { label: 'ns', value: 'ns' }, +]; + export class ConfigEditor extends PureComponent { onURLChange = (event: ChangeEvent) => { const { onOptionsChange, options } = this.props; @@ -42,6 +48,15 @@ export class ConfigEditor extends PureComponent { onOptionsChange({ ...options, jsonData }); }; + onTimestampPrecisionChange = (option: SelectableValue) => { + const { onOptionsChange, options } = this.props; + const jsonData = { + ...options.jsonData, + timestampPrecision: option.value ?? 'ms', + }; + onOptionsChange({ ...options, jsonData }); + }; + // Secure field (only sent to the backend) onPasswordChange = (event: ChangeEvent) => { const { onOptionsChange, options } = this.props; @@ -101,6 +116,18 @@ export class ConfigEditor extends PureComponent { onChange={this.onPasswordChange} /> + + + + +
+ +