From 4b0c84521a79e6c37b7c3f8e6c6b9e5ec0f1ca9d Mon Sep 17 00:00:00 2001 From: hardy <2545433047@qq.com> Date: Tue, 1 Sep 2026 09:26:41 +0800 Subject: [PATCH 1/2] feat: TAP-12467 ingest task alerts and send data integrity emails --- packages/api/src/core/alarm.ts | 1 + packages/business/src/locale/lang/en.js | 1 + packages/business/src/locale/lang/zh-CN.js | 1 + packages/business/src/locale/lang/zh-TW.js | 1 + packages/business/src/shared/alarm.ts | 3 ++ .../src/views/setting/EmailTemplateDialog.vue | 37 ++++++++++++++++ packages/dag/src/components/SettingsPanel.vue | 42 ++++++++++++------- packages/dag/src/constants.ts | 14 +++++++ packages/dag/src/locale/lang/en.js | 2 + packages/dag/src/locale/lang/zh-CN.js | 2 + packages/dag/src/locale/lang/zh-TW.js | 2 + packages/dag/src/stores/dataflow.store.ts | 24 +++++++---- packages/i18n/src/locale/lang/en.js | 4 ++ packages/i18n/src/locale/lang/zh-CN.js | 4 ++ packages/i18n/src/locale/lang/zh-TW.js | 4 ++ 15 files changed, 118 insertions(+), 24 deletions(-) diff --git a/packages/api/src/core/alarm.ts b/packages/api/src/core/alarm.ts index d03711d6fa..a2bc46602e 100644 --- a/packages/api/src/core/alarm.ts +++ b/packages/api/src/core/alarm.ts @@ -32,6 +32,7 @@ export enum Key { TaskIncrementDelay = 'TASK_INCREMENT_DELAY', TaskIncrementStart = 'TASK_INCREMENT_START', TaskDdlWarning = 'TASK_DDL_WARNING', + TaskDataIntegrityRisk = 'TASK_DATA_INTEGRITY_RISK', TaskInspectError = 'TASK_INSPECT_ERROR', TaskStatusError = 'TASK_STATUS_ERROR', TaskStatusStop = 'TASK_STATUS_STOP', diff --git a/packages/business/src/locale/lang/en.js b/packages/business/src/locale/lang/en.js index 9ab659f971..b09cfa0edf 100644 --- a/packages/business/src/locale/lang/en.js +++ b/packages/business/src/locale/lang/en.js @@ -1772,6 +1772,7 @@ export default { packages_business_monitor_cron_tip: `Syntax: seconds * minutes * hours * days * months * weeks ? years *\nExample: 0 */1 * * * ? Run every minute 0 0 2 * * ? Run every day at 2am`, packages_business_datasource_monitor_alter: 'Data Source Monitor Alert', packages_business_task_ddl_warning: 'Task Dangerous DDL Alert', + packages_business_task_data_integrity_risk: 'Task Data Integrity Risk Alert', packages_business_import_mode: 'Handling for Name Conflicts', packages_business_import_as_copy: 'Import as Copy', packages_business_import_replace: 'Replace Existing', diff --git a/packages/business/src/locale/lang/zh-CN.js b/packages/business/src/locale/lang/zh-CN.js index bc3589ccbc..16cd20bcbb 100644 --- a/packages/business/src/locale/lang/zh-CN.js +++ b/packages/business/src/locale/lang/zh-CN.js @@ -1576,6 +1576,7 @@ export default { packages_business_monitor_cron_tip: `语法: 秒 * 分钟 * 小时 * 日 * 月 * 星期 ? 年 *\n举例：0 */1 * * * ? 每分钟运行一次 0 0 2 * * ? 每天2点运行`, packages_business_datasource_monitor_alter: '数据源监控告警', packages_business_task_ddl_warning: '任务风险DDL告警', + packages_business_task_data_integrity_risk: '任务数据完整性风险告警', packages_business_import_mode: '同名处理方式', packages_business_import_as_copy: '以副本导入', packages_business_import_replace: '覆盖导入', diff --git a/packages/business/src/locale/lang/zh-TW.js b/packages/business/src/locale/lang/zh-TW.js index 6fd27c1b06..67bdca7d11 100644 --- a/packages/business/src/locale/lang/zh-TW.js +++ b/packages/business/src/locale/lang/zh-TW.js @@ -1561,6 +1561,7 @@ export default { packages_business_monitor_cron_tip: `語法: 秒 * 分鐘 * 小時 * 日 * 月 * 星期 ? 年 *\n舉例：0 */1 * * * ? 每分鐘運行一次 0 0 2 * * ? 每天2點運行`, packages_business_datasource_monitor_alter: '數據源監控告警', packages_business_task_ddl_warning: '任務風險DDL告警', + packages_business_task_data_integrity_risk: '任務資料完整性風險告警', packages_business_import_mode: '同名處理方式', packages_business_import_as_copy: '以副本導入', packages_business_import_replace: '覆蓋導入', diff --git a/packages/business/src/shared/alarm.ts b/packages/business/src/shared/alarm.ts index c011ba3b93..7a0e01fdf1 100644 --- a/packages/business/src/shared/alarm.ts +++ b/packages/business/src/shared/alarm.ts @@ -106,6 +106,9 @@ export function getAlarmKeyMap() { ), DATASOURCE_MONITOR_ALTER: t('packages_business_datasource_monitor_alter'), TASK_DDL_WARNING: t('packages_business_task_ddl_warning'), + TASK_DATA_INTEGRITY_RISK: t( + 'packages_business_task_data_integrity_risk', + ), API_SERVER_P95_WARN: t('packages_business_api_server_p95_warn'), API_SERVER_P99_WARN: t('packages_business_api_server_p99_warn'), API_SERVER_ERROR_RATE_WARN: t( diff --git a/packages/business/src/views/setting/EmailTemplateDialog.vue b/packages/business/src/views/setting/EmailTemplateDialog.vue index c84c92dfc7..00d85958ce 100644 --- a/packages/business/src/views/setting/EmailTemplateDialog.vue +++ b/packages/business/src/views/setting/EmailTemplateDialog.vue @@ -129,6 +129,43 @@ const variablesMap = reactive({ icon: IconLucideFileText, }, ], + TASK_DATA_INTEGRITY_RISK: [ + { + name: 'taskName', + label: t('public_task_name'), + icon: IconLucideFileText, + }, + { + name: 'taskId', + label: t('public_task_id'), + icon: IconLucideHash, + }, + { + name: 'taskDesc', + label: t('public_task_desc'), + icon: IconLucideFileText, + }, + { + name: 'nodeName', + label: t('public_node_name'), + icon: IconLucideFileText, + }, + { + name: 'alertMessage', + label: t('public_alert_message'), + icon: IconLucideTriangleAlert, + }, + { + name: 'occurredTime', + label: t('public_occurred_time'), + icon: IconLucideClock, + }, + { + name: 'occurrenceCount', + label: t('public_occurrence_count'), + icon: IconLucideHash, + }, + ], DATANODE_CANNOT_CONNECT: [ { name: 'taskName', diff --git a/packages/dag/src/components/SettingsPanel.vue b/packages/dag/src/components/SettingsPanel.vue index b5d2bd4922..b629f34c32 100644 --- a/packages/dag/src/components/SettingsPanel.vue +++ b/packages/dag/src/components/SettingsPanel.vue @@ -37,7 +37,11 @@ import { import { useStore } from 'vuex' import { FormTab } from '../../../form' import * as _components from '../components/form' -import { alarmSettingKeys, type AlarmSettingKey } from '../constants' +import { + alarmSettingKeys, + alignAlarmSettings, + type AlarmSettingKey, +} from '../constants' import { useDataflowStore } from '../stores/dataflow.store' const dataflowStore = useDataflowStore() @@ -185,22 +189,27 @@ const alarmSettingSchemaConfigMap: Record< TASK_DDL_WARNING: { title: t('packages_dag_migration_alarmpanel_renwufengxianddl'), }, + TASK_DATA_INTEGRITY_RISK: { + title: t('packages_dag_migration_alarmpanel_renwushujuwanzhengxing'), + }, } -function getAlarmSettingsDefault() { - return alarmSettingKeys.map((key) => { - const config = alarmSettingSchemaConfigMap[key] +function createDefaultAlarmSetting(key: AlarmSettingKey) { + const config = alarmSettingSchemaConfigMap[key] - return { - type: 'TASK', - open: isDaas, - key, - sort: config.sort, - notify: config.notify ?? ['SYSTEM', 'EMAIL'], - interval: config.interval ?? 300, - unit: 'SECOND', - } - }) + return { + type: 'TASK', + open: isDaas, + key, + sort: config.sort, + notify: config.notify ?? ['SYSTEM', 'EMAIL'], + interval: config.interval ?? 300, + unit: 'SECOND', + } +} + +function getAlarmSettingsDefault() { + return alignAlarmSettings([], createDefaultAlarmSetting) } function getAlarmSettingSchema( @@ -724,7 +733,10 @@ function saveAlarmConfig() { updateTaskAlarm({ taskId: values.id, - alarmSettings: values.alarmSettings, + alarmSettings: alignAlarmSettings( + values.alarmSettings, + createDefaultAlarmSetting, + ), alarmRules: values.alarmRules, emailReceivers: values.emailReceivers, }) diff --git a/packages/dag/src/constants.ts b/packages/dag/src/constants.ts index 82a73fdd81..97863befd5 100644 --- a/packages/dag/src/constants.ts +++ b/packages/dag/src/constants.ts @@ -122,6 +122,20 @@ export const alarmSettingKeys = [ 'TASK_RETRY_WARN', 'TASK_SOURCE_NO_INCREMENTAL_EVENT', 'TASK_DDL_WARNING', + 'TASK_DATA_INTEGRITY_RISK', ] as const export type AlarmSettingKey = (typeof alarmSettingKeys)[number] + +export function alignAlarmSettings( + alarmSettings: T[] | null | undefined, + createDefault: (key: AlarmSettingKey) => T, +): T[] { + const byKey = new Map() + for (const setting of alarmSettings || []) { + if (setting?.key) { + byKey.set(setting.key, setting) + } + } + return alarmSettingKeys.map((key) => byKey.get(key) ?? createDefault(key)) +} diff --git a/packages/dag/src/locale/lang/en.js b/packages/dag/src/locale/lang/en.js index 8037b402df..c1a63dd4d2 100644 --- a/packages/dag/src/locale/lang/en.js +++ b/packages/dag/src/locale/lang/en.js @@ -601,6 +601,8 @@ export default { packages_dag_migration_alarmpanel_renwujiaoyanchu: 'Task validation error', packages_dag_migration_alarmpanel_renwuyunxingchu: 'Task running error alert', packages_dag_migration_alarmpanel_renwufengxianddl: 'Task risk DDL alert', + packages_dag_migration_alarmpanel_renwushujuwanzhengxing: + 'Task data integrity risk alert', packages_dag_migration_alarmpanel_jiedianpingjunchu: 'Node average processing time alert', packages_dag_migration_alarmpanel_shujuyuanxieyi: diff --git a/packages/dag/src/locale/lang/zh-CN.js b/packages/dag/src/locale/lang/zh-CN.js index 7baa9efa84..3a6bad3e6f 100644 --- a/packages/dag/src/locale/lang/zh-CN.js +++ b/packages/dag/src/locale/lang/zh-CN.js @@ -514,6 +514,8 @@ export default { packages_dag_migration_alarmpanel_renwujiaoyanchu: '任务校验出错告警', packages_dag_migration_alarmpanel_renwuyunxingchu: '任务运行出错告警', packages_dag_migration_alarmpanel_renwufengxianddl: '任务风险DDL告警', + packages_dag_migration_alarmpanel_renwushujuwanzhengxing: + '任务数据完整性风险告警', packages_dag_migration_alarmpanel_jiedianpingjunchu: '当节点的平均处理耗时', packages_dag_migration_alarmpanel_shujuyuanxieyi: '数据源协议连接耗时告警', packages_dag_migration_alarmpanel_shujuyuanwangluo: '数据源网络连接耗时告警', diff --git a/packages/dag/src/locale/lang/zh-TW.js b/packages/dag/src/locale/lang/zh-TW.js index fe33f12617..f86005ad92 100644 --- a/packages/dag/src/locale/lang/zh-TW.js +++ b/packages/dag/src/locale/lang/zh-TW.js @@ -507,6 +507,8 @@ export default { packages_dag_migration_alarmpanel_renwujiaoyanchu: '任务校验出错告警', packages_dag_migration_alarmpanel_renwuyunxingchu: '任务运行出错告警', packages_dag_migration_alarmpanel_renwufengxianddl: '任务风险DDL告警', + packages_dag_migration_alarmpanel_renwushujuwanzhengxing: + '任务数据完整性风险告警', packages_dag_migration_alarmpanel_jiedianpingjunchu: '节点平均处理耗时告警', packages_dag_migration_alarmpanel_shujuyuanxieyi: '数据源协议连接耗时告警', packages_dag_migration_alarmpanel_shujuyuanwangluo: '数据源网络连接耗时告警', diff --git a/packages/dag/src/stores/dataflow.store.ts b/packages/dag/src/stores/dataflow.store.ts index 2c8e07af80..85af87683a 100644 --- a/packages/dag/src/stores/dataflow.store.ts +++ b/packages/dag/src/stores/dataflow.store.ts @@ -21,7 +21,11 @@ import { isObject } from '@tap/shared' import { debounce, isString } from 'lodash-es' import { defineStore } from 'pinia' import { markRaw, reactive, ref, shallowRef } from 'vue' -import { alarmSettingKeys, DEFAULT_SETTINGS } from '../constants' +import { + alignAlarmSettings, + DEFAULT_SETTINGS, + type AlarmSettingKey, +} from '../constants' import { CustomProcessor } from '../nodes/extends/CustomProcessor' import { allResourceIns as resourceIns } from '../nodes/loader' @@ -77,14 +81,16 @@ function hasCycle( function sortAlarmSettings(alarmSettings?: T[]) { if (!Array.isArray(alarmSettings)) return alarmSettings - return [...alarmSettings].sort((a, b) => { - const aIndex = (alarmSettingKeys as readonly string[]).indexOf(a.key || '') - const bIndex = (alarmSettingKeys as readonly string[]).indexOf(b.key || '') - - return ( - (aIndex === -1 ? Number.MAX_SAFE_INTEGER : aIndex) - - (bIndex === -1 ? Number.MAX_SAFE_INTEGER : bIndex) - ) + return alignAlarmSettings(alarmSettings, (key: AlarmSettingKey) => { + return { + type: 'TASK', + open: isDaas, + key, + sort: 0, + notify: ['SYSTEM', 'EMAIL'], + interval: 300, + unit: 'SECOND', + } as T }) } diff --git a/packages/i18n/src/locale/lang/en.js b/packages/i18n/src/locale/lang/en.js index e819b2ab68..39c82050ab 100644 --- a/packages/i18n/src/locale/lang/en.js +++ b/packages/i18n/src/locale/lang/en.js @@ -470,6 +470,10 @@ export default { public_error_time: 'Error Time', public_error_log: 'Error Log', + public_task_id: 'Task ID', + public_alert_message: 'Alert Details', + public_occurred_time: 'Occurred Time', + public_occurrence_count: 'Occurrence Count', public_complete_time: 'Complete Time', public_cdc_time: 'CDC Time', public_stop_time: 'Stop Time', diff --git a/packages/i18n/src/locale/lang/zh-CN.js b/packages/i18n/src/locale/lang/zh-CN.js index fca8798c4d..d48bbb4ed6 100644 --- a/packages/i18n/src/locale/lang/zh-CN.js +++ b/packages/i18n/src/locale/lang/zh-CN.js @@ -466,6 +466,10 @@ export default { public_error_time: '错误时间', public_error_log: '错误日志', + public_task_id: '任务 ID', + public_alert_message: '告警详情', + public_occurred_time: '发生时间', + public_occurrence_count: '发生次数', public_complete_time: '完成时间', public_cdc_time: 'CDC 时间', public_stop_time: '停止时间', diff --git a/packages/i18n/src/locale/lang/zh-TW.js b/packages/i18n/src/locale/lang/zh-TW.js index 9ff3286f20..0d6b1f148d 100644 --- a/packages/i18n/src/locale/lang/zh-TW.js +++ b/packages/i18n/src/locale/lang/zh-TW.js @@ -463,6 +463,10 @@ export default { public_error_time: '錯誤時間', public_error_log: '錯誤日誌', + public_task_id: '任務 ID', + public_alert_message: '告警詳情', + public_occurred_time: '發生時間', + public_occurrence_count: '發生次數', public_complete_time: '完成時間', public_cdc_time: 'CDC 時間', public_stop_time: '停止時間', From c5e6d731cfc19d4a6bd4052bb05db5d6e69de395 Mon Sep 17 00:00:00 2001 From: hardy <2545433047@qq.com> Date: Fri, 11 Sep 2026 09:33:43 +0800 Subject: [PATCH 2/2] style(TAP-12467): fix prettier formatting for CI --- packages/business/src/locale/lang/zh-CN.js | 6 ++++-- packages/business/src/locale/lang/zh-TW.js | 18 ++++++++++++------ packages/business/src/shared/alarm.ts | 4 +--- packages/i18n/src/locale/lang/en.js | 3 ++- packages/i18n/src/locale/lang/zh-CN.js | 3 ++- packages/i18n/src/locale/lang/zh-TW.js | 3 ++- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/packages/business/src/locale/lang/zh-CN.js b/packages/business/src/locale/lang/zh-CN.js index 16cd20bcbb..4f35fcbe78 100644 --- a/packages/business/src/locale/lang/zh-CN.js +++ b/packages/business/src/locale/lang/zh-CN.js @@ -1594,6 +1594,8 @@ export default { public_connector_alpha_tip: '正在持续开发中，部分功能可能还在完善', packages_business_column_setting: '显示设置', packages_business_column_reset: '重置', - packages_business_api_server_connection_pool_deficiency_warn: '当任一API Server 连接池资源紧张触发告警阈值时', - packages_business_api_server_connection_pool_idle_warn: '当任一API Server 连接池连接数低于告警阈值时', + packages_business_api_server_connection_pool_deficiency_warn: + '当任一API Server 连接池资源紧张触发告警阈值时', + packages_business_api_server_connection_pool_idle_warn: + '当任一API Server 连接池连接数低于告警阈值时', } diff --git a/packages/business/src/locale/lang/zh-TW.js b/packages/business/src/locale/lang/zh-TW.js index 67bdca7d11..baa4d2dbf6 100644 --- a/packages/business/src/locale/lang/zh-TW.js +++ b/packages/business/src/locale/lang/zh-TW.js @@ -726,10 +726,14 @@ export default { '當任一API Server的工作行程記憶體使用量超過告警閾值時', packages_business_setting_alarmnotification_api_server_worker_memory_usage_alter: '當任一API Server的工作行程記憶體使用量超過通知閾值時', - packages_business_api_server_p95_warn: '當任一API Server請求耗時P95超過告警閾值時', - packages_business_api_server_p99_warn: '當任一API Server請求耗時P99超過告警閾值時', - packages_business_api_server_error_rate_warn: '當任一API Server請求錯誤率超過告警閾值時', - packages_business_task_source_no_incremental_event: '當源端已開啟心跳表，連續 60 秒未接收到增量事件時', + packages_business_api_server_p95_warn: + '當任一API Server請求耗時P95超過告警閾值時', + packages_business_api_server_p99_warn: + '當任一API Server請求耗時P99超過告警閾值時', + packages_business_api_server_error_rate_warn: + '當任一API Server請求錯誤率超過告警閾值時', + packages_business_task_source_no_incremental_event: + '當源端已開啟心跳表，連續 60 秒未接收到增量事件時', packages_business_setting_alarmnotification_dangrenwuuP: 'Agent服务啓動時', packages_business_setting_alarmnotification_dangyinqinglixian: '當引擎停止時', packages_business_setting_alarmnotification_dangyinqinghuifu: '當引擎啓動時', @@ -1579,6 +1583,8 @@ export default { public_connector_alpha_tip: '正在持續開發中，部分功能可能還在完善', packages_business_column_setting: '顯示設定', packages_business_column_reset: '重置', - packages_business_api_server_connection_pool_deficiency_warn: '當任一API Server 連接池資源緊張触发告警閾值時', - packages_business_api_server_connection_pool_idle_warn: '當任一API Server 連接池連接數低於告警閾值時', + packages_business_api_server_connection_pool_deficiency_warn: + '當任一API Server 連接池資源緊張触发告警閾值時', + packages_business_api_server_connection_pool_idle_warn: + '當任一API Server 連接池連接數低於告警閾值時', } diff --git a/packages/business/src/shared/alarm.ts b/packages/business/src/shared/alarm.ts index 7a0e01fdf1..b0c2b64978 100644 --- a/packages/business/src/shared/alarm.ts +++ b/packages/business/src/shared/alarm.ts @@ -106,9 +106,7 @@ export function getAlarmKeyMap() { ), DATASOURCE_MONITOR_ALTER: t('packages_business_datasource_monitor_alter'), TASK_DDL_WARNING: t('packages_business_task_ddl_warning'), - TASK_DATA_INTEGRITY_RISK: t( - 'packages_business_task_data_integrity_risk', - ), + TASK_DATA_INTEGRITY_RISK: t('packages_business_task_data_integrity_risk'), API_SERVER_P95_WARN: t('packages_business_api_server_p95_warn'), API_SERVER_P99_WARN: t('packages_business_api_server_p99_warn'), API_SERVER_ERROR_RATE_WARN: t( diff --git a/packages/i18n/src/locale/lang/en.js b/packages/i18n/src/locale/lang/en.js index 39c82050ab..f3a37ab0e2 100644 --- a/packages/i18n/src/locale/lang/en.js +++ b/packages/i18n/src/locale/lang/en.js @@ -692,5 +692,6 @@ export default { public_last_capture_time: 'Last Capture Time', public_alarm_date: 'Alarm Time', public_duration: 'Duration', - packages_dag_validate_shared_cdc_title: 'The following connections have not enabled Shared Mining. Running the task will cause errors.', + packages_dag_validate_shared_cdc_title: + 'The following connections have not enabled Shared Mining. Running the task will cause errors.', } diff --git a/packages/i18n/src/locale/lang/zh-CN.js b/packages/i18n/src/locale/lang/zh-CN.js index d48bbb4ed6..060eab7d2a 100644 --- a/packages/i18n/src/locale/lang/zh-CN.js +++ b/packages/i18n/src/locale/lang/zh-CN.js @@ -684,5 +684,6 @@ export default { public_last_capture_time: '最近观测时间', public_alarm_date: '告警时间', public_duration: '时长', - packages_dag_validate_shared_cdc_title: '以下连接尚未开启共享挖掘，运行任务将会出错', + packages_dag_validate_shared_cdc_title: + '以下连接尚未开启共享挖掘，运行任务将会出错', } diff --git a/packages/i18n/src/locale/lang/zh-TW.js b/packages/i18n/src/locale/lang/zh-TW.js index 0d6b1f148d..95f7cf8008 100644 --- a/packages/i18n/src/locale/lang/zh-TW.js +++ b/packages/i18n/src/locale/lang/zh-TW.js @@ -681,5 +681,6 @@ export default { public_last_capture_time: '最近觀測時間', public_alarm_date: '告警時間', public_duration: '時長', - packages_dag_validate_shared_cdc_title: '以下連接尚未開啟共享挖掘，運行任務將會出錯', + packages_dag_validate_shared_cdc_title: + '以下連接尚未開啟共享挖掘，運行任務將會出錯', }