|
@@ -1,6 +1,6 @@
|
|
|
# -*- coding:utf-8 -*-
|
|
# -*- coding:utf-8 -*-
|
|
|
"""
|
|
"""
|
|
|
-bin/cdc-monitor.py 单测:复制槽判定 / Flink 作业存活 / 消息渲染 / main 串联。
|
|
|
|
|
|
|
+bin/cdc-monitor.py 单测:复制槽判定 / Flink 作业存活 / 告警与状态渲染 / main 串联。
|
|
|
|
|
|
|
|
不连真 PG(pgdb.connect + query 走替身)、不打真 JobManager(urlopen 走替身)、
|
|
不连真 PG(pgdb.connect + query 走替身)、不打真 JobManager(urlopen 走替身)、
|
|
|
不发真消息(Alerter 走替身)。
|
|
不发真消息(Alerter 走替身)。
|
|
@@ -43,6 +43,12 @@ def _stub_pg(monkeypatch, rows, conn=None):
|
|
|
return conn
|
|
return conn
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+def _slot_alerts(monkeypatch, rows):
|
|
|
|
|
+ """只取判定结果,facts 另有用例覆盖。"""
|
|
|
|
|
+ _stub_pg(monkeypatch, rows)
|
|
|
|
|
+ return MON.check_slots('postgresql/x', LAG_CRIT)[1]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def _stub_flink(monkeypatch, jobs):
|
|
def _stub_flink(monkeypatch, jobs):
|
|
|
"""替身 JobManager:/jobs/overview 返回给定作业列表。返回捕获的请求参数。"""
|
|
"""替身 JobManager:/jobs/overview 返回给定作业列表。返回捕获的请求参数。"""
|
|
|
captured = {}
|
|
captured = {}
|
|
@@ -59,48 +65,41 @@ def _stub_flink(monkeypatch, jobs):
|
|
|
return captured
|
|
return captured
|
|
|
|
|
|
|
|
|
|
|
|
|
-# ---------- 复制槽 ----------
|
|
|
|
|
|
|
+# ---------- 复制槽:判定 ----------
|
|
|
|
|
|
|
|
def test_cdc_slot_inactive_alerts(monkeypatch):
|
|
def test_cdc_slot_inactive_alerts(monkeypatch):
|
|
|
- _stub_pg(monkeypatch, [('cdc_large', False, 0)])
|
|
|
|
|
- assert MON.check_slots('postgresql/x', LAG_CRIT) == [
|
|
|
|
|
|
|
+ assert _slot_alerts(monkeypatch, [('cdc_large', False, 0)]) == [
|
|
|
('槽 cdc_large', '空闲,采集作业可能已停')]
|
|
('槽 cdc_large', '空闲,采集作业可能已停')]
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_non_cdc_slot_inactive_is_ignored(monkeypatch):
|
|
def test_non_cdc_slot_inactive_is_ignored(monkeypatch):
|
|
|
"""PG→PG 那条槽 apply 卡住时会反复重连,active 抖动不算告警。"""
|
|
"""PG→PG 那条槽 apply 卡住时会反复重连,active 抖动不算告警。"""
|
|
|
- _stub_pg(monkeypatch, [('finance_pg2pg', False, 0)])
|
|
|
|
|
- assert MON.check_slots('postgresql/x', LAG_CRIT) == []
|
|
|
|
|
|
|
+ assert _slot_alerts(monkeypatch, [('finance_pg2pg', False, 0)]) == []
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_slot_lag_over_threshold_alerts(monkeypatch):
|
|
def test_slot_lag_over_threshold_alerts(monkeypatch):
|
|
|
- _stub_pg(monkeypatch, [('cdc_small', True, 6 * GB)])
|
|
|
|
|
- alerts = MON.check_slots('postgresql/x', LAG_CRIT)
|
|
|
|
|
- assert alerts == [('槽 cdc_small', '滞后 6.0 GB')]
|
|
|
|
|
|
|
+ assert _slot_alerts(monkeypatch, [('cdc_small', True, 6 * GB)]) == [
|
|
|
|
|
+ ('槽 cdc_small', '滞后 6.0 GB')]
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_slot_lag_at_threshold_not_alerted(monkeypatch):
|
|
def test_slot_lag_at_threshold_not_alerted(monkeypatch):
|
|
|
"""阈值是 >,正好等于不报。"""
|
|
"""阈值是 >,正好等于不报。"""
|
|
|
- _stub_pg(monkeypatch, [('cdc_small', True, LAG_CRIT)])
|
|
|
|
|
- assert MON.check_slots('postgresql/x', LAG_CRIT) == []
|
|
|
|
|
|
|
+ assert _slot_alerts(monkeypatch, [('cdc_small', True, LAG_CRIT)]) == []
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_lag_applies_to_non_cdc_slot_too(monkeypatch):
|
|
def test_lag_applies_to_non_cdc_slot_too(monkeypatch):
|
|
|
"""active 只对 cdc_ 判,滞后对所有槽都判。"""
|
|
"""active 只对 cdc_ 判,滞后对所有槽都判。"""
|
|
|
- _stub_pg(monkeypatch, [('finance_pg2pg', True, 9 * GB)])
|
|
|
|
|
- assert MON.check_slots('postgresql/x', LAG_CRIT) == [
|
|
|
|
|
|
|
+ assert _slot_alerts(monkeypatch, [('finance_pg2pg', True, 9 * GB)]) == [
|
|
|
('槽 finance_pg2pg', '滞后 9.0 GB')]
|
|
('槽 finance_pg2pg', '滞后 9.0 GB')]
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_null_lag_does_not_crash(monkeypatch):
|
|
def test_null_lag_does_not_crash(monkeypatch):
|
|
|
"""confirmed_flush_lsn 为 NULL 时 lag 是 None,当 0 处理。"""
|
|
"""confirmed_flush_lsn 为 NULL 时 lag 是 None,当 0 处理。"""
|
|
|
- _stub_pg(monkeypatch, [('cdc_large', True, None)])
|
|
|
|
|
- assert MON.check_slots('postgresql/x', LAG_CRIT) == []
|
|
|
|
|
|
|
+ assert _slot_alerts(monkeypatch, [('cdc_large', True, None)]) == []
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_inactive_and_lagging_yields_two_alerts(monkeypatch):
|
|
def test_inactive_and_lagging_yields_two_alerts(monkeypatch):
|
|
|
- _stub_pg(monkeypatch, [('cdc_large', False, 7 * GB)])
|
|
|
|
|
- assert MON.check_slots('postgresql/x', LAG_CRIT) == [
|
|
|
|
|
|
|
+ assert _slot_alerts(monkeypatch, [('cdc_large', False, 7 * GB)]) == [
|
|
|
('槽 cdc_large', '空闲,采集作业可能已停'),
|
|
('槽 cdc_large', '空闲,采集作业可能已停'),
|
|
|
('槽 cdc_large', '滞后 7.0 GB'),
|
|
('槽 cdc_large', '滞后 7.0 GB'),
|
|
|
]
|
|
]
|
|
@@ -116,12 +115,29 @@ def test_db_failure_becomes_alert(monkeypatch):
|
|
|
def _boom(ds_ref):
|
|
def _boom(ds_ref):
|
|
|
raise RuntimeError('数据源 ini 不存在')
|
|
raise RuntimeError('数据源 ini 不存在')
|
|
|
monkeypatch.setattr(MON.pgdb, 'connect', _boom)
|
|
monkeypatch.setattr(MON.pgdb, 'connect', _boom)
|
|
|
- alerts = MON.check_slots('postgresql/x', LAG_CRIT)
|
|
|
|
|
|
|
+ facts, alerts = MON.check_slots('postgresql/x', LAG_CRIT)
|
|
|
|
|
+ assert facts == []
|
|
|
assert len(alerts) == 1
|
|
assert len(alerts) == 1
|
|
|
assert alerts[0][0] == '主库'
|
|
assert alerts[0][0] == '主库'
|
|
|
assert '数据源 ini 不存在' in alerts[0][1]
|
|
assert '数据源 ini 不存在' in alerts[0][1]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+# ---------- 复制槽:事实 ----------
|
|
|
|
|
+
|
|
|
|
|
+def test_slot_facts_cover_every_slot(monkeypatch):
|
|
|
|
|
+ """健康的槽也要出现在 facts 里,-report 才有东西可列。"""
|
|
|
|
|
+ _stub_pg(monkeypatch, [('cdc_large', True, 0),
|
|
|
|
|
+ ('finance_pg2pg', True, 3 * GB)])
|
|
|
|
|
+ facts, alerts = MON.check_slots('postgresql/x', LAG_CRIT)
|
|
|
|
|
+ assert facts == [('cdc_large', True, 0), ('finance_pg2pg', True, 3 * GB)]
|
|
|
|
|
+ assert alerts == []
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_slot_facts_normalize_null_lag(monkeypatch):
|
|
|
|
|
+ _stub_pg(monkeypatch, [('cdc_large', True, None)])
|
|
|
|
|
+ assert MON.check_slots('postgresql/x', LAG_CRIT)[0] == [('cdc_large', True, 0)]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
# ---------- Flink 作业存活 ----------
|
|
# ---------- Flink 作业存活 ----------
|
|
|
|
|
|
|
|
def test_all_jobs_running_no_alert(monkeypatch):
|
|
def test_all_jobs_running_no_alert(monkeypatch):
|
|
@@ -129,14 +145,17 @@ def test_all_jobs_running_no_alert(monkeypatch):
|
|
|
{'name': 'st-cdc-large', 'state': 'RUNNING'},
|
|
{'name': 'st-cdc-large', 'state': 'RUNNING'},
|
|
|
{'name': 'st-cdc-small', 'state': 'RUNNING'},
|
|
{'name': 'st-cdc-small', 'state': 'RUNNING'},
|
|
|
])
|
|
])
|
|
|
- assert MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small']) == []
|
|
|
|
|
|
|
+ facts, alerts = MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small'])
|
|
|
|
|
+ assert alerts == []
|
|
|
|
|
+ assert facts == [('st-cdc-large', 'RUNNING'), ('st-cdc-small', 'RUNNING')]
|
|
|
assert captured['url'] == 'http://cdhmaster02:8081/jobs/overview'
|
|
assert captured['url'] == 'http://cdhmaster02:8081/jobs/overview'
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_missing_job_alerts(monkeypatch):
|
|
def test_missing_job_alerts(monkeypatch):
|
|
|
_stub_flink(monkeypatch, [{'name': 'st-cdc-large', 'state': 'RUNNING'}])
|
|
_stub_flink(monkeypatch, [{'name': 'st-cdc-large', 'state': 'RUNNING'}])
|
|
|
- assert MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small']) == [
|
|
|
|
|
- ('作业 st-cdc-small', '不在 Flink 集群上')]
|
|
|
|
|
|
|
+ facts, alerts = MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small'])
|
|
|
|
|
+ assert alerts == [('作业 st-cdc-small', '不在 Flink 集群上')]
|
|
|
|
|
+ assert facts == [('st-cdc-large', 'RUNNING'), ('st-cdc-small', None)]
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_restarting_job_alerts(monkeypatch):
|
|
def test_restarting_job_alerts(monkeypatch):
|
|
@@ -145,7 +164,7 @@ def test_restarting_job_alerts(monkeypatch):
|
|
|
{'name': 'st-cdc-large', 'state': 'RESTARTING'},
|
|
{'name': 'st-cdc-large', 'state': 'RESTARTING'},
|
|
|
{'name': 'st-cdc-small', 'state': 'RESTARTING'},
|
|
{'name': 'st-cdc-small', 'state': 'RESTARTING'},
|
|
|
])
|
|
])
|
|
|
- assert MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small']) == [
|
|
|
|
|
|
|
+ assert MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small'])[1] == [
|
|
|
('作业 st-cdc-large', '状态 RESTARTING'),
|
|
('作业 st-cdc-large', '状态 RESTARTING'),
|
|
|
('作业 st-cdc-small', '状态 RESTARTING'),
|
|
('作业 st-cdc-small', '状态 RESTARTING'),
|
|
|
]
|
|
]
|
|
@@ -154,7 +173,7 @@ def test_restarting_job_alerts(monkeypatch):
|
|
|
def test_canceled_job_alerts(monkeypatch):
|
|
def test_canceled_job_alerts(monkeypatch):
|
|
|
"""取消的作业仍留在 /jobs/overview 里。"""
|
|
"""取消的作业仍留在 /jobs/overview 里。"""
|
|
|
_stub_flink(monkeypatch, [{'name': 'st-cdc-large', 'state': 'CANCELED'}])
|
|
_stub_flink(monkeypatch, [{'name': 'st-cdc-large', 'state': 'CANCELED'}])
|
|
|
- assert MON.check_flink(JM, ['st-cdc-large']) == [
|
|
|
|
|
|
|
+ assert MON.check_flink(JM, ['st-cdc-large'])[1] == [
|
|
|
('作业 st-cdc-large', '状态 CANCELED')]
|
|
('作业 st-cdc-large', '状态 CANCELED')]
|
|
|
|
|
|
|
|
|
|
|
|
@@ -164,20 +183,30 @@ def test_stale_record_does_not_mask_running(monkeypatch):
|
|
|
{'name': 'st-cdc-large', 'state': 'CANCELED'},
|
|
{'name': 'st-cdc-large', 'state': 'CANCELED'},
|
|
|
{'name': 'st-cdc-large', 'state': 'RUNNING'},
|
|
{'name': 'st-cdc-large', 'state': 'RUNNING'},
|
|
|
])
|
|
])
|
|
|
- assert MON.check_flink(JM, ['st-cdc-large']) == []
|
|
|
|
|
|
|
+ assert MON.check_flink(JM, ['st-cdc-large']) == ([('st-cdc-large', 'RUNNING')], [])
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_untracked_job_not_reported(monkeypatch):
|
|
|
|
|
+ """集群上的其他作业不进 facts——这是 CDC 链路状态,不是集群状态。"""
|
|
|
|
|
+ _stub_flink(monkeypatch, [
|
|
|
|
|
+ {'name': 'st-finaltest', 'state': 'CANCELED'},
|
|
|
|
|
+ {'name': 'st-cdc-large', 'state': 'RUNNING'},
|
|
|
|
|
+ ])
|
|
|
|
|
+ assert MON.check_flink(JM, ['st-cdc-large']) == ([('st-cdc-large', 'RUNNING')], [])
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_flink_failure_becomes_single_alert(monkeypatch):
|
|
def test_flink_failure_becomes_single_alert(monkeypatch):
|
|
|
def _boom(url, timeout=None):
|
|
def _boom(url, timeout=None):
|
|
|
raise OSError('Connection refused')
|
|
raise OSError('Connection refused')
|
|
|
monkeypatch.setattr(MON.urllib.request, 'urlopen', _boom)
|
|
monkeypatch.setattr(MON.urllib.request, 'urlopen', _boom)
|
|
|
- alerts = MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small'])
|
|
|
|
|
|
|
+ facts, alerts = MON.check_flink(JM, ['st-cdc-large', 'st-cdc-small'])
|
|
|
|
|
+ assert facts == []
|
|
|
assert len(alerts) == 1 # 查不到就是一条,不按作业数量刷屏
|
|
assert len(alerts) == 1 # 查不到就是一条,不按作业数量刷屏
|
|
|
assert alerts[0][0] == 'Flink 集群'
|
|
assert alerts[0][0] == 'Flink 集群'
|
|
|
assert 'Connection refused' in alerts[0][1]
|
|
assert 'Connection refused' in alerts[0][1]
|
|
|
|
|
|
|
|
|
|
|
|
|
-# ---------- 渲染:走一圈全部六类告警 ----------
|
|
|
|
|
|
|
+# ---------- 告警渲染:走一圈全部六类告警 ----------
|
|
|
|
|
|
|
|
FIXED_NOW = datetime(2026, 8, 26, 15, 21, 27)
|
|
FIXED_NOW = datetime(2026, 8, 26, 15, 21, 27)
|
|
|
|
|
|
|
@@ -213,13 +242,73 @@ def test_render_all_alert_kinds():
|
|
|
assert line == '> {0}:<font color="warning">{1}</font>'.format(key, desc)
|
|
assert line == '> {0}:<font color="warning">{1}</font>'.format(key, desc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+# ---------- 状态报告渲染 ----------
|
|
|
|
|
+
|
|
|
|
|
+HEALTHY_SLOTS = [('cdc_large', True, 0), ('cdc_small', True, 0),
|
|
|
|
|
+ ('finance_pg2pg', True, 107374182)]
|
|
|
|
|
+HEALTHY_JOBS = [('st-cdc-large', 'RUNNING'), ('st-cdc-small', 'RUNNING')]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_status_all_healthy():
|
|
|
|
|
+ assert MON.render_status(HEALTHY_SLOTS, HEALTHY_JOBS, [], now=FIXED_NOW) == (
|
|
|
|
|
+ '### <font color="info">CDC 链路状态</font>\n'
|
|
|
|
|
+ '> 检测时间:2026-08-26 15:21:27\n'
|
|
|
|
|
+ '> 槽 cdc_large:<font color="info">活跃,滞后 0.0 GB</font>\n'
|
|
|
|
|
+ '> 槽 cdc_small:<font color="info">活跃,滞后 0.0 GB</font>\n'
|
|
|
|
|
+ '> 槽 finance_pg2pg:<font color="info">活跃,滞后 0.1 GB</font>\n'
|
|
|
|
|
+ '> 作业 st-cdc-large:<font color="info">RUNNING</font>\n'
|
|
|
|
|
+ '> 作业 st-cdc-small:<font color="info">RUNNING</font>')
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_status_marks_only_bad_items():
|
|
|
|
|
+ """异常项转 warning,其余保持 info,标题跟着转 warning。"""
|
|
|
|
|
+ slots = [('cdc_large', False, 0), ('cdc_small', True, 0)]
|
|
|
|
|
+ jobs = [('st-cdc-large', 'RESTARTING'), ('st-cdc-small', 'RUNNING')]
|
|
|
|
|
+ alerts = [('槽 cdc_large', '空闲,采集作业可能已停'),
|
|
|
|
|
+ ('作业 st-cdc-large', '状态 RESTARTING')]
|
|
|
|
|
+ lines = MON.render_status(slots, jobs, alerts, now=FIXED_NOW).split('\n')
|
|
|
|
|
+ assert lines[0] == '### <font color="warning">CDC 链路状态</font>'
|
|
|
|
|
+ assert lines[2] == '> 槽 cdc_large:<font color="warning">空闲,滞后 0.0 GB</font>'
|
|
|
|
|
+ assert lines[3] == '> 槽 cdc_small:<font color="info">活跃,滞后 0.0 GB</font>'
|
|
|
|
|
+ assert lines[4] == '> 作业 st-cdc-large:<font color="warning">RESTARTING</font>'
|
|
|
|
|
+ assert lines[5] == '> 作业 st-cdc-small:<font color="info">RUNNING</font>'
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_status_shows_missing_job():
|
|
|
|
|
+ lines = MON.render_status([], [('st-cdc-small', None)],
|
|
|
|
|
+ [('作业 st-cdc-small', '不在 Flink 集群上')],
|
|
|
|
|
+ now=FIXED_NOW).split('\n')
|
|
|
|
|
+ assert lines[2] == '> 作业 st-cdc-small:<font color="warning">不在集群上</font>'
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_status_falls_back_to_failure_reason():
|
|
|
|
|
+ """查询失败时没有事实可列,报告不能是空的。"""
|
|
|
|
|
+ alerts = [('主库', '复制槽查询失败:connection refused'),
|
|
|
|
|
+ ('Flink 集群', '查询失败:timeout')]
|
|
|
|
|
+ lines = MON.render_status([], [], alerts, now=FIXED_NOW).split('\n')
|
|
|
|
|
+ assert lines[0] == '### <font color="warning">CDC 链路状态</font>'
|
|
|
|
|
+ assert lines[2] == '> 主库:<font color="warning">复制槽查询失败:connection refused</font>'
|
|
|
|
|
+ assert lines[3] == '> Flink 集群:<font color="warning">查询失败:timeout</font>'
|
|
|
|
|
+ assert len(lines) == 4
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_status_does_not_repeat_items_already_listed():
|
|
|
|
|
+ """槽/作业的告警已经体现在它自己那行上,末尾不再重复一遍。"""
|
|
|
|
|
+ slots = [('cdc_large', False, 0)]
|
|
|
|
|
+ alerts = [('槽 cdc_large', '空闲,采集作业可能已停')]
|
|
|
|
|
+ assert len(MON.render_status(slots, [], alerts, now=FIXED_NOW).split('\n')) == 3
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
# ---------- main 串联 ----------
|
|
# ---------- main 串联 ----------
|
|
|
|
|
|
|
|
-def _run_main(monkeypatch, slot_alerts, flink_alerts):
|
|
|
|
|
|
|
+def _run_main(monkeypatch, slot_alerts, flink_alerts, argv=None,
|
|
|
|
|
+ slot_facts=(), flink_facts=()):
|
|
|
"""跑 main,返回替身 alerter 实例供断言。"""
|
|
"""跑 main,返回替身 alerter 实例供断言。"""
|
|
|
- monkeypatch.setattr(sys, 'argv', ['cdc-monitor.py'])
|
|
|
|
|
- monkeypatch.setattr(MON, 'check_slots', lambda ds, lag: list(slot_alerts))
|
|
|
|
|
- monkeypatch.setattr(MON, 'check_flink', lambda jm, jobs: list(flink_alerts))
|
|
|
|
|
|
|
+ monkeypatch.setattr(sys, 'argv', argv or ['cdc-monitor.py'])
|
|
|
|
|
+ monkeypatch.setattr(MON, 'check_slots',
|
|
|
|
|
+ lambda ds, lag: (list(slot_facts), list(slot_alerts)))
|
|
|
|
|
+ monkeypatch.setattr(MON, 'check_flink',
|
|
|
|
|
+ lambda jm, jobs: (list(flink_facts), list(flink_alerts)))
|
|
|
instance = MagicMock()
|
|
instance = MagicMock()
|
|
|
monkeypatch.setattr(MON, 'Alerter', MagicMock(return_value=instance))
|
|
monkeypatch.setattr(MON, 'Alerter', MagicMock(return_value=instance))
|
|
|
MON.main()
|
|
MON.main()
|
|
@@ -242,11 +331,32 @@ def test_main_sends_once_with_all_alerts(monkeypatch):
|
|
|
assert '作业 st-cdc-small' in content
|
|
assert '作业 st-cdc-small' in content
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+def test_main_report_sends_even_when_healthy(monkeypatch):
|
|
|
|
|
+ """-report 是上线核对用的,正常也要发。"""
|
|
|
|
|
+ instance = _run_main(monkeypatch, [], [],
|
|
|
|
|
+ argv=['cdc-monitor.py', '-report'],
|
|
|
|
|
+ slot_facts=HEALTHY_SLOTS, flink_facts=HEALTHY_JOBS)
|
|
|
|
|
+ instance.send_markdown.assert_called_once()
|
|
|
|
|
+ content = instance.send_markdown.call_args[0][0]
|
|
|
|
|
+ assert content.startswith('### <font color="info">CDC 链路状态</font>')
|
|
|
|
|
+ assert '> 作业 st-cdc-large:<font color="info">RUNNING</font>' in content
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_main_report_shows_abnormal_state(monkeypatch):
|
|
|
|
|
+ """-report 遇到异常发的仍是状态报告,不是告警消息。"""
|
|
|
|
|
+ instance = _run_main(monkeypatch, [], [('作业 st-cdc-large', '状态 RESTARTING')],
|
|
|
|
|
+ argv=['cdc-monitor.py', '-report'],
|
|
|
|
|
+ flink_facts=[('st-cdc-large', 'RESTARTING')])
|
|
|
|
|
+ content = instance.send_markdown.call_args[0][0]
|
|
|
|
|
+ assert content.startswith('### <font color="warning">CDC 链路状态</font>')
|
|
|
|
|
+ assert '> 作业 st-cdc-large:<font color="warning">RESTARTING</font>' in content
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def test_main_builds_alerter_before_checks(monkeypatch):
|
|
def test_main_builds_alerter_before_checks(monkeypatch):
|
|
|
"""健康时也要构造 Alerter:配置坏了要立刻暴露,不能等真出事才发现发不出去。"""
|
|
"""健康时也要构造 Alerter:配置坏了要立刻暴露,不能等真出事才发现发不出去。"""
|
|
|
monkeypatch.setattr(sys, 'argv', ['cdc-monitor.py'])
|
|
monkeypatch.setattr(sys, 'argv', ['cdc-monitor.py'])
|
|
|
- monkeypatch.setattr(MON, 'check_slots', lambda ds, lag: [])
|
|
|
|
|
- monkeypatch.setattr(MON, 'check_flink', lambda jm, jobs: [])
|
|
|
|
|
|
|
+ monkeypatch.setattr(MON, 'check_slots', lambda ds, lag: ([], []))
|
|
|
|
|
+ monkeypatch.setattr(MON, 'check_flink', lambda jm, jobs: ([], []))
|
|
|
fake_cls = MagicMock()
|
|
fake_cls = MagicMock()
|
|
|
monkeypatch.setattr(MON, 'Alerter', fake_cls)
|
|
monkeypatch.setattr(MON, 'Alerter', fake_cls)
|
|
|
MON.main()
|
|
MON.main()
|
|
@@ -257,9 +367,9 @@ def test_main_defaults_cover_both_jobs(monkeypatch):
|
|
|
"""默认值必须带上小表组,否则它挂了不报警。"""
|
|
"""默认值必须带上小表组,否则它挂了不报警。"""
|
|
|
seen = {}
|
|
seen = {}
|
|
|
monkeypatch.setattr(sys, 'argv', ['cdc-monitor.py'])
|
|
monkeypatch.setattr(sys, 'argv', ['cdc-monitor.py'])
|
|
|
- monkeypatch.setattr(MON, 'check_slots', lambda ds, lag: [])
|
|
|
|
|
|
|
+ monkeypatch.setattr(MON, 'check_slots', lambda ds, lag: ([], []))
|
|
|
monkeypatch.setattr(MON, 'check_flink',
|
|
monkeypatch.setattr(MON, 'check_flink',
|
|
|
- lambda jm, jobs: seen.update(jm=jm, jobs=jobs) or [])
|
|
|
|
|
|
|
+ lambda jm, jobs: seen.update(jm=jm, jobs=jobs) or ([], []))
|
|
|
monkeypatch.setattr(MON, 'Alerter', MagicMock())
|
|
monkeypatch.setattr(MON, 'Alerter', MagicMock())
|
|
|
|
|
|
|
|
MON.main()
|
|
MON.main()
|
|
@@ -274,9 +384,9 @@ def test_main_passes_cli_overrides(monkeypatch):
|
|
|
'cdc-monitor.py', '-ds', 'postgresql/other', '-channel', 'realtime',
|
|
'cdc-monitor.py', '-ds', 'postgresql/other', '-channel', 'realtime',
|
|
|
'-jm', 'other-host:9081', '-jobs', 'a, b ,', '-lag-gb', '2'])
|
|
'-jm', 'other-host:9081', '-jobs', 'a, b ,', '-lag-gb', '2'])
|
|
|
monkeypatch.setattr(MON, 'check_slots',
|
|
monkeypatch.setattr(MON, 'check_slots',
|
|
|
- lambda ds, lag: seen.update(ds=ds, lag=lag) or [])
|
|
|
|
|
|
|
+ lambda ds, lag: seen.update(ds=ds, lag=lag) or ([], []))
|
|
|
monkeypatch.setattr(MON, 'check_flink',
|
|
monkeypatch.setattr(MON, 'check_flink',
|
|
|
- lambda jm, jobs: seen.update(jm=jm, jobs=jobs) or [])
|
|
|
|
|
|
|
+ lambda jm, jobs: seen.update(jm=jm, jobs=jobs) or ([], []))
|
|
|
fake_cls = MagicMock()
|
|
fake_cls = MagicMock()
|
|
|
monkeypatch.setattr(MON, 'Alerter', fake_cls)
|
|
monkeypatch.setattr(MON, 'Alerter', fake_cls)
|
|
|
|
|
|