服务延迟报警 [no ci]

cmd/dashboard/controller/member_api.go

@@ -393,6 +393,9 @@ type monitorForm struct {
 	NotificationTag string
 	SkipServersRaw  string
 	Duration        uint64
+	MinLatency      float32
+	MaxLatency      float32
+	LatencyNotify   string
 }
 
 func (ma *memberAPI) addOrEditMonitor(c *gin.Context) {
@@ -409,6 +412,9 @@ func (ma *memberAPI) addOrEditMonitor(c *gin.Context) {
 		m.Notify = mf.Notify == "on"
 		m.NotificationTag = mf.NotificationTag
 		m.Duration = mf.Duration
+		m.LatencyNotify = mf.LatencyNotify == "on"
+		m.MinLatency = mf.MinLatency
+		m.MaxLatency = mf.MaxLatency
 		err = m.InitSkipServers()
 	}
 	if err == nil {

model/monitor.go

@@ -48,6 +48,10 @@ type Monitor struct {
 	NotificationTag string // 当前服务监控所属的通知组
 	Cover           uint8
 
+	MinLatency    float32
+	MaxLatency    float32
+	LatencyNotify bool
+
 	SkipServers map[uint64]bool `gorm:"-" json:"-"`
 	CronJobID   cron.EntryID    `gorm:"-" json:"-"`
 }

resource/l10n/zh-CN.toml

@@ -130,6 +130,21 @@ other = "秒"
 [EnableFailureNotification]
 other = "启用故障通知"
 
+[FailureNotification]
+other = "故障通知"
+
+[MaxLatency]
+other = "最大延迟(ms)"
+
+[MinLatency]
+other = "最小延迟(ms)"
+
+[EnableLatencyNotification]
+other = "启用延迟通知"
+
+[LatencyNotification]
+other = "延迟通知"
+
 [IntroductionOfMonitor]
 other = """
 类型为 <b>HTTP-GET</b> 时输入URL(带 http/https, HTTPS协议的会顺带监控SSL证书)；<br>

resource/static/main.js

@@ -77,6 +77,8 @@ function showFormModal(modelSelector, formID, URL, getData) {
                 item.name === "Duration"
               ) {
                 obj[item.name] = parseInt(item.value);
+              } else if (item.name.endsWith("Latency")) {
+                obj[item.name] = parseFloat(item.value);
               } else {
                 obj[item.name] = item.value;
               }
@@ -313,6 +315,13 @@ function addOrEditMonitor(monitor) {
   } else {
     modal.find(".ui.nb-notify.checkbox").checkbox("set unchecked");
   }
+  modal.find("input[name=MaxLatency]").val(monitor ? monitor.MaxLatency : null);
+  modal.find("input[name=MinLatency]").val(monitor ? monitor.MinLatency : null);
+  if (monitor && monitor.LatencyNotify) {
+    modal.find(".ui.nb-lt-notify.checkbox").checkbox("set checked");
+  } else {
+    modal.find(".ui.nb-lt-notify.checkbox").checkbox("set unchecked");
+  }
   modal.find("a.ui.label.visible").each((i, el) => {
     el.remove();
   });

resource/template/common/footer.html

@@ -10,7 +10,7 @@
 <script src="https://cdn.staticfile.org/semantic-ui/2.4.1/semantic.min.js"></script>
 <script src="/static/semantic-ui-alerts.min.js"></script>
 <script src="https://cdn.staticfile.org/vue/2.6.14/vue.min.js"></script>
-<script src="/static/main.js?v20220915"></script>
+<script src="/static/main.js?v20220917"></script>
 <script>
     (function () {
         updateLang({{.LANG }});

resource/template/component/monitor.html

@@ -54,6 +54,20 @@
           <label>{{tr "EnableFailureNotification"}}</label>
         </div>
       </div>
+      <div class="field">
+        <label>{{tr "MaxLatency"}}</label>
+        <input type="number" name="MaxLatency" placeholder="100.88" />
+      </div>
+      <div class="field">
+        <label>{{tr "MinLatency"}}</label>
+        <input type="number" name="MinLatency" placeholder="100.88" />
+      </div>
+      <div class="field">
+        <div class="ui nb-lt-notify checkbox">
+          <input name="LatencyNotify" type="checkbox" tabindex="0" class="hidden" />
+          <label>{{tr "EnableLatencyNotification"}}</label>
+        </div>
+      </div>
     </form>
     <div class="ui warning message">
       <p>

resource/template/dashboard-default/monitor.html

@@ -20,7 +20,8 @@
           <th>{{tr "Type"}}</th>
           <th>{{tr "Duration"}}</th>
           <th>{{tr "NotificationMethodGroup"}}</th>
-          <th>{{tr "Notification"}}</th>
+          <th>{{tr "FailureNotification"}}</th>
+          <th>{{tr "LatencyNotification"}}</th>
           <th>{{tr "Administration"}}</th>
         </tr>
       </thead>
@@ -39,6 +40,7 @@
           <td>{{$monitor.Duration}} {{tr "Seconds"}}</td>
           <td>{{$monitor.NotificationTag}}</td>
           <td>{{$monitor.Notify}}</td>
+          <td>{{$monitor.LatencyNotify}}</td>
           <td>
             <div class="ui mini icon buttons">
               <button class="ui button" onclick="addOrEditMonitor({{$monitor}})">

service/singleton/servicesentinel.go

@@ -82,10 +82,10 @@ func NewServiceSentinel(serviceSentinelDispatchBus chan<- model.Monitor) {
 }
 
 /*
-   使用缓存 channel，处理上报的 Service 请求结果，然后判断是否需要报警
+使用缓存 channel，处理上报的 Service 请求结果，然后判断是否需要报警
-   需要记录上一次的状态信息
+需要记录上一次的状态信息
 
-   加锁顺序：serviceResponseDataStoreLock > monthlyStatusLock > monitorsLock
+加锁顺序：serviceResponseDataStoreLock > monthlyStatusLock > monitorsLock
 */
 type ServiceSentinel struct {
 	// 服务监控任务上报通道
@@ -364,6 +364,20 @@ func (ss *ServiceSentinel) worker() {
 				log.Println("NEZHA>> 服务监控数据持久化失败：", err)
 			}
 		}
+		// 延迟报警
+		if mh.Delay > 0 {
+			ss.monitorsLock.RLock()
+			if ss.monitors[mh.MonitorID].LatencyNotify {
+				if mh.Delay > ss.monitors[mh.MonitorID].MaxLatency {
+					go SendNotification(ss.monitors[mh.MonitorID].NotificationTag, fmt.Sprintf("[Latency] %s %2f > %2f", ss.monitors[mh.MonitorID].Name, mh.Delay, ss.monitors[mh.MonitorID].MaxLatency), true)
+				}
+				if mh.Delay < ss.monitors[mh.MonitorID].MinLatency {
+					go SendNotification(ss.monitors[mh.MonitorID].NotificationTag, fmt.Sprintf("[Latency] %s %2f < %2f", ss.monitors[mh.MonitorID].Name, mh.Delay, ss.monitors[mh.MonitorID].MinLatency), true)
+				}
+			}
+			ss.monitorsLock.RUnlock()
+		}
+		// 故障报警
 		if stateCode == StatusDown || stateCode != ss.lastStatus[mh.MonitorID] {
 			ss.monitorsLock.RLock()
 			isNeedSendNotification := (ss.lastStatus[mh.MonitorID] != 0 || stateCode == StatusDown) && ss.monitors[mh.MonitorID].Notify