dashboard supports removing of unhealth machines

sentinel-dashboard/Sentinel_Dashboard_Feature.md

@@ -47,7 +47,35 @@ Sentinel 提供了多种规则来保护系统的不同部分。流量控制规
 本控制台只是用于演示 Sentinel 的基本能力和工作流程，并没有依赖生产环境中所必需的组件，比如**持久化的后端数据库、可靠的配置中心**等。
 目前 Sentinel 采用内存态的方式存储监控和规则数据，监控最长存储时间为 5 分钟，控制台重启后数据丢失。
 
+## 3. 配置项
+
+控制台的一些特性可以通过配置项来进行配置，配置项主要有两个来源：`System.getProperty()`和`System.getenv()`，同时存在时后者可以覆盖前者。  
+> 环境变量因为不支持`.`所以需要将其更换为`_`。
+
+项 | 类型 | 默认值 | 最小值 | 描述
+--- | --- | --- | --- | ---
+sentinel.dashboard.app.hideAppNoMachineMillis | Integer | 0 | 60000 | 是否隐藏无健康节点的应用，距离最近一次主机心跳时间的毫秒数，默认关闭
+sentinel.dashboard.removeAppNoMachineMillis | Integer | 0 | 120000 | 是否自动删除无健康节点的应用，距离最近一次其下节点的心跳时间毫秒数，默认关闭
+sentinel.dashboard.unhealthyMachineMillis | Integer | 60000 | 30000 | 主机失联判定，不可关闭
+sentinel.dashboard.autoRemoveMachineMillis | Integer | 0 | 300000 | 距离最近心跳时间超过指定时间是否自动删除失联节点，默认关闭
+
+配置示例：
+
+命令行
+```shell
+java -Dsentinel.dashboard.app.hideAppNoMachineMillis=60000
+```
+java
+```java
+System.setProperty("sentinel.dashboard.app.hideAppNoMachineMillis", "60000");
+```
+环境变量
+```shell
+sentinel_dashboard_app_hideAppNoMachineMillis=60000
+```
+
 更多：
 
 - [Sentinel 控制台启动和客户端接入](./README.md)
-- [控制台 Wiki](https://github.com/alibaba/Sentinel/wiki/%E6%8E%A7%E5%88%B6%E5%8F%B0)
+- [控制台 Wiki](https://github.com/alibaba/Sentinel/wiki/%E6%8E%A7%E5%88%B6%E5%8F%B0)
+

sentinel-dashboard/pom.xml

@@ -23,7 +23,6 @@
         <dependency>
             <groupId>com.alibaba.csp</groupId>
             <artifactId>sentinel-core</artifactId>
-            <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.alibaba.csp</groupId>
@@ -33,7 +32,6 @@
         <dependency>
             <groupId>com.alibaba.csp</groupId>
             <artifactId>sentinel-transport-simple-http</artifactId>
-            <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.alibaba.csp</groupId>
@@ -118,7 +116,12 @@
             <version>${apollo.openapi.version}</version>
             <scope>test</scope>
         </dependency>
-
+        <dependency>
+        	<groupId>com.github.stefanbirkner</groupId>
+        	<artifactId>system-rules</artifactId>
+        	<version>1.16.1</version>
+        	<scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/config/DashboardConfig.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright 1999-2018 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.dashboard.config;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.springframework.lang.NonNull;
+
+/**
+ * Dashboard config support
+ * <p>
+ * Dashboard supports configuration loading by several ways by order:<br>
+ * 1. System.properties<br>
+ * 2. Env
+ * 
+ * @author jason
+ * @since 1.5.0
+ *
+ */
+public class DashboardConfig {
+    /**
+     * hide app in sidebar when it had no healthy machine after specific period in millis
+     */
+    public static final String CONFIG_HIDE_APP_NO_MACHINE_MILLIS = "sentinel.dashboard.app.hideAppNoMachineMillis";
+    /**
+     * remove app when it had no healthy machine after specific period in millis
+     */
+    public static final String CONFIG_REMOVE_APP_NO_MACHINE_MILLIS = "sentinel.dashboard.removeAppNoMachineMillis";
+    /**
+     * unhealthy millis
+     */
+    public static final String CONFIG_UNHEALTHY_MACHINE_MILLIS = "sentinel.dashboard.unhealthyMachineMillis";
+    /**
+     * auto remove unhealthy machine after specific period in millis
+     */
+    public static final String CONFIG_AUTO_REMOVE_MACHINE_MILLIS = "sentinel.dashboard.autoRemoveMachineMillis";
+    private static final ConcurrentMap<String, Object> cacheMap = new ConcurrentHashMap<>();
+
+    @NonNull
+    private static String getConfig(String name) {
+        // env
+        String val = System.getenv(name);
+        if (StringUtils.isNotEmpty(val)) {
+            return val;
+        }
+        // properties
+        val = System.getProperty(name);
+        if (StringUtils.isNotEmpty(val)) {
+            return val;
+        }
+        return "";
+    }
+
+    protected static int getConfigInt(String name, int defaultVal, int minVal) {
+        if (cacheMap.containsKey(name)) {
+            return (int)cacheMap.get(name);
+        }
+        int val = NumberUtils.toInt(getConfig(name));
+        if (val == 0) {
+            val = defaultVal;
+        } else if (val < minVal) {
+            val = minVal;
+        }
+        cacheMap.put(name, val);
+        return val;
+    }
+
+    public static int getHideAppNoMachineMillis() {
+        return getConfigInt(CONFIG_HIDE_APP_NO_MACHINE_MILLIS, 0, 60000);
+    }
+
+    public static int getRemoveAppNoMachineMillis() {
+        return getConfigInt(CONFIG_REMOVE_APP_NO_MACHINE_MILLIS, 0, 120000);
+    }
+
+    public static int getAutoRemoveMachineMillis() {
+        return getConfigInt(CONFIG_AUTO_REMOVE_MACHINE_MILLIS, 0, 300000);
+    }
+
+    public static int getUnhealthyMachineMillis() {
+        return getConfigInt(CONFIG_UNHEALTHY_MACHINE_MILLIS, 60000, 30000);
+    }
+
+    public static void clearCache() {
+        cacheMap.clear();
+    }
+}

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/controller/AppController.java

@@ -32,6 +32,7 @@
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 
 /**
@@ -79,4 +80,21 @@ Result<List<MachineInfoVo>> getMachinesByApp(@PathVariable("app") String app) {
         });
         return Result.ofSuccess(MachineInfoVo.fromMachineInfoList(list));
     }
+
+    @ResponseBody
+    @RequestMapping(value = "/{app}/machine/remove.json")
+    Result<String> removeMachineById(
+            @PathVariable("app") String app,
+            @RequestParam(name = "ip") String ip,
+            @RequestParam(name = "port") int port) {
+        AppInfo appInfo = appManagement.getDetailApp(app);
+        if (appInfo == null) {
+            return Result.ofSuccess(null);
+        }
+        if (appManagement.removeMachine(app, ip, port)) {
+            return Result.ofSuccessMsg("success");
+        } else {
+            return Result.ofFail(1, "remove failed");
+        }
+    }
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/controller/MachineRegistryController.java

@@ -15,8 +15,6 @@
  */
 package com.alibaba.csp.sentinel.dashboard.controller;
 
-import java.util.Date;
-
 import com.alibaba.csp.sentinel.dashboard.discovery.AppManagement;
 import com.alibaba.csp.sentinel.util.StringUtil;
 
@@ -57,14 +55,15 @@ public Result<?> receiveHeartBeat(String app, Long version, String v, String hos
             return Result.ofFail(-1, "your port not set yet");
         }
         String sentinelVersion = StringUtil.isEmpty(v) ? "unknown" : v;
-        long timestamp = version == null ? System.currentTimeMillis() : version;
+        version = version == null ? System.currentTimeMillis() : version;
         try {
             MachineInfo machineInfo = new MachineInfo();
             machineInfo.setApp(app);
             machineInfo.setHostname(hostname);
             machineInfo.setIp(ip);
             machineInfo.setPort(port);
-            machineInfo.setTimestamp(new Date(timestamp));
+            machineInfo.setHeartbeatVersion(version);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis());
             machineInfo.setVersion(sentinelVersion);
             appManagement.addMachine(machineInfo);
             return Result.ofSuccessMsg("success");

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/datasource/entity/MachineEntity.java

@@ -103,7 +103,8 @@ public MachineInfo toMachineInfo() {
         machineInfo.setHostname(hostname);
         machineInfo.setIp(ip);
         machineInfo.setPort(port);
-        machineInfo.setTimestamp(timestamp);
+        machineInfo.setLastHeatbeat(timestamp.getTime());
+        machineInfo.setHeartbeatVersion(timestamp.getTime());
 
         return machineInfo;
     }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/AppInfo.java

@@ -15,12 +15,29 @@
  */
 package com.alibaba.csp.sentinel.dashboard.discovery;
 
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Optional;
+import java.util.Iterator;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.alibaba.csp.sentinel.dashboard.config.DashboardConfig;
+
 public class AppInfo {
+    private static final Comparator<MachineInfo> COMPARATOR_BY_MACHINE_HEARTBEAT_DESC = new Comparator<MachineInfo>() {
+
+        @Override
+        public int compare(MachineInfo o1, MachineInfo o2) {
+            if (o1.getLastHeatbeat() < o2.getLastHeatbeat()) {
+                return -1;
+            }
+            if (o1.getLastHeatbeat() > o2.getLastHeatbeat()) {
+                return 1;
+            }
+            return 0;
+        }
+    };
 
     private String app = "";
 
@@ -59,10 +76,58 @@ public boolean addMachine(MachineInfo machineInfo) {
         machines.remove(machineInfo);
         return machines.add(machineInfo);
     }
-
+
+    public synchronized boolean removeMachine(String ip, int port) {
+        Iterator<MachineInfo> it = machines.iterator();
+        while (it.hasNext()) {
+            MachineInfo machine = it.next();
+            if (machine.getIp().equals(ip) && machine.getPort() == port) {
+                it.remove();
+                return true;
+            }
+        }
+        return false;
+    }
+
     public Optional<MachineInfo> getMachine(String ip, int port) {
         return machines.stream()
             .filter(e -> e.getIp().equals(ip) && e.getPort().equals(port))
             .findFirst();
     }
+
+    private boolean heartbeatJudge(int threshold) {
+        if (machines.size() == 0) {
+            return false;
+        }
+        if (threshold > 0) {
+            long healthyCount = machines.stream()
+                    .filter(m -> m.isHealthy())
+                    .count();
+            if (healthyCount == 0) {
+                // no machine
+                long recentHeartBeat = machines.stream()
+                        .max(COMPARATOR_BY_MACHINE_HEARTBEAT_DESC).get().getLastHeatbeat();
+                return System.currentTimeMillis() - recentHeartBeat < threshold;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * having no healthy machine and should not be displayed
+     * 
+     * @return
+     */
+    public boolean isShown() {
+        return heartbeatJudge(DashboardConfig.getHideAppNoMachineMillis());
+    }
+
+    /**
+     * having no healthy machine and should be removed
+     * 
+     * @return
+     */
+    public boolean isDead() {
+        return !heartbeatJudge(DashboardConfig.getRemoveAppNoMachineMillis());
+    }
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/AppManagement.java

@@ -52,6 +52,11 @@ public Set<AppInfo> getBriefApps() {
     public long addMachine(MachineInfo machineInfo) {
         return machineDiscovery.addMachine(machineInfo);
     }
+
+    @Override
+    public boolean removeMachine(String app, String ip, int port) {
+        return machineDiscovery.removeMachine(app, ip, port);
+    }
 
     @Override
     public List<String> getAppNames() {
@@ -62,5 +67,10 @@ public List<String> getAppNames() {
     public AppInfo getDetailApp(String app) {
         return machineDiscovery.getDetailApp(app);
     }
+
+    @Override
+    public void removeApp(String app) {
+        machineDiscovery.removeApp(app);
+    }
 
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/MachineDiscovery.java

@@ -20,14 +20,17 @@
 
 public interface MachineDiscovery {
 
-    long MAX_CLIENT_LIVE_TIME_MS = 1000 * 60 * 5;
     String UNKNOWN_APP_NAME = "CLUSTER_NOT_STARTED";
 
     List<String> getAppNames();
 
     Set<AppInfo> getBriefApps();
 
     AppInfo getDetailApp(String app);
+
+    void removeApp(String app);
 
     long addMachine(MachineInfo machineInfo);
+
+    boolean removeMachine(String app, String ip, int port);
 }