fix: show live node and tunnel deployment progress in list view

This commit is contained in:
sagitchu
2026-11-10 10:47:46 +08:00
parent e0dc2a9ca7
commit edc8faa5f8
13 changed files with 1134 additions and 103 deletions
@@ -228,6 +228,7 @@ func (h *Handler) Register(mux *http.ServeMux) {
mux.HandleFunc("/api/v1/tunnel/update-order", h.tunnelUpdateOrder)
mux.HandleFunc("/api/v1/tunnel/batch-delete", h.tunnelBatchDelete)
mux.HandleFunc("/api/v1/tunnel/batch-redeploy", h.tunnelBatchRedeploy)
mux.HandleFunc("/api/v1/tunnel/batch-redeploy/stream", h.tunnelBatchRedeployStream)
mux.HandleFunc("/api/v1/tunnel/user/assign", h.userTunnelAssign)
mux.HandleFunc("/api/v1/tunnel/user/batch-assign", h.userTunnelBatchAssign)
mux.HandleFunc("/api/v1/tunnel/user/remove", h.userTunnelRemove)
+23 -37
View File
@@ -2046,51 +2046,37 @@ func normalizeBatchFailureReason(reason string) string {
return trimmed
}
func (h *Handler) redeployTunnelConfiguration(tunnelID int64) error {
if _, err := h.getTunnelRecord(tunnelID); err != nil {
return err
}
if nftMode, entryNodeIDs, err := h.tunnelUsesNftables(tunnelID); err != nil {
return err
} else if nftMode {
if len(entryNodeIDs) == 0 {
return errors.New("nftables 转发缺少入口节点")
}
return h.reconcileNftablesNodeByRequest(entryNodeIDs[0])
}
return h.redeployTunnelAndForwards(tunnelID)
}
func (h *Handler) tunnelBatchRedeploy(w http.ResponseWriter, r *http.Request) {
ids := idsFromBody(r, w)
if ids == nil {
return
}
success := 0
fail := 0
failures := make([]batchFailureDetail, 0)
result := batchOperationResult{}
for _, tunnelID := range ids {
tunnel, tunnelErr := h.getTunnelRecord(tunnelID)
tunnelName := ""
if tunnelErr == nil && tunnel != nil {
tunnelName, _ = h.repo.GetTunnelName(tunnelID)
name, _ := h.repo.GetTunnelName(tunnelID)
if err := h.redeployTunnelConfiguration(tunnelID); err != nil {
result.FailCount++
result.Failures = appendBatchFailure(result.Failures, tunnelID, name, err)
} else {
result.SuccessCount++
}
if tunnelErr != nil {
fail++
failures = appendBatchFailure(failures, tunnelID, tunnelName, tunnelErr)
continue
}
if nftMode, entryNodeIDs, modeErr := h.tunnelUsesNftables(tunnelID); modeErr != nil {
fail++
failures = appendBatchFailure(failures, tunnelID, tunnelName, modeErr)
continue
} else if nftMode {
if len(entryNodeIDs) == 0 {
fail++
failures = appendBatchFailureReason(failures, tunnelID, tunnelName, "nftables 转发缺少入口节点")
continue
}
if reconcileErr := h.reconcileNftablesNodeByRequest(entryNodeIDs[0]); reconcileErr != nil {
fail++
failures = appendBatchFailure(failures, tunnelID, tunnelName, reconcileErr)
continue
}
success++
continue
}
if err := h.redeployTunnelAndForwards(tunnelID); err != nil {
fail++
failures = appendBatchFailure(failures, tunnelID, tunnelName, err)
continue
}
success++
}
response.WriteJSON(w, response.OK(batchOperationResult{SuccessCount: success, FailCount: fail, Failures: failures}))
response.WriteJSON(w, response.OK(result))
}
func (h *Handler) userTunnelAssign(w http.ResponseWriter, r *http.Request) {
@@ -0,0 +1,105 @@
package handler
import (
"encoding/json"
"errors"
"net/http"
"time"
"go-backend/internal/http/response"
)
type redeployProgress struct {
Total int `json:"total"`
Completed int `json:"completed"`
Success int `json:"success"`
Failed int `json:"failed"`
}
type redeployItemResult struct {
ID int64 `json:"id"`
Name string `json:"name"`
Success bool `json:"success"`
Reason string `json:"reason,omitempty"`
}
// tunnelBatchRedeployStream reports completed tunnels, not estimated runtime.
// A disconnected caller may have an in-flight mutation whose result it did not
// receive. Finish that mutation, but never start another or automatically retry.
func (h *Handler) tunnelBatchRedeployStream(w http.ResponseWriter, r *http.Request) {
ids := idsFromBody(r, w)
if ids == nil {
return
}
// idsFromBody sorts IDs; ignore duplicate selections so each tunnel runs once.
unique := ids[:0]
for _, id := range ids {
if len(unique) == 0 || unique[len(unique)-1] != id {
unique = append(unique, id)
}
}
ids = unique
if len(ids) == 0 {
response.WriteJSON(w, response.ErrDefault("ids不能为空"))
return
}
_, err := prepareDiagnosisStreamResponse(w)
if err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
encoder := json.NewEncoder(w)
controller := http.NewResponseController(w)
emit := func(kind string, data interface{}) bool {
if r.Context().Err() != nil {
return false
}
// The server's WriteTimeout is an absolute request deadline. Renew only
// this stream's bounded write window as progress is emitted, so a long
// batch is not cut off despite making steady progress. Recorders and
// other non-network writers may not support deadlines.
if err := controller.SetWriteDeadline(time.Now().Add(2 * time.Minute)); err != nil && !errors.Is(err, http.ErrNotSupported) {
return false
}
event := diagnosisStreamEvent{Type: kind, Data: data, TS: time.Now().UnixMilli()}
if err := encoder.Encode(event); err != nil {
return false
}
// Buffered writes can succeed while the actual socket flush fails.
// Do not start a mutation after its processing event failed to flush.
return controller.Flush() == nil
}
progress := redeployProgress{Total: len(ids)}
result := batchOperationResult{}
if !emit("start", progress) {
return
}
for _, id := range ids {
if r.Context().Err() != nil {
return
}
name, _ := h.repo.GetTunnelName(id)
if !emit("processing", map[string]interface{}{"id": id, "name": name, "progress": progress}) {
return
}
if r.Context().Err() != nil {
return
}
item := redeployItemResult{ID: id, Name: name}
if err := h.redeployTunnelConfiguration(id); err != nil {
result.FailCount++
result.Failures = appendBatchFailure(result.Failures, id, name, err)
item.Reason = result.Failures[len(result.Failures)-1].Reason
progress.Failed++
} else {
result.SuccessCount++
item.Success = true
progress.Success++
}
progress.Completed++
if !emit("item", map[string]interface{}{"result": item, "progress": progress}) {
return
}
}
emit("done", map[string]interface{}{"progress": progress, "result": result})
}
@@ -0,0 +1,248 @@
package handler
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"go-backend/internal/http/middleware"
runtimenft "go-backend/internal/runtime/nftables"
)
type blockingRedeployManager struct {
fakeNftablesManager
entered chan struct{}
release chan struct{}
once sync.Once
}
func (m *blockingRedeployManager) Reconcile(ctx context.Context, cfg runtimenft.SSHConfig, plan runtimenft.NodePlan) (runtimenft.ApplyResult, error) {
m.once.Do(func() { close(m.entered); <-m.release })
return m.fakeNftablesManager.Reconcile(ctx, cfg, plan)
}
func TestTunnelRedeployStreamFlushesBeforeSlowMutationAndReportsMixedResults(t *testing.T) {
fixture := setupNftablesHandler(t)
h := fixture.handler
seedNftablesSSHConfig(t, h, fixture.nodeID)
id := seedTunnelForNftables(t, h, "stream-tunnel", fixture.nodeID)
manager := &blockingRedeployManager{entered: make(chan struct{}), release: make(chan struct{})}
h.nftablesManager = manager
var releaseOnce sync.Once
release := func() { releaseOnce.Do(func() { close(manager.release) }) }
defer release()
server := httptest.NewServer(http.HandlerFunc(h.tunnelBatchRedeployStream))
defer server.Close()
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Post(server.URL, "application/json", strings.NewReader(fmt.Sprintf(`{"ids":[%d,999999,%d]}`, id, id)))
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
decoder := json.NewDecoder(response.Body)
read := func(want string) map[string]interface{} {
t.Helper()
var event struct {
Type string
Data map[string]interface{}
}
if err := decoder.Decode(&event); err != nil {
t.Fatal(err)
}
if event.Type != want {
t.Fatalf("expected %s, got %s", want, event.Type)
}
return event.Data
}
if start := read("start"); start["total"] != float64(2) {
t.Fatalf("duplicate IDs not deduplicated: %+v", start)
}
processing := read("processing")
if processing["name"] != "stream-tunnel" || processing["id"] != float64(id) {
t.Fatalf("wrong current tunnel: %+v", processing)
}
select {
case <-manager.entered:
case <-time.After(2 * time.Second):
t.Fatal("did not start mutation")
}
// Both events arrived over HTTP while the actual mutation is still blocked.
release()
item := read("item")
if item["result"].(map[string]interface{})["success"] != true || item["progress"].(map[string]interface{})["completed"] != float64(1) {
t.Fatalf("wrong success result: %+v", item)
}
read("processing")
item = read("item")
if item["result"].(map[string]interface{})["success"] != false || item["result"].(map[string]interface{})["reason"] == "" {
t.Fatalf("wrong failure result: %+v", item)
}
done := read("done")
progress := done["progress"].(map[string]interface{})
result := done["result"].(map[string]interface{})
if progress["completed"] != float64(2) || progress["success"] != float64(1) || progress["failed"] != float64(1) || result["successCount"] != float64(1) || result["failCount"] != float64(1) {
t.Fatalf("incorrect final result: %+v", done)
}
if manager.reconcileCount() != 1 {
t.Fatalf("expected one execution, got %d", manager.reconcileCount())
}
if err := decoder.Decode(new(interface{})); err != io.EOF {
t.Fatalf("expected EOF after done, got %v", err)
}
}
func TestTunnelRedeployStreamStopsAfterDisconnect(t *testing.T) {
fixture := setupNftablesHandler(t)
h := fixture.handler
seedNftablesSSHConfig(t, h, fixture.nodeID)
first := seedTunnelForNftables(t, h, "first", fixture.nodeID)
second := seedTunnelForNftables(t, h, "second", fixture.nodeID)
manager := &blockingRedeployManager{entered: make(chan struct{}), release: make(chan struct{})}
h.nftablesManager = manager
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req := httptest.NewRequest(http.MethodPost, "/api/v1/tunnel/batch-redeploy/stream", strings.NewReader(fmt.Sprintf(`{"ids":[%d,%d]}`, first, second))).WithContext(ctx)
res := httptest.NewRecorder()
finished := make(chan struct{})
go func() { defer close(finished); h.tunnelBatchRedeployStream(res, req) }()
select {
case <-manager.entered:
case <-time.After(2 * time.Second):
close(manager.release)
t.Fatal("mutation did not start")
}
cancel()
close(manager.release)
select {
case <-finished:
case <-time.After(2 * time.Second):
t.Fatal("handler did not stop")
}
if manager.reconcileCount() != 1 {
t.Fatalf("started another mutation after disconnect: %d", manager.reconcileCount())
}
if strings.Contains(res.Body.String(), `"type":"done"`) {
t.Fatal("disconnected stream claimed completion")
}
}
type failedRedeployWriter struct{ *httptest.ResponseRecorder }
func (w failedRedeployWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }
func TestTunnelRedeployStreamDoesNotMutateWhenInitialWriteFails(t *testing.T) {
fixture := setupNftablesHandler(t)
h := fixture.handler
seedNftablesSSHConfig(t, h, fixture.nodeID)
id := seedTunnelForNftables(t, h, "stream-tunnel", fixture.nodeID)
manager := &fakeNftablesManager{}
h.nftablesManager = manager
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(fmt.Sprintf(`{"ids":[%d]}`, id)))
h.tunnelBatchRedeployStream(failedRedeployWriter{httptest.NewRecorder()}, req)
if manager.reconcileCount() != 0 {
t.Fatal("mutation ran despite failed progress stream")
}
}
type delayedRedeployManager struct{ fakeNftablesManager }
func (m *delayedRedeployManager) Reconcile(ctx context.Context, cfg runtimenft.SSHConfig, plan runtimenft.NodePlan) (runtimenft.ApplyResult, error) {
time.Sleep(100 * time.Millisecond)
return m.fakeNftablesManager.Reconcile(ctx, cfg, plan)
}
type shortRedeployDeadlineWriter struct {
http.ResponseWriter
deadlineCalls *atomic.Int32
}
func (w shortRedeployDeadlineWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
func (w shortRedeployDeadlineWriter) SetWriteDeadline(deadline time.Time) error {
remaining := time.Until(deadline)
if remaining < time.Minute || remaining > 2*time.Minute {
return fmt.Errorf("expected bounded two-minute write window, got %v", remaining)
}
w.deadlineCalls.Add(1)
// Shorten the production idle window for a real, fast timeout regression.
return http.NewResponseController(w.ResponseWriter).SetWriteDeadline(time.Now().Add(200 * time.Millisecond))
}
func TestTunnelRedeployStreamRenewsDeadlineThroughRequestLog(t *testing.T) {
fixture := setupNftablesHandler(t)
h := fixture.handler
seedNftablesSSHConfig(t, h, fixture.nodeID)
ids := make([]int64, 3)
for i := range ids {
ids[i] = seedTunnelForNftables(t, h, fmt.Sprintf("slow-%d", i), fixture.nodeID)
}
manager := &delayedRedeployManager{}
h.nftablesManager = manager
var deadlineCalls atomic.Int32
logged := middleware.RequestLog(http.HandlerFunc(h.tunnelBatchRedeployStream))
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
logged.ServeHTTP(shortRedeployDeadlineWriter{w, &deadlineCalls}, r)
}))
server.Config.WriteTimeout = 50 * time.Millisecond
server.Start()
defer server.Close()
body, _ := json.Marshal(map[string]interface{}{"ids": ids})
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Post(server.URL, "application/json", strings.NewReader(string(body)))
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
payload, err := io.ReadAll(response.Body)
if err != nil {
t.Fatalf("stream timed out across batch: %v (body %s)", err, payload)
}
if !strings.Contains(string(payload), `"type":"done"`) {
t.Fatalf("batch did not finish: %s", payload)
}
if manager.reconcileCount() != 3 {
t.Fatalf("wrong execution count: %d", manager.reconcileCount())
}
if deadlineCalls.Load() != 8 {
t.Fatalf("each of 8 events must renew its deadline, got %d", deadlineCalls.Load())
}
}
type flushFailureRedeployWriter struct {
*httptest.ResponseRecorder
flushes int
}
func (w *flushFailureRedeployWriter) FlushError() error {
w.flushes++
if w.flushes == 2 {
return io.ErrClosedPipe
}
w.ResponseRecorder.Flush()
return nil
}
func TestTunnelRedeployStreamStopsBeforeMutationOnProcessingFlushError(t *testing.T) {
fixture := setupNftablesHandler(t)
h := fixture.handler
seedNftablesSSHConfig(t, h, fixture.nodeID)
id := seedTunnelForNftables(t, h, "flush-error", fixture.nodeID)
manager := &fakeNftablesManager{}
h.nftablesManager = manager
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(fmt.Sprintf(`{"ids":[%d]}`, id)))
writer := &flushFailureRedeployWriter{ResponseRecorder: httptest.NewRecorder()}
middleware.RequestLog(http.HandlerFunc(h.tunnelBatchRedeployStream)).ServeHTTP(writer, req)
if writer.flushes != 2 {
t.Fatalf("expected start and processing flush, got %d", writer.flushes)
}
if manager.reconcileCount() != 0 {
t.Fatal("mutation ran after failed processing flush")
}
}
@@ -14,6 +14,11 @@ type statusWriter struct {
status int
}
// Unwrap preserves ResponseController features such as per-response deadlines.
func (w *statusWriter) Unwrap() http.ResponseWriter {
return w.ResponseWriter
}
func (w *statusWriter) WriteHeader(code int) {
w.status = code
w.ResponseWriter.WriteHeader(code)
@@ -27,10 +32,12 @@ func (w *statusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return hj.Hijack()
}
func (w *statusWriter) FlushError() error {
return http.NewResponseController(w.ResponseWriter).Flush()
}
func (w *statusWriter) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
_ = w.FlushError()
}
func (w *statusWriter) ReadFrom(r io.Reader) (int64, error) {
@@ -0,0 +1,98 @@
package contract_test
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"go-backend/internal/auth"
)
func TestTunnelRedeployStreamContract(t *testing.T) {
const secret = "contract-jwt-secret"
router, _ := setupContractRouter(t, secret)
token, err := auth.GenerateToken(1, "admin_user", 0, secret)
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodPost, "/api/v1/tunnel/batch-redeploy/stream", strings.NewReader(`{"ids":[999998,999999]}`))
req.Header.Set("Authorization", token)
res := httptest.NewRecorder()
router.ServeHTTP(res, req)
if !strings.HasPrefix(res.Header().Get("Content-Type"), "application/x-ndjson") {
t.Fatalf("expected streamed progress, got %s: %s", res.Header().Get("Content-Type"), res.Body.String())
}
decoder := json.NewDecoder(res.Body)
for _, want := range []string{"start", "processing", "item", "processing", "item", "done"} {
var event struct {
Type string `json:"type"`
Data json.RawMessage `json:"data"`
}
if err := decoder.Decode(&event); err != nil {
t.Fatal(err)
}
if event.Type != want {
t.Fatalf("got %s, want %s", event.Type, want)
}
if want == "item" {
var data struct{ Result map[string]interface{} }
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatal(err)
}
if _, ok := data.Result["name"].(string); !ok {
t.Fatalf("item name must be a string, including missing tunnels: %s", event.Data)
}
}
if want == "done" {
var data struct {
Progress struct{ Total, Completed, Success, Failed int }
Result struct {
SuccessCount, FailCount int
Failures []struct {
ID int64
Reason string
}
}
}
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatal(err)
}
if data.Progress.Total != 2 || data.Progress.Completed != 2 || data.Progress.Failed != 2 || data.Progress.Success != 0 || data.Result.FailCount != 2 || len(data.Result.Failures) != 2 {
t.Fatalf("incorrect completion: %+v", data)
}
if data.Result.Failures[0].Reason == "" {
t.Fatal("missing failure detail")
}
}
}
if !res.Flushed {
t.Fatal("stream did not flush")
}
}
func TestTunnelRedeployStreamAuthorizationContract(t *testing.T) {
const secret = "contract-jwt-secret"
router, repository := setupContractRouter(t, secret)
seedLegacyUser(t, repository, 9111, "stream-normal-user", "password")
token, err := auth.GenerateToken(9111, "stream-normal-user", 1, secret)
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name, token string
code int
}{{"anonymous", "", 401}, {"non-admin", token, 403}} {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/v1/tunnel/batch-redeploy/stream", strings.NewReader(`{"ids":[999999]}`))
req.Header.Set("Authorization", tc.token)
res := httptest.NewRecorder()
router.ServeHTTP(res, req)
assertCode(t, res, tc.code)
if strings.Contains(res.Body.String(), `"type":"start"`) {
t.Fatal("unauthorized request started stream")
}
})
}
}