mirror of
https://github.com/Sagit-chu/flvx.git
synced 2026-11-16 17:34:23 +08:00
fix: show live node and tunnel deployment progress in list view
This commit is contained in:
@@ -228,6 +228,7 @@ func (h *Handler) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("/api/v1/tunnel/update-order", h.tunnelUpdateOrder)
|
||||
mux.HandleFunc("/api/v1/tunnel/batch-delete", h.tunnelBatchDelete)
|
||||
mux.HandleFunc("/api/v1/tunnel/batch-redeploy", h.tunnelBatchRedeploy)
|
||||
mux.HandleFunc("/api/v1/tunnel/batch-redeploy/stream", h.tunnelBatchRedeployStream)
|
||||
mux.HandleFunc("/api/v1/tunnel/user/assign", h.userTunnelAssign)
|
||||
mux.HandleFunc("/api/v1/tunnel/user/batch-assign", h.userTunnelBatchAssign)
|
||||
mux.HandleFunc("/api/v1/tunnel/user/remove", h.userTunnelRemove)
|
||||
|
||||
@@ -2046,51 +2046,37 @@ func normalizeBatchFailureReason(reason string) string {
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func (h *Handler) redeployTunnelConfiguration(tunnelID int64) error {
|
||||
if _, err := h.getTunnelRecord(tunnelID); err != nil {
|
||||
return err
|
||||
}
|
||||
if nftMode, entryNodeIDs, err := h.tunnelUsesNftables(tunnelID); err != nil {
|
||||
return err
|
||||
} else if nftMode {
|
||||
if len(entryNodeIDs) == 0 {
|
||||
return errors.New("nftables 转发缺少入口节点")
|
||||
}
|
||||
return h.reconcileNftablesNodeByRequest(entryNodeIDs[0])
|
||||
}
|
||||
return h.redeployTunnelAndForwards(tunnelID)
|
||||
}
|
||||
|
||||
func (h *Handler) tunnelBatchRedeploy(w http.ResponseWriter, r *http.Request) {
|
||||
ids := idsFromBody(r, w)
|
||||
if ids == nil {
|
||||
return
|
||||
}
|
||||
success := 0
|
||||
fail := 0
|
||||
failures := make([]batchFailureDetail, 0)
|
||||
result := batchOperationResult{}
|
||||
for _, tunnelID := range ids {
|
||||
tunnel, tunnelErr := h.getTunnelRecord(tunnelID)
|
||||
tunnelName := ""
|
||||
if tunnelErr == nil && tunnel != nil {
|
||||
tunnelName, _ = h.repo.GetTunnelName(tunnelID)
|
||||
name, _ := h.repo.GetTunnelName(tunnelID)
|
||||
if err := h.redeployTunnelConfiguration(tunnelID); err != nil {
|
||||
result.FailCount++
|
||||
result.Failures = appendBatchFailure(result.Failures, tunnelID, name, err)
|
||||
} else {
|
||||
result.SuccessCount++
|
||||
}
|
||||
if tunnelErr != nil {
|
||||
fail++
|
||||
failures = appendBatchFailure(failures, tunnelID, tunnelName, tunnelErr)
|
||||
continue
|
||||
}
|
||||
if nftMode, entryNodeIDs, modeErr := h.tunnelUsesNftables(tunnelID); modeErr != nil {
|
||||
fail++
|
||||
failures = appendBatchFailure(failures, tunnelID, tunnelName, modeErr)
|
||||
continue
|
||||
} else if nftMode {
|
||||
if len(entryNodeIDs) == 0 {
|
||||
fail++
|
||||
failures = appendBatchFailureReason(failures, tunnelID, tunnelName, "nftables 转发缺少入口节点")
|
||||
continue
|
||||
}
|
||||
if reconcileErr := h.reconcileNftablesNodeByRequest(entryNodeIDs[0]); reconcileErr != nil {
|
||||
fail++
|
||||
failures = appendBatchFailure(failures, tunnelID, tunnelName, reconcileErr)
|
||||
continue
|
||||
}
|
||||
success++
|
||||
continue
|
||||
}
|
||||
if err := h.redeployTunnelAndForwards(tunnelID); err != nil {
|
||||
fail++
|
||||
failures = appendBatchFailure(failures, tunnelID, tunnelName, err)
|
||||
continue
|
||||
}
|
||||
success++
|
||||
}
|
||||
response.WriteJSON(w, response.OK(batchOperationResult{SuccessCount: success, FailCount: fail, Failures: failures}))
|
||||
response.WriteJSON(w, response.OK(result))
|
||||
}
|
||||
|
||||
func (h *Handler) userTunnelAssign(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/http/response"
|
||||
)
|
||||
|
||||
type redeployProgress struct {
|
||||
Total int `json:"total"`
|
||||
Completed int `json:"completed"`
|
||||
Success int `json:"success"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
type redeployItemResult struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Success bool `json:"success"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// tunnelBatchRedeployStream reports completed tunnels, not estimated runtime.
|
||||
// A disconnected caller may have an in-flight mutation whose result it did not
|
||||
// receive. Finish that mutation, but never start another or automatically retry.
|
||||
func (h *Handler) tunnelBatchRedeployStream(w http.ResponseWriter, r *http.Request) {
|
||||
ids := idsFromBody(r, w)
|
||||
if ids == nil {
|
||||
return
|
||||
}
|
||||
// idsFromBody sorts IDs; ignore duplicate selections so each tunnel runs once.
|
||||
unique := ids[:0]
|
||||
for _, id := range ids {
|
||||
if len(unique) == 0 || unique[len(unique)-1] != id {
|
||||
unique = append(unique, id)
|
||||
}
|
||||
}
|
||||
ids = unique
|
||||
if len(ids) == 0 {
|
||||
response.WriteJSON(w, response.ErrDefault("ids不能为空"))
|
||||
return
|
||||
}
|
||||
_, err := prepareDiagnosisStreamResponse(w)
|
||||
if err != nil {
|
||||
response.WriteJSON(w, response.ErrDefault(err.Error()))
|
||||
return
|
||||
}
|
||||
encoder := json.NewEncoder(w)
|
||||
controller := http.NewResponseController(w)
|
||||
emit := func(kind string, data interface{}) bool {
|
||||
if r.Context().Err() != nil {
|
||||
return false
|
||||
}
|
||||
// The server's WriteTimeout is an absolute request deadline. Renew only
|
||||
// this stream's bounded write window as progress is emitted, so a long
|
||||
// batch is not cut off despite making steady progress. Recorders and
|
||||
// other non-network writers may not support deadlines.
|
||||
if err := controller.SetWriteDeadline(time.Now().Add(2 * time.Minute)); err != nil && !errors.Is(err, http.ErrNotSupported) {
|
||||
return false
|
||||
}
|
||||
event := diagnosisStreamEvent{Type: kind, Data: data, TS: time.Now().UnixMilli()}
|
||||
if err := encoder.Encode(event); err != nil {
|
||||
return false
|
||||
}
|
||||
// Buffered writes can succeed while the actual socket flush fails.
|
||||
// Do not start a mutation after its processing event failed to flush.
|
||||
return controller.Flush() == nil
|
||||
}
|
||||
progress := redeployProgress{Total: len(ids)}
|
||||
result := batchOperationResult{}
|
||||
if !emit("start", progress) {
|
||||
return
|
||||
}
|
||||
for _, id := range ids {
|
||||
if r.Context().Err() != nil {
|
||||
return
|
||||
}
|
||||
name, _ := h.repo.GetTunnelName(id)
|
||||
if !emit("processing", map[string]interface{}{"id": id, "name": name, "progress": progress}) {
|
||||
return
|
||||
}
|
||||
if r.Context().Err() != nil {
|
||||
return
|
||||
}
|
||||
item := redeployItemResult{ID: id, Name: name}
|
||||
if err := h.redeployTunnelConfiguration(id); err != nil {
|
||||
result.FailCount++
|
||||
result.Failures = appendBatchFailure(result.Failures, id, name, err)
|
||||
item.Reason = result.Failures[len(result.Failures)-1].Reason
|
||||
progress.Failed++
|
||||
} else {
|
||||
result.SuccessCount++
|
||||
item.Success = true
|
||||
progress.Success++
|
||||
}
|
||||
progress.Completed++
|
||||
if !emit("item", map[string]interface{}{"result": item, "progress": progress}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
emit("done", map[string]interface{}{"progress": progress, "result": result})
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/http/middleware"
|
||||
runtimenft "go-backend/internal/runtime/nftables"
|
||||
)
|
||||
|
||||
type blockingRedeployManager struct {
|
||||
fakeNftablesManager
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (m *blockingRedeployManager) Reconcile(ctx context.Context, cfg runtimenft.SSHConfig, plan runtimenft.NodePlan) (runtimenft.ApplyResult, error) {
|
||||
m.once.Do(func() { close(m.entered); <-m.release })
|
||||
return m.fakeNftablesManager.Reconcile(ctx, cfg, plan)
|
||||
}
|
||||
|
||||
func TestTunnelRedeployStreamFlushesBeforeSlowMutationAndReportsMixedResults(t *testing.T) {
|
||||
fixture := setupNftablesHandler(t)
|
||||
h := fixture.handler
|
||||
seedNftablesSSHConfig(t, h, fixture.nodeID)
|
||||
id := seedTunnelForNftables(t, h, "stream-tunnel", fixture.nodeID)
|
||||
manager := &blockingRedeployManager{entered: make(chan struct{}), release: make(chan struct{})}
|
||||
h.nftablesManager = manager
|
||||
var releaseOnce sync.Once
|
||||
release := func() { releaseOnce.Do(func() { close(manager.release) }) }
|
||||
defer release()
|
||||
server := httptest.NewServer(http.HandlerFunc(h.tunnelBatchRedeployStream))
|
||||
defer server.Close()
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
response, err := client.Post(server.URL, "application/json", strings.NewReader(fmt.Sprintf(`{"ids":[%d,999999,%d]}`, id, id)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
decoder := json.NewDecoder(response.Body)
|
||||
read := func(want string) map[string]interface{} {
|
||||
t.Helper()
|
||||
var event struct {
|
||||
Type string
|
||||
Data map[string]interface{}
|
||||
}
|
||||
if err := decoder.Decode(&event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if event.Type != want {
|
||||
t.Fatalf("expected %s, got %s", want, event.Type)
|
||||
}
|
||||
return event.Data
|
||||
}
|
||||
if start := read("start"); start["total"] != float64(2) {
|
||||
t.Fatalf("duplicate IDs not deduplicated: %+v", start)
|
||||
}
|
||||
processing := read("processing")
|
||||
if processing["name"] != "stream-tunnel" || processing["id"] != float64(id) {
|
||||
t.Fatalf("wrong current tunnel: %+v", processing)
|
||||
}
|
||||
select {
|
||||
case <-manager.entered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("did not start mutation")
|
||||
}
|
||||
// Both events arrived over HTTP while the actual mutation is still blocked.
|
||||
release()
|
||||
item := read("item")
|
||||
if item["result"].(map[string]interface{})["success"] != true || item["progress"].(map[string]interface{})["completed"] != float64(1) {
|
||||
t.Fatalf("wrong success result: %+v", item)
|
||||
}
|
||||
read("processing")
|
||||
item = read("item")
|
||||
if item["result"].(map[string]interface{})["success"] != false || item["result"].(map[string]interface{})["reason"] == "" {
|
||||
t.Fatalf("wrong failure result: %+v", item)
|
||||
}
|
||||
done := read("done")
|
||||
progress := done["progress"].(map[string]interface{})
|
||||
result := done["result"].(map[string]interface{})
|
||||
if progress["completed"] != float64(2) || progress["success"] != float64(1) || progress["failed"] != float64(1) || result["successCount"] != float64(1) || result["failCount"] != float64(1) {
|
||||
t.Fatalf("incorrect final result: %+v", done)
|
||||
}
|
||||
if manager.reconcileCount() != 1 {
|
||||
t.Fatalf("expected one execution, got %d", manager.reconcileCount())
|
||||
}
|
||||
if err := decoder.Decode(new(interface{})); err != io.EOF {
|
||||
t.Fatalf("expected EOF after done, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTunnelRedeployStreamStopsAfterDisconnect(t *testing.T) {
|
||||
fixture := setupNftablesHandler(t)
|
||||
h := fixture.handler
|
||||
seedNftablesSSHConfig(t, h, fixture.nodeID)
|
||||
first := seedTunnelForNftables(t, h, "first", fixture.nodeID)
|
||||
second := seedTunnelForNftables(t, h, "second", fixture.nodeID)
|
||||
manager := &blockingRedeployManager{entered: make(chan struct{}), release: make(chan struct{})}
|
||||
h.nftablesManager = manager
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/tunnel/batch-redeploy/stream", strings.NewReader(fmt.Sprintf(`{"ids":[%d,%d]}`, first, second))).WithContext(ctx)
|
||||
res := httptest.NewRecorder()
|
||||
finished := make(chan struct{})
|
||||
go func() { defer close(finished); h.tunnelBatchRedeployStream(res, req) }()
|
||||
select {
|
||||
case <-manager.entered:
|
||||
case <-time.After(2 * time.Second):
|
||||
close(manager.release)
|
||||
t.Fatal("mutation did not start")
|
||||
}
|
||||
cancel()
|
||||
close(manager.release)
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("handler did not stop")
|
||||
}
|
||||
if manager.reconcileCount() != 1 {
|
||||
t.Fatalf("started another mutation after disconnect: %d", manager.reconcileCount())
|
||||
}
|
||||
if strings.Contains(res.Body.String(), `"type":"done"`) {
|
||||
t.Fatal("disconnected stream claimed completion")
|
||||
}
|
||||
}
|
||||
|
||||
type failedRedeployWriter struct{ *httptest.ResponseRecorder }
|
||||
|
||||
func (w failedRedeployWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }
|
||||
|
||||
func TestTunnelRedeployStreamDoesNotMutateWhenInitialWriteFails(t *testing.T) {
|
||||
fixture := setupNftablesHandler(t)
|
||||
h := fixture.handler
|
||||
seedNftablesSSHConfig(t, h, fixture.nodeID)
|
||||
id := seedTunnelForNftables(t, h, "stream-tunnel", fixture.nodeID)
|
||||
manager := &fakeNftablesManager{}
|
||||
h.nftablesManager = manager
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(fmt.Sprintf(`{"ids":[%d]}`, id)))
|
||||
h.tunnelBatchRedeployStream(failedRedeployWriter{httptest.NewRecorder()}, req)
|
||||
if manager.reconcileCount() != 0 {
|
||||
t.Fatal("mutation ran despite failed progress stream")
|
||||
}
|
||||
}
|
||||
|
||||
type delayedRedeployManager struct{ fakeNftablesManager }
|
||||
|
||||
func (m *delayedRedeployManager) Reconcile(ctx context.Context, cfg runtimenft.SSHConfig, plan runtimenft.NodePlan) (runtimenft.ApplyResult, error) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return m.fakeNftablesManager.Reconcile(ctx, cfg, plan)
|
||||
}
|
||||
|
||||
type shortRedeployDeadlineWriter struct {
|
||||
http.ResponseWriter
|
||||
deadlineCalls *atomic.Int32
|
||||
}
|
||||
|
||||
func (w shortRedeployDeadlineWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
|
||||
func (w shortRedeployDeadlineWriter) SetWriteDeadline(deadline time.Time) error {
|
||||
remaining := time.Until(deadline)
|
||||
if remaining < time.Minute || remaining > 2*time.Minute {
|
||||
return fmt.Errorf("expected bounded two-minute write window, got %v", remaining)
|
||||
}
|
||||
w.deadlineCalls.Add(1)
|
||||
// Shorten the production idle window for a real, fast timeout regression.
|
||||
return http.NewResponseController(w.ResponseWriter).SetWriteDeadline(time.Now().Add(200 * time.Millisecond))
|
||||
}
|
||||
|
||||
func TestTunnelRedeployStreamRenewsDeadlineThroughRequestLog(t *testing.T) {
|
||||
fixture := setupNftablesHandler(t)
|
||||
h := fixture.handler
|
||||
seedNftablesSSHConfig(t, h, fixture.nodeID)
|
||||
ids := make([]int64, 3)
|
||||
for i := range ids {
|
||||
ids[i] = seedTunnelForNftables(t, h, fmt.Sprintf("slow-%d", i), fixture.nodeID)
|
||||
}
|
||||
manager := &delayedRedeployManager{}
|
||||
h.nftablesManager = manager
|
||||
var deadlineCalls atomic.Int32
|
||||
logged := middleware.RequestLog(http.HandlerFunc(h.tunnelBatchRedeployStream))
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
logged.ServeHTTP(shortRedeployDeadlineWriter{w, &deadlineCalls}, r)
|
||||
}))
|
||||
server.Config.WriteTimeout = 50 * time.Millisecond
|
||||
server.Start()
|
||||
defer server.Close()
|
||||
body, _ := json.Marshal(map[string]interface{}{"ids": ids})
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
response, err := client.Post(server.URL, "application/json", strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
payload, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("stream timed out across batch: %v (body %s)", err, payload)
|
||||
}
|
||||
if !strings.Contains(string(payload), `"type":"done"`) {
|
||||
t.Fatalf("batch did not finish: %s", payload)
|
||||
}
|
||||
if manager.reconcileCount() != 3 {
|
||||
t.Fatalf("wrong execution count: %d", manager.reconcileCount())
|
||||
}
|
||||
if deadlineCalls.Load() != 8 {
|
||||
t.Fatalf("each of 8 events must renew its deadline, got %d", deadlineCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
type flushFailureRedeployWriter struct {
|
||||
*httptest.ResponseRecorder
|
||||
flushes int
|
||||
}
|
||||
|
||||
func (w *flushFailureRedeployWriter) FlushError() error {
|
||||
w.flushes++
|
||||
if w.flushes == 2 {
|
||||
return io.ErrClosedPipe
|
||||
}
|
||||
w.ResponseRecorder.Flush()
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTunnelRedeployStreamStopsBeforeMutationOnProcessingFlushError(t *testing.T) {
|
||||
fixture := setupNftablesHandler(t)
|
||||
h := fixture.handler
|
||||
seedNftablesSSHConfig(t, h, fixture.nodeID)
|
||||
id := seedTunnelForNftables(t, h, "flush-error", fixture.nodeID)
|
||||
manager := &fakeNftablesManager{}
|
||||
h.nftablesManager = manager
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(fmt.Sprintf(`{"ids":[%d]}`, id)))
|
||||
writer := &flushFailureRedeployWriter{ResponseRecorder: httptest.NewRecorder()}
|
||||
middleware.RequestLog(http.HandlerFunc(h.tunnelBatchRedeployStream)).ServeHTTP(writer, req)
|
||||
if writer.flushes != 2 {
|
||||
t.Fatalf("expected start and processing flush, got %d", writer.flushes)
|
||||
}
|
||||
if manager.reconcileCount() != 0 {
|
||||
t.Fatal("mutation ran after failed processing flush")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user