[优化] 优化 WAF IP 组同步功能及相关文档更新

openflare_agent/internal/agent/runner.go

@@ -24,6 +24,8 @@ type SyncService interface {
 	SyncOnStartup(ctx context.Context, target *protocol.ActiveConfigMeta) error
 	SyncOnce(ctx context.Context, target *protocol.ActiveConfigMeta) error
 	ForceSyncOnce(ctx context.Context, target *protocol.ActiveConfigMeta) error
+	WAFIPGroupChecksums() (map[string]string, error)
+	ApplyWAFIPGroups(ctx context.Context, groups []protocol.WAFIPGroup) error
 }
 
 type Updater interface {
@@ -161,6 +163,7 @@ func (r *Runner) performHeartbeatCycle(ctx context.Context, nodeID string, start
 	}
 	slog.Debug("agent heartbeat succeeded", "mode", mode, "node_id", nodeID)
 	changed := r.applySettings(heartbeatResult.AgentSettings)
+	r.applyWAFIPGroups(ctx, heartbeatResult.WAFIPGroups)
 	if startup {
 		if err = r.SyncService.SyncOnStartup(ctx, heartbeatResult.ActiveConfig); err != nil {
 			r.recordSyncError(err)
@@ -307,6 +310,14 @@ func (r *Runner) handleWebSocketMessage(ctx context.Context, message protocol.WS
 			slog.Error("agent ws triggered force sync failed", "version", target.Version, "error", err)
 		}
 		return false, nil
+	case protocol.WSMessageTypeWAFIPGroups:
+		var groups []protocol.WAFIPGroup
+		if err := json.Unmarshal(message.Payload, &groups); err != nil {
+			slog.Debug("agent ws waf ip groups decode failed", "error", err)
+			return false, nil
+		}
+		r.applyWAFIPGroups(ctx, groups)
+		return false, nil
 	case protocol.WSMessageTypePing:
 		slog.Debug("agent ws ping received")
 		return false, conn.SendPong()
@@ -470,6 +481,7 @@ func (r *Runner) tryRegister(ctx context.Context, nodeID *string) error {
 		heartbeatResult = &protocol.HeartbeatResult{}
 	}
 	r.applySettings(heartbeatResult.AgentSettings)
+	r.applyWAFIPGroups(ctx, heartbeatResult.WAFIPGroups)
 	if err = r.SyncService.SyncOnStartup(ctx, heartbeatResult.ActiveConfig); err != nil {
 		r.recordSyncError(err)
 		slog.Error("agent post-register startup sync failed", "error", err)
@@ -563,7 +575,7 @@ func (r *Runner) nodePayload(nodeID string) protocol.NodePayload {
 	}
 	metricSnapshot := observability.BuildSnapshot(r.Config, r.StateStore, managedOpenRestyMetrics)
 	healthEvents := observability.BuildHealthEvents(snapshot)
-	return protocol.NodePayload{
+	payload := protocol.NodePayload{
 		NodeID:           nodeID,
 		Name:             r.Config.NodeName,
 		IP:               r.Config.NodeIP,
@@ -579,6 +591,25 @@ func (r *Runner) nodePayload(nodeID string) protocol.NodePayload {
 		AccessLogs:       accessLogs,
 		HealthEvents:     healthEvents,
 	}
+	if r.SyncService != nil {
+		checksums, err := r.SyncService.WAFIPGroupChecksums()
+		if err != nil {
+			slog.Debug("load local waf ip group checksums failed", "error", err)
+		} else if len(checksums) > 0 {
+			payload.WAFIPGroupChecksums = checksums
+		}
+	}
+	return payload
+}
+
+func (r *Runner) applyWAFIPGroups(ctx context.Context, groups []protocol.WAFIPGroup) {
+	if len(groups) == 0 || r.SyncService == nil {
+		return
+	}
+	if err := r.SyncService.ApplyWAFIPGroups(ctx, groups); err != nil {
+		r.recordSyncError(err)
+		slog.Error("agent apply waf ip groups failed", "error", err)
+	}
 }
 
 func (r *Runner) prepareHeartbeatPayload(nodeID string) (protocol.NodePayload, []int64) {

openflare_agent/internal/agent/runner_test.go

@@ -70,6 +70,8 @@ type fakeSyncService struct {
 	syncOnceCalls  int
 	lastTarget     *protocol.ActiveConfigMeta
 	onSyncOnceCall func(int)
+	wafChecksums   map[string]string
+	wafGroups      []protocol.WAFIPGroup
 }
 
 type fakeRuntimeManager struct {
@@ -135,6 +137,20 @@ func (f *fakeSyncService) ForceSyncOnce(ctx context.Context, target *protocol.Ac
 	return f.syncOnceErr
 }
 
+func (f *fakeSyncService) WAFIPGroupChecksums() (map[string]string, error) {
+	if f.wafChecksums == nil {
+		return map[string]string{}, nil
+	}
+	return f.wafChecksums, nil
+}
+
+func (f *fakeSyncService) ApplyWAFIPGroups(ctx context.Context, groups []protocol.WAFIPGroup) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.wafGroups = append(f.wafGroups, groups...)
+	return nil
+}
+
 type fakeWebSocketConnection struct {
 	pongCalls int
 }

openflare_agent/internal/httpclient/client.go

@@ -54,6 +54,7 @@ func (c *Client) Heartbeat(ctx context.Context, payload protocol.NodePayload) (*
 	return &protocol.HeartbeatResult{
 		AgentSettings: resp.AgentSettings,
 		ActiveConfig:  resp.ActiveConfig,
+		WAFIPGroups:   resp.WAFIPGroups,
 	}, nil
 }
 
@@ -74,6 +75,17 @@ func (c *Client) ReportApplyLog(ctx context.Context, payload protocol.ApplyLogPa
 	return c.postJSON(ctx, "/api/agent/apply-logs", payload, nil)
 }
 
+func (c *Client) SyncWAFIPGroups(ctx context.Context, payload protocol.WAFIPGroupSyncRequest) (*protocol.WAFIPGroupSyncResponse, error) {
+	resp := protocol.APIResponse[protocol.WAFIPGroupSyncResponse]{}
+	if err := c.postJSON(ctx, "/api/agent/waf/ip-groups/sync", payload, &resp); err != nil {
+		return nil, err
+	}
+	if !resp.Success {
+		return nil, errors.New(resp.Message)
+	}
+	return &resp.Data, nil
+}
+
 func (c *Client) SetToken(token string) {
 	c.token = strings.TrimSpace(token)
 	slog.Debug("http client token updated")

openflare_agent/internal/nginx/manager.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -27,6 +28,7 @@ import (
 
 const RuntimeConfigDirPlaceholder = "__OPENFLARE_RUNTIME_CONFIG_DIR__"
 const ResolverDirectivePlaceholder = "__OPENFLARE_RESOLVER_DIRECTIVE__"
+const WAFIPGroupsConfigFileName = "waf_ip_groups.json"
 
 type Executor interface {
 	Test(ctx context.Context) error
@@ -190,6 +192,10 @@ type ApplyOutcome struct {
 	Message string
 }
 
+type wafIPGroupsRuntimeConfig struct {
+	Groups map[string]protocol.WAFIPGroup `json:"groups"`
+}
+
 func (m *Manager) Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) ApplyOutcome {
 	slog.Info("openresty apply started", "main_config", m.MainConfigPath, "route_config", m.RouteConfigPath, "cert_files", len(supportFiles))
 	backup, err := m.backup()
@@ -425,6 +431,77 @@ func (m *Manager) CurrentChecksum() (string, error) {
 	return result, nil
 }
 
+func (m *Manager) WAFIPGroupChecksums() (map[string]string, error) {
+	config, err := m.readWAFIPGroupsRuntimeConfig()
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[string]string, len(config.Groups))
+	for id, group := range config.Groups {
+		if strings.TrimSpace(group.Checksum) != "" {
+			result[id] = strings.TrimSpace(group.Checksum)
+		}
+	}
+	return result, nil
+}
+
+func (m *Manager) SyncWAFIPGroups(groups []protocol.WAFIPGroup) error {
+	if m.RuntimeConfigDir == "" || len(groups) == 0 {
+		return nil
+	}
+	config, err := m.readWAFIPGroupsRuntimeConfig()
+	if err != nil {
+		return err
+	}
+	if config.Groups == nil {
+		config.Groups = make(map[string]protocol.WAFIPGroup)
+	}
+	for _, group := range groups {
+		if group.ID == 0 {
+			continue
+		}
+		config.Groups[fmt.Sprintf("%d", group.ID)] = group
+	}
+	data, err := json.Marshal(config)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(m.RuntimeConfigDir, 0o755); err != nil {
+		return err
+	}
+	path := filepath.Join(m.RuntimeConfigDir, WAFIPGroupsConfigFileName)
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		return fmt.Errorf("write %s: %w", WAFIPGroupsConfigFileName, err)
+	}
+	slog.Info("synced waf ip groups", "path", path, "group_count", len(groups))
+	return nil
+}
+
+func (m *Manager) readWAFIPGroupsRuntimeConfig() (*wafIPGroupsRuntimeConfig, error) {
+	config := &wafIPGroupsRuntimeConfig{Groups: map[string]protocol.WAFIPGroup{}}
+	if m.RuntimeConfigDir == "" {
+		return config, nil
+	}
+	path := filepath.Join(m.RuntimeConfigDir, WAFIPGroupsConfigFileName)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return config, nil
+		}
+		return nil, err
+	}
+	if len(data) == 0 {
+		return config, nil
+	}
+	if err := json.Unmarshal(data, config); err != nil {
+		return nil, err
+	}
+	if config.Groups == nil {
+		config.Groups = map[string]protocol.WAFIPGroup{}
+	}
+	return config, nil
+}
+
 type ExecutorOptions struct {
 	NginxPath                  string
 	MainConfigPath             string

openflare_agent/internal/nginx/manager_test.go

@@ -915,6 +915,37 @@ func TestManagerApplyRejectsCertFilePathTraversal(t *testing.T) {
 	}
 }
 
+func TestManagerSyncWAFIPGroupsWritesDeltaRuntimeFile(t *testing.T) {
+	manager := &Manager{RuntimeConfigDir: t.TempDir()}
+
+	if err := manager.SyncWAFIPGroups([]protocol.WAFIPGroup{
+		{ID: 1, Enabled: true, IPList: []string{"203.0.113.10"}, Checksum: "sum-1"},
+	}); err != nil {
+		t.Fatalf("SyncWAFIPGroups failed: %v", err)
+	}
+	if err := manager.SyncWAFIPGroups([]protocol.WAFIPGroup{
+		{ID: 2, Enabled: true, IPList: []string{"198.51.100.10"}, Checksum: "sum-2"},
+	}); err != nil {
+		t.Fatalf("SyncWAFIPGroups second delta failed: %v", err)
+	}
+
+	checksums, err := manager.WAFIPGroupChecksums()
+	if err != nil {
+		t.Fatalf("WAFIPGroupChecksums failed: %v", err)
+	}
+	if checksums["1"] != "sum-1" || checksums["2"] != "sum-2" {
+		t.Fatalf("expected merged checksums, got %#v", checksums)
+	}
+	data, err := os.ReadFile(filepath.Join(manager.RuntimeConfigDir, WAFIPGroupsConfigFileName))
+	if err != nil {
+		t.Fatalf("failed to read runtime ip group file: %v", err)
+	}
+	text := string(data)
+	if !strings.Contains(text, "203.0.113.10") || !strings.Contains(text, "198.51.100.10") {
+		t.Fatalf("expected runtime file to keep both groups, got %s", text)
+	}
+}
+
 func TestObservabilityListenAddress(t *testing.T) {
 	if got := ObservabilityListenAddress(18081); got != "127.0.0.1:18081" {
 		t.Fatalf("unexpected default observability listen address: %s", got)

openflare_agent/internal/nginx/waf_assets.go

@@ -49,6 +49,36 @@ local function load_config()
     return nil
 end
 
+local function load_ip_groups()
+    local paths = {
+        "__OPENFLARE_RUNTIME_CONFIG_DIR__/waf_ip_groups.json",
+        "/etc/nginx/openflare-lua/waf_ip_groups.json",
+        "/usr/local/openresty/nginx/conf/waf_ip_groups.json"
+    }
+    for _, path in ipairs(paths) do
+        local content = read_file(path)
+        if content and content ~= "" then
+            local hash = ngx.md5(content)
+            if config_dict:get("_ip_groups_hash") == hash then
+                local cached = config_dict:get("_ip_groups_json")
+                if cached then
+                    local decoded = cjson.decode(cached)
+                    if decoded then
+                        return decoded
+                    end
+                end
+            end
+            local decoded = cjson.decode(content)
+            if decoded then
+                config_dict:set("_ip_groups_hash", hash, 0)
+                config_dict:set("_ip_groups_json", content, 0)
+                return decoded
+            end
+        end
+    end
+    return { groups = {} }
+end
+
 local function list_contains(items, value)
     if not items or type(items) ~= "table" or not value or value == "" then
         return false
@@ -109,6 +139,20 @@ local function ip_matches(items, ip)
     return false
 end
 
+local function ip_matches_group_ids(group_ids, ip, ip_groups_config)
+    if not group_ids or type(group_ids) ~= "table" or not ip or ip == "" then
+        return false
+    end
+    local groups = (ip_groups_config or {}).groups or {}
+    for _, id in ipairs(group_ids) do
+        local group = groups[tostring(id)]
+        if group and group.enabled and ip_matches(group.ip_list, ip) then
+            return true
+        end
+    end
+    return false
+end
+
 local function lookup_country(ip)
     local ok, maxminddb = pcall(require, "resty.maxminddb")
     if not ok or not maxminddb then
@@ -181,6 +225,7 @@ end
 
 local ip = ngx.var.remote_addr or ""
 local groups = active_groups(config)
+local ip_groups_config = load_ip_groups()
 if #groups == 0 then
     if config_dict:add("_empty_groups_logged", true, 60) then
         ngx.log(ngx.WARN, "openflare waf has no active rule group for site: ", ngx.var.openflare_waf_site or "")
@@ -189,7 +234,7 @@ if #groups == 0 then
 end
 
 for _, group in ipairs(groups) do
-    if ip_matches(group.ip_whitelist, ip) then
+    if ip_matches(group.ip_whitelist, ip) or ip_matches_group_ids(group.ip_whitelist_group_ids, ip, ip_groups_config) then
         return
     end
 end
@@ -205,7 +250,7 @@ for _, group in ipairs(groups) do
 end
 
 for _, group in ipairs(groups) do
-    if ip_matches(group.ip_blacklist, ip) then
+    if ip_matches(group.ip_blacklist, ip) or ip_matches_group_ids(group.ip_blacklist_group_ids, ip, ip_groups_config) then
         return exit_with_group(group)
     end
 end

openflare_agent/internal/protocol/agent_api.go

@@ -14,11 +14,13 @@ type HeartbeatAPIResponse struct {
 	Data          any               `json:"data"`
 	AgentSettings *AgentSettings    `json:"agent_settings,omitempty"`
 	ActiveConfig  *ActiveConfigMeta `json:"active_config,omitempty"`
+	WAFIPGroups   []WAFIPGroup      `json:"waf_ip_groups,omitempty"`
 }
 
 type HeartbeatResult struct {
 	AgentSettings *AgentSettings
 	ActiveConfig  *ActiveConfigMeta
+	WAFIPGroups   []WAFIPGroup
 }
 
 type AgentSettings struct {
@@ -37,6 +39,7 @@ const (
 	WSMessageTypeSettings        = "settings"
 	WSMessageTypeActiveConfig    = "active_config"
 	WSMessageTypeForceSyncConfig = "force_sync_config"
+	WSMessageTypeWAFIPGroups     = "waf_ip_groups"
 	WSMessageTypePing            = "ping"
 	WSMessageTypePong            = "pong"
 )
@@ -81,6 +84,7 @@ type NodePayload struct {
 	AccessLogs            []NodeAccessLog               `json:"access_logs,omitempty"`
 	BufferedObservability []BufferedObservabilityRecord `json:"buffered_observability,omitempty"`
 	HealthEvents          []NodeHealthEvent             `json:"health_events"`
+	WAFIPGroupChecksums   map[string]string             `json:"waf_ip_group_checksums,omitempty"`
 }
 
 type NodeSystemProfile struct {
@@ -177,6 +181,24 @@ type ActiveConfigMeta struct {
 	Checksum string `json:"checksum"`
 }
 
+type WAFIPGroup struct {
+	ID       uint     `json:"id"`
+	Name     string   `json:"name"`
+	Type     string   `json:"type"`
+	Enabled  bool     `json:"enabled"`
+	IPList   []string `json:"ip_list"`
+	Checksum string   `json:"checksum"`
+}
+
+type WAFIPGroupSyncRequest struct {
+	IDs       []uint            `json:"ids,omitempty"`
+	Checksums map[string]string `json:"checksums,omitempty"`
+}
+
+type WAFIPGroupSyncResponse struct {
+	Groups []WAFIPGroup `json:"groups"`
+}
+
 type SupportFile struct {
 	Path    string `json:"path"`
 	Content string `json:"content"`

openflare_agent/internal/sync/service.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
 	openrestyrender "openflare/utils/render/openresty"
+	"sort"
 	"strings"
 
 	"openflare-agent/internal/nginx"
@@ -24,6 +26,7 @@ const (
 type ConfigClient interface {
 	GetActiveConfig(ctx context.Context) (*protocol.ActiveConfigResponse, error)
 	ReportApplyLog(ctx context.Context, payload protocol.ApplyLogPayload) error
+	SyncWAFIPGroups(ctx context.Context, payload protocol.WAFIPGroupSyncRequest) (*protocol.WAFIPGroupSyncResponse, error)
 }
 
 type NginxManager interface {
@@ -31,6 +34,8 @@ type NginxManager interface {
 	EnsureRuntime(ctx context.Context, recreate bool) error
 	EnsureSafeFallbackRuntime(ctx context.Context, reason string) error
 	CurrentChecksum() (string, error)
+	WAFIPGroupChecksums() (map[string]string, error)
+	SyncWAFIPGroups(groups []protocol.WAFIPGroup) error
 }
 
 type Service struct {
@@ -162,6 +167,20 @@ func (s *Service) ForceSyncOnce(ctx context.Context, target *protocol.ActiveConf
 	return s.applyIfNeeded(ctx, "force", true, snapshot, currentChecksum, target, config)
 }
 
+func (s *Service) WAFIPGroupChecksums() (map[string]string, error) {
+	if s.nginxManager == nil {
+		return map[string]string{}, nil
+	}
+	return s.nginxManager.WAFIPGroupChecksums()
+}
+
+func (s *Service) ApplyWAFIPGroups(ctx context.Context, groups []protocol.WAFIPGroup) error {
+	if len(groups) == 0 || s.nginxManager == nil {
+		return nil
+	}
+	return s.nginxManager.SyncWAFIPGroups(groups)
+}
+
 func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool, snapshot *state.Snapshot, currentChecksum string, target *protocol.ActiveConfigMeta, config *protocol.ActiveConfigResponse) error {
 	if currentChecksum == config.Checksum && !startup {
 		slog.Debug("local openresty config already up to date", "mode", mode, "version", config.Version)
@@ -273,10 +292,36 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
 		slog.Warn("failed apply log reported", "version", config.Version)
 		return outcomeError(config.Version, message)
 	}
+	if err := s.syncReferencedWAFIPGroups(ctx, rendered.supportFiles); err != nil {
+		slog.Error("sync referenced waf ip groups failed", "version", config.Version, "error", err)
+		return err
+	}
 	slog.Debug("apply log reported", "version", config.Version, "result", reportResult)
 	return nil
 }
 
+func (s *Service) syncReferencedWAFIPGroups(ctx context.Context, supportFiles []protocol.SupportFile) error {
+	ids := referencedWAFIPGroupIDs(supportFiles)
+	if len(ids) == 0 {
+		return nil
+	}
+	checksums, err := s.WAFIPGroupChecksums()
+	if err != nil {
+		return err
+	}
+	response, err := s.client.SyncWAFIPGroups(ctx, protocol.WAFIPGroupSyncRequest{
+		IDs:       ids,
+		Checksums: checksums,
+	})
+	if err != nil {
+		return err
+	}
+	if response == nil || len(response.Groups) == 0 {
+		return nil
+	}
+	return s.ApplyWAFIPGroups(ctx, response.Groups)
+}
+
 type renderedActiveConfig struct {
 	mainConfig   string
 	routeConfig  string
@@ -326,6 +371,48 @@ func fromOpenRestySupportFiles(files []openrestyrender.SupportFile) []protocol.S
 	return result
 }
 
+func referencedWAFIPGroupIDs(supportFiles []protocol.SupportFile) []uint {
+	var content string
+	for _, file := range supportFiles {
+		if file.Path == "waf_config.json" {
+			content = strings.TrimSpace(file.Content)
+			break
+		}
+	}
+	if content == "" {
+		return []uint{}
+	}
+	var payload struct {
+		RuleGroups []struct {
+			IPWhitelistGroups []uint `json:"ip_whitelist_group_ids"`
+			IPBlacklistGroups []uint `json:"ip_blacklist_group_ids"`
+		} `json:"rule_groups"`
+	}
+	if err := json.Unmarshal([]byte(content), &payload); err != nil {
+		slog.Debug("decode waf_config.json for ip group references failed", "error", err)
+		return []uint{}
+	}
+	seen := make(map[uint]struct{})
+	for _, group := range payload.RuleGroups {
+		for _, id := range group.IPWhitelistGroups {
+			if id > 0 {
+				seen[id] = struct{}{}
+			}
+		}
+		for _, id := range group.IPBlacklistGroups {
+			if id > 0 {
+				seen[id] = struct{}{}
+			}
+		}
+	}
+	ids := make([]uint, 0, len(seen))
+	for id := range seen {
+		ids = append(ids, id)
+	}
+	sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
+	return ids
+}
+
 func shouldReportNoopApply(snapshot *state.Snapshot, version string, checksum string) bool {
 	if snapshot == nil {
 		return false

openflare_agent/internal/sync/service_test.go

@@ -72,6 +72,10 @@ func (f *fakeClient) ReportApplyLog(ctx context.Context, payload protocol.ApplyL
 	return nil
 }
 
+func (f *fakeClient) SyncWAFIPGroups(ctx context.Context, payload protocol.WAFIPGroupSyncRequest) (*protocol.WAFIPGroupSyncResponse, error) {
+	return &protocol.WAFIPGroupSyncResponse{}, nil
+}
+
 func (m *fakeManager) Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) nginx.ApplyOutcome {
 	m.applyMainContents = append(m.applyMainContents, mainConfig)
 	m.applyRouteContents = append(m.applyRouteContents, routeConfig)
@@ -96,6 +100,14 @@ func (m *fakeManager) CurrentChecksum() (string, error) {
 	return m.currentChecksum, m.currentChecksumErr
 }
 
+func (m *fakeManager) WAFIPGroupChecksums() (map[string]string, error) {
+	return map[string]string{}, nil
+}
+
+func (m *fakeManager) SyncWAFIPGroups(groups []protocol.WAFIPGroup) error {
+	return nil
+}
+
 func TestSyncOnceSuccess(t *testing.T) {
 	client := &fakeClient{
 		config: protocol.ActiveConfigResponse{