OpenFlare/internal/repository/openflare_observability.go

// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0

package repository

import (
	"context"
	"encoding/json"
	"errors"
	"strings"
	"time"

	"gorm.io/gorm"
	"gorm.io/gorm/clause"

	db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
	"github.com/Rain-kl/Wavelet/internal/model"
	analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
	analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
	"github.com/Rain-kl/Wavelet/internal/repository/logstore"
	"github.com/Rain-kl/Wavelet/pkg/logger"
)

const (
	openFlareHealthEventStatusActive   = "active"
	openFlareHealthEventStatusResolved = "resolved"
	openFlareHealthSeverityInfo        = "info"
	openFlareHealthSeverityWarning     = "warning"
	openFlareHealthSeverityCritical    = "critical"
	openFlareHealthEventMessageMaxLen  = 4096

	// logStoreNameClickHouse 与 logstore 内部 dbNameClickHouse 取值一致。
	logStoreNameClickHouse = "clickhouse"
)

// OpenFlareHealthEventInput describes a desired active health event for reconciliation.
type OpenFlareHealthEventInput struct {
	EventType       string
	Severity        string
	Message         string
	TriggeredAtUnix int64
	Metadata        map[string]string
}

func isMissingTableError(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, gorm.ErrRecordNotFound) {
		return false
	}
	msg := strings.ToLower(err.Error())
	return strings.Contains(msg, "no such table") ||
		strings.Contains(msg, "doesn't exist") ||
		strings.Contains(msg, "does not exist")
}

// InsertOpenFlareMetricSnapshot inserts a metric snapshot into ClickHouse.
func InsertOpenFlareMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
	s, err := logstore.Active(ctx)
	if err != nil {
		return err
	}
	return s.Observability.InsertMetricSnapshot(ctx, record)
}

// InsertOpenFlareEdgeHealth inserts an L2 edge health snapshot into ClickHouse.
func InsertOpenFlareEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
	s, err := logstore.Active(ctx)
	if err != nil {
		return err
	}
	return s.Observability.InsertEdgeHealth(ctx, record)
}

// InsertOpenFlareNodeObservationFrps inserts an FRPS observation into ClickHouse.
func InsertOpenFlareNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
	s, err := logstore.Active(ctx)
	if err != nil {
		return err
	}
	return s.Observability.InsertNodeObservationFrps(ctx, record)
}

// InsertOpenFlareNodeObservationFrpc inserts an FRPC observation into ClickHouse.
func InsertOpenFlareNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
	s, err := logstore.Active(ctx)
	if err != nil {
		return err
	}
	return s.Observability.InsertNodeObservationFrpc(ctx, record)
}

// ListOpenFlareMetricSnapshotsSince returns metric snapshots since the given time.
func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return nil, err
	}
	return s.Observability.ListMetricSnapshots(ctx, nodeID, since, limit)
}

// ListOpenFlareLatestMetricSnapshotsSince returns the latest metric snapshot per node.
// The ClickHouse LIMIT 1 BY fast path is used only when ClickHouse is the ACTIVE log
// database; otherwise the request goes straight to the active log store (PG/SQLite),
// avoiding stale reads of the previous CH store after a migration.
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
	active, err := logstore.ActiveDatabase(ctx)
	if err != nil {
		logger.ErrorF(ctx, "failed to resolve active log database for latest metric snapshots: %v", err)
	} else if active == logStoreNameClickHouse {
		rows, chErr := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
			NodeID: nodeID,
			Since:  since,
		})
		if chErr == nil {
			return fromAnalyticsNodeMetricSnapshots(rows), nil
		}
		logger.ErrorF(ctx, "clickhouse fast-path ListLatestNodeMetricSnapshots failed: %v", chErr)
		return nil, chErr
	}
	// Routes through the active log store (PG/SQLite active).
	all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
	if listErr != nil {
		return nil, listErr
	}
	return openFlareLatestMetricSnapshots(all), nil
}

func openFlareLatestMetricSnapshots(snapshots []*model.OpenFlareMetricSnapshot) []*model.OpenFlareMetricSnapshot {
	latestByNode := make(map[string]*model.OpenFlareMetricSnapshot, len(snapshots))
	for _, snapshot := range snapshots {
		if snapshot == nil || snapshot.NodeID == "" {
			continue
		}
		if existing, ok := latestByNode[snapshot.NodeID]; ok && !snapshot.CapturedAt.After(existing.CapturedAt) {
			continue
		}
		latestByNode[snapshot.NodeID] = snapshot
	}
	result := make([]*model.OpenFlareMetricSnapshot, 0, len(latestByNode))
	for _, snapshot := range latestByNode {
		result = append(result, snapshot)
	}
	return result
}

// fromAnalyticsNodeMetricSnapshots converts analytics rows back to the business model.
func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot) []*model.OpenFlareMetricSnapshot {
	result := make([]*model.OpenFlareMetricSnapshot, len(rows))
	for index, row := range rows {
		result[index] = &model.OpenFlareMetricSnapshot{
			ID:                uint(row.ID),
			NodeID:            row.NodeID,
			CapturedAt:        row.CapturedAt,
			CPUUsagePercent:   row.CPUUsagePercent,
			MemoryUsedBytes:   row.MemoryUsedBytes,
			MemoryTotalBytes:  row.MemoryTotalBytes,
			StorageUsedBytes:  row.StorageUsedBytes,
			StorageTotalBytes: row.StorageTotalBytes,
			DiskReadBytes:     row.DiskReadBytes,
			DiskWriteBytes:    row.DiskWriteBytes,
			NetworkRxBytes:    row.NetworkRxBytes,
			NetworkTxBytes:    row.NetworkTxBytes,
			CreatedAt:         row.CreatedAt,
		}
	}
	return result
}

// ListOpenFlareTrafficHourlySince returns hourly traffic rollup rows since the given time.
// CH 读 of_access_log_hourly rollup；PG/SQLite 经 logstore 从 of_node_access_logs 实时聚合。
func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareTrafficHourly, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return nil, err
	}
	rows, err := s.Observability.ListTrafficHourly(ctx, nodeID, since)
	if err != nil {
		return nil, err
	}
	result := make([]*model.OpenFlareTrafficHourly, len(rows))
	for index, row := range rows {
		result[index] = &model.OpenFlareTrafficHourly{
			NodeID:             row.NodeID,
			Hour:               row.Hour,
			RequestCount:       row.RequestCount,
			ErrorCount:         row.ErrorCount,
			UniqueVisitorCount: row.UniqueVisitorCount,
		}
	}
	return result, nil
}

// ListOpenFlareAccessLogHourlySince returns hourly access-log rollups since the given time,
// read through logstore's active backend (ClickHouse of_access_log_hourly rollup;
// PostgreSQL/SQLite real-time aggregation from of_node_access_logs).
func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareAccessLogHourly, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return nil, err
	}
	rows, err := s.Observability.ListAccessLogHourly(ctx, nodeID, since)
	if err != nil {
		return nil, err
	}
	result := make([]*model.OpenFlareAccessLogHourly, len(rows))
	for index, row := range rows {
		result[index] = &model.OpenFlareAccessLogHourly{
			NodeID:        row.NodeID,
			Hour:          row.Hour,
			Host:          row.Host,
			RequestCount:  row.RequestCount,
			ErrorCount:    row.ErrorCount,
			BytesSent:     row.BytesSent,
			RequestLength: row.RequestLength,
		}
	}
	return result, nil
}

// ListOpenFlareMetricHourlySince returns hourly metric aggregates since the given time.
func ListOpenFlareMetricHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricHourly, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return nil, err
	}
	rows, err := s.Observability.ListMetricHourly(ctx, nodeID, since)
	if err != nil {
		return nil, err
	}
	result := make([]*model.OpenFlareMetricHourly, len(rows))
	for index, row := range rows {
		result[index] = &model.OpenFlareMetricHourly{
			Hour:                      row.Hour,
			AverageCPUUsagePercent:    row.AverageCPUUsagePercent,
			AverageMemoryUsagePercent: row.AverageMemoryUsagePercent,
			NetworkRxBytes:            row.NetworkRxBytes,
			NetworkTxBytes:            row.NetworkTxBytes,
			DiskReadBytes:             row.DiskReadBytes,
			DiskWriteBytes:            row.DiskWriteBytes,
			ReportedNodes:             row.ReportedNodes,
		}
	}
	return result, nil
}

// ListOpenFlareActiveHealthEvents returns active health events across all nodes.
func ListOpenFlareActiveHealthEvents(ctx context.Context) ([]*model.OpenFlareHealthEvent, error) {
	conn := db.DB(ctx)
	if conn == nil {
		return nil, errors.New(errDatabaseNotInitialized)
	}
	var rows []*model.OpenFlareHealthEvent
	if err := conn.Where("status = ?", "active").Order("last_triggered_at desc").Find(&rows).Error; err != nil {
		if isMissingTableError(err) {
			return []*model.OpenFlareHealthEvent{}, nil
		}
		return nil, err
	}
	return rows, nil
}

// ListOpenFlareHealthEvents returns health events for a node.
func ListOpenFlareHealthEvents(ctx context.Context, nodeID string, activeOnly bool, limit int) ([]*model.OpenFlareHealthEvent, error) {
	conn := db.DB(ctx)
	if conn == nil {
		return nil, errors.New(errDatabaseNotInitialized)
	}
	query := conn.Model(&model.OpenFlareHealthEvent{}).Where("node_id = ?", nodeID).Order("last_triggered_at desc")
	if activeOnly {
		query = query.Where("status = ?", "active")
	}
	if limit > 0 {
		query = query.Limit(limit)
	}
	var rows []*model.OpenFlareHealthEvent
	if err := query.Find(&rows).Error; err != nil {
		if isMissingTableError(err) {
			return []*model.OpenFlareHealthEvent{}, nil
		}
		return nil, err
	}
	return rows, nil
}

// DeleteOpenFlareMetricSnapshotsBefore deletes metric snapshots captured before cutoff.
func DeleteOpenFlareMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteMetricSnapshotsBefore(ctx, cutoff)
}

// DeleteAllOpenFlareMetricSnapshots deletes all metric snapshots.
func DeleteAllOpenFlareMetricSnapshots(ctx context.Context) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteAllMetricSnapshots(ctx)
}

// DeleteOpenFlareEdgeHealthBefore deletes edge health rows captured before cutoff.
func DeleteOpenFlareEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
}

// DeleteAllOpenFlareEdgeHealth deletes all edge health snapshots.
func DeleteAllOpenFlareEdgeHealth(ctx context.Context) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteAllEdgeHealth(ctx)
}

// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
func DeleteOpenFlareNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
}

// DeleteAllOpenFlareNodeObservationFrps deletes all FRPS observations.
func DeleteAllOpenFlareNodeObservationFrps(ctx context.Context) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteAllNodeObservationFrps(ctx)
}

// DeleteOpenFlareNodeObservationFrpcBefore deletes FRPC observations captured before cutoff.
func DeleteOpenFlareNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
}

// DeleteAllOpenFlareNodeObservationFrpc deletes all FRPC observations.
func DeleteAllOpenFlareNodeObservationFrpc(ctx context.Context) (int64, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return 0, err
	}
	return s.Observability.DeleteAllNodeObservationFrpc(ctx)
}

// DeleteOpenFlareHealthEventsByNodeID deletes all health events for a node.
func DeleteOpenFlareHealthEventsByNodeID(ctx context.Context, nodeID string) (int64, error) {
	conn := db.DB(ctx)
	if conn == nil {
		return 0, errors.New(errDatabaseNotInitialized)
	}
	result := conn.Where("node_id = ?", nodeID).Delete(&model.OpenFlareHealthEvent{})
	if result.Error != nil {
		if isMissingTableError(result.Error) {
			return 0, nil
		}
		return 0, result.Error
	}
	return result.RowsAffected, nil
}

// GetOpenFlareNodeSystemProfile returns the system profile for a node.
func GetOpenFlareNodeSystemProfile(ctx context.Context, nodeID string) (*model.OpenFlareNodeSystemProfile, error) {
	conn := db.DB(ctx)
	if conn == nil {
		return nil, errors.New(errDatabaseNotInitialized)
	}
	var profile model.OpenFlareNodeSystemProfile
	if err := conn.Where("node_id = ?", nodeID).First(&profile).Error; err != nil {
		if errors.Is(err, gorm.ErrRecordNotFound) || isMissingTableError(err) {
			return nil, gorm.ErrRecordNotFound
		}
		return nil, err
	}
	return &profile, nil
}

// UpsertOpenFlareNodeSystemProfile inserts or updates the latest system profile for a node.
func UpsertOpenFlareNodeSystemProfile(ctx context.Context, record *model.OpenFlareNodeSystemProfile) error {
	if record == nil {
		return nil
	}
	conn := db.DB(ctx)
	if conn == nil {
		return errors.New(errDatabaseNotInitialized)
	}
	return upsertOpenFlareNodeSystemProfileTx(conn, record)
}

func upsertOpenFlareNodeSystemProfileTx(tx *gorm.DB, record *model.OpenFlareNodeSystemProfile) error {
	return tx.Clauses(clause.OnConflict{
		Columns: []clause.Column{{Name: "node_id"}},
		DoUpdates: clause.AssignmentColumns([]string{
			"hostname",
			"os_name",
			"os_version",
			"kernel_version",
			"architecture",
			"cpu_model",
			"cpu_cores",
			"total_memory_bytes",
			"total_disk_bytes",
			"uptime_seconds",
			"reported_at",
			"updated_at",
		}),
	}).Create(record).Error
}

// ReconcileOpenFlareHealthEvents reconciles active health events for a node.
// Desired active events are created or updated; previously active types not present are resolved.
// When managedEventTypes is non-empty, only those event types are considered.
// Runs inside a transaction so multi-row create/update/resolve stays atomic.
func ReconcileOpenFlareHealthEvents(
	ctx context.Context,
	nodeID string,
	events []OpenFlareHealthEventInput,
	reportedAt time.Time,
	managedEventTypes map[string]struct{},
) error {
	conn := db.DB(ctx)
	if conn == nil {
		return errors.New(errDatabaseNotInitialized)
	}
	return conn.Transaction(func(tx *gorm.DB) error {
		return reconcileOpenFlareHealthEventsTx(tx, nodeID, events, reportedAt, managedEventTypes)
	})
}

// PersistOpenFlareNodePGObservability upserts an optional system profile and optionally reconciles
// health events in a single transaction (Postgres-side heartbeat observability).
// When reconcileHealth is false, health events are left untouched.
func PersistOpenFlareNodePGObservability(
	ctx context.Context,
	profile *model.OpenFlareNodeSystemProfile,
	nodeID string,
	events []OpenFlareHealthEventInput,
	reconcileHealth bool,
	reportedAt time.Time,
	managedEventTypes map[string]struct{},
) error {
	if profile == nil && !reconcileHealth {
		return nil
	}
	conn := db.DB(ctx)
	if conn == nil {
		return errors.New(errDatabaseNotInitialized)
	}
	return conn.Transaction(func(tx *gorm.DB) error {
		if profile != nil {
			if err := upsertOpenFlareNodeSystemProfileTx(tx, profile); err != nil {
				return err
			}
		}
		if reconcileHealth {
			if err := reconcileOpenFlareHealthEventsTx(tx, nodeID, events, reportedAt, managedEventTypes); err != nil {
				return err
			}
		}
		return nil
	})
}

func reconcileOpenFlareHealthEventsTx(
	tx *gorm.DB,
	nodeID string,
	events []OpenFlareHealthEventInput,
	reportedAt time.Time,
	managedEventTypes map[string]struct{},
) error {
	activeTypes := make(map[string]OpenFlareHealthEventInput, len(events))
	for _, event := range events {
		eventType := normalizeOpenFlareHealthEventType(event.EventType)
		if eventType == "" {
			continue
		}
		if len(managedEventTypes) > 0 {
			if _, ok := managedEventTypes[eventType]; !ok {
				continue
			}
		}
		event.EventType = eventType
		event.Severity = normalizeOpenFlareHealthSeverity(event.Severity)
		if event.TriggeredAtUnix <= 0 {
			event.TriggeredAtUnix = reportedAt.Unix()
		}
		activeTypes[eventType] = event
	}

	var activeEvents []*model.OpenFlareHealthEvent
	query := tx.Where("node_id = ? AND status = ?", nodeID, openFlareHealthEventStatusActive)
	if len(managedEventTypes) > 0 {
		scopedTypes := make([]string, 0, len(managedEventTypes))
		for eventType := range managedEventTypes {
			eventType = normalizeOpenFlareHealthEventType(eventType)
			if eventType != "" {
				scopedTypes = append(scopedTypes, eventType)
			}
		}
		if len(scopedTypes) == 0 {
			return nil
		}
		query = query.Where("event_type IN ?", scopedTypes)
	}
	if err := query.Find(&activeEvents).Error; err != nil {
		return err
	}

	activeByType := make(map[string]*model.OpenFlareHealthEvent, len(activeEvents))
	for _, event := range activeEvents {
		activeByType[event.EventType] = event
	}

	for eventType, event := range activeTypes {
		triggeredAt := timeFromUnixSeconds(event.TriggeredAtUnix, reportedAt)
		if existing, ok := activeByType[eventType]; ok {
			existing.Severity = event.Severity
			existing.Message = normalizeOpenFlareHealthEventMessage(event.Message)
			existing.LastTriggeredAt = triggeredAt
			existing.ReportedAt = reportedAt
			existing.MetadataJSON = marshalOpenFlareHealthMetadata(event.Metadata)
			existing.ResolvedAt = nil
			if err := tx.Save(existing).Error; err != nil {
				return err
			}
			continue
		}
		record := &model.OpenFlareHealthEvent{
			NodeID:           nodeID,
			EventType:        eventType,
			Severity:         event.Severity,
			Status:           openFlareHealthEventStatusActive,
			Message:          normalizeOpenFlareHealthEventMessage(event.Message),
			FirstTriggeredAt: triggeredAt,
			LastTriggeredAt:  triggeredAt,
			ReportedAt:       reportedAt,
			MetadataJSON:     marshalOpenFlareHealthMetadata(event.Metadata),
		}
		if err := tx.Create(record).Error; err != nil {
			return err
		}
	}

	for _, existing := range activeEvents {
		if _, ok := activeTypes[existing.EventType]; ok {
			continue
		}
		resolvedAt := reportedAt
		existing.Status = openFlareHealthEventStatusResolved
		existing.ReportedAt = reportedAt
		existing.ResolvedAt = &resolvedAt
		if err := tx.Save(existing).Error; err != nil {
			return err
		}
	}

	return nil
}

func normalizeOpenFlareHealthEventType(eventType string) string {
	eventType = strings.TrimSpace(strings.ToLower(eventType))
	eventType = strings.ReplaceAll(eventType, " ", "_")
	return eventType
}

func normalizeOpenFlareHealthSeverity(severity string) string {
	switch strings.ToLower(strings.TrimSpace(severity)) {
	case openFlareHealthSeverityCritical:
		return openFlareHealthSeverityCritical
	case openFlareHealthSeverityInfo:
		return openFlareHealthSeverityInfo
	default:
		return openFlareHealthSeverityWarning
	}
}

func normalizeOpenFlareHealthEventMessage(message string) string {
	if openFlareHealthEventMessageMaxLen <= 0 {
		return ""
	}
	runes := []rune(strings.TrimSpace(message))
	if len(runes) <= openFlareHealthEventMessageMaxLen {
		return string(runes)
	}
	return string(runes[:openFlareHealthEventMessageMaxLen])
}

func timeFromUnixSeconds(unixSeconds int64, fallback time.Time) time.Time {
	if unixSeconds <= 0 {
		return fallback
	}
	return time.Unix(unixSeconds, 0).UTC()
}

func marshalOpenFlareHealthMetadata(value map[string]string) string {
	if value == nil {
		return ""
	}
	raw, err := json.Marshal(value)
	if err != nil {
		return ""
	}
	return string(raw)
}

// ListOpenFlareEdgeHealth returns L2 edge health snapshots.
func ListOpenFlareEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return nil, err
	}
	return s.Observability.ListEdgeHealth(ctx, nodeID, since, limit)
}

// ListOpenFlareNodeObservationFrpc returns frpc observations.
func ListOpenFlareNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return nil, err
	}
	return s.Observability.ListNodeObservationFrpc(ctx, nodeID, since, limit)
}

// ListOpenFlareNodeObservationFrps returns frps observations.
func ListOpenFlareNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
	s, err := logstore.Active(ctx)
	if err != nil {
		return nil, err
	}
	return s.Observability.ListNodeObservationFrps(ctx, nodeID, since, limit)
}