refactor(layout): consolidate backend codebase into backend/ package and clean root directory

- Moved cmd/, core/, plugins/, pkg/, downstream/, and main.go into backend/ directory - Batch updated all Go source files to import github.com/Rain-kl/Wavelet/backend/... - Updated Makefile, scripts/swagger.sh, architecture guards, and platform skills - Passed all quality gates (100% tests, 0 lint issues, clean build)
2026-10-04 07:06:36 +08:00 · 2026-08-28 12:56:02 +08:00
parent 33b38f8687
commit 43dc97e48c
319 changed files with 912 additions and 1031 deletions
@@ -0,0 +1,69 @@
+// Copyright 2026 Arctel.net
+// SPDX-License-Identifier: Apache-2.0
+
+package batchwriter
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	defaultQueueSize    = 10_000
+	defaultMaxBatchSize = 1_000
+	defaultMinBatchSize = 50
+	defaultFlushEvery   = time.Second
+)
+
+// Config controls queue capacity and flush thresholds for a Writer instance.
+type Config struct {
+	// Name identifies the writer in logs and diagnostics. Optional.
+	Name string
+
+	// QueueSize is the buffered channel capacity.
+	QueueSize int
+
+	// MaxBatchSize triggers a flush when the in-memory batch reaches this count.
+	MaxBatchSize int
+
+	// MinBatchSize is the minimum in-memory batch size for time-based flushes.
+	// Zero disables the threshold and preserves legacy interval flush behavior.
+	// When set, interval flushes below this size are skipped unless MaxFlushWait elapses.
+	MinBatchSize int
+
+	// FlushInterval is how often the worker checks whether a time-based flush should run.
+	FlushInterval time.Duration
+
+	// MaxFlushWait forces a flush of any non-empty batch once the oldest item has waited
+	// this long, even if MinBatchSize has not been reached. Zero disables the force path.
+	MaxFlushWait time.Duration
+}
+
+// DefaultConfig returns production-friendly defaults aligned with audit log batching.
+func DefaultConfig() Config {
+	return Config{
+		QueueSize:     defaultQueueSize,
+		MaxBatchSize:  defaultMaxBatchSize,
+		MinBatchSize:  defaultMinBatchSize,
+		FlushInterval: defaultFlushEvery,
+	}
+}
+
+func (c Config) validate() error {
+	if c.QueueSize <= 0 {
+		return fmt.Errorf("batchwriter: queue size must be positive")
+	}
+	if c.MaxBatchSize <= 0 {
+		return fmt.Errorf("batchwriter: max batch size must be positive")
+	}
+	if c.MinBatchSize < 0 {
+		return fmt.Errorf("batchwriter: min batch size must be non-negative")
+	}
+	if c.FlushInterval <= 0 {
+		return fmt.Errorf("batchwriter: flush interval must be positive")
+	}
+	if c.MaxFlushWait < 0 {
+		return fmt.Errorf("batchwriter: max flush wait must be non-negative")
+	}
+	return nil
+}
@@ -0,0 +1,8 @@
+// Copyright 2026 Arctel.net
+// SPDX-License-Identifier: Apache-2.0
+
+package batchwriter
+
+import "errors"
+
+var errNilFlushFunc = errors.New("batchwriter: flush func is required")
@@ -0,0 +1,264 @@
+// Copyright 2026 Arctel.net
+// SPDX-License-Identifier: Apache-2.0
+
+// Package batchwriter provides a reusable buffered batch writer for high-throughput
+// append-only sinks such as ClickHouse. Each business domain should own an independent
+// Writer instance with its own queue, flush callback, and tuning parameters.
+package batchwriter
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// FlushFunc persists a batch of queued items. It is invoked from the worker goroutine.
+type FlushFunc[T any] func(ctx context.Context, items []T) error
+
+// FlushErrorHandler is called when FlushFunc returns an error after optional retries.
+// The batch is discarded after the handler returns; the worker continues processing.
+// Handlers receive the failed items so callers can release dedup keys or re-queue.
+type FlushErrorHandler[T any] func(ctx context.Context, items []T, err error)
+
+// Stats is a point-in-time snapshot of Writer queue and failure counters.
+type Stats struct {
+	Name        string
+	Depth       int
+	Cap         int
+	Drops       int64
+	FlushErrors int64
+	Running     bool
+}
+
+// Writer buffers items and flushes them by size or interval.
+type Writer[T any] struct {
+	cfg   Config
+	flush FlushFunc[T]
+
+	onFlushError FlushErrorHandler[T]
+	onDrop       func(T)
+
+	startOnce sync.Once
+	stopOnce  sync.Once
+
+	mu        sync.RWMutex
+	ch        chan T
+	workerCtx context.Context
+	done      chan struct{}
+
+	drops       atomic.Int64
+	flushErrors atomic.Int64
+}
+
+// Option configures optional Writer callbacks.
+type Option[T any] func(*Writer[T])
+
+// WithFlushErrorHandler registers a callback for flush failures.
+func WithFlushErrorHandler[T any](handler FlushErrorHandler[T]) Option[T] {
+	return func(w *Writer[T]) {
+		w.onFlushError = handler
+	}
+}
+
+// WithDropHandler registers a callback when TryEnqueue cannot accept an item.
+func WithDropHandler[T any](handler func(T)) Option[T] {
+	return func(w *Writer[T]) {
+		w.onDrop = handler
+	}
+}
+
+// New creates a Writer. Call Start before enqueueing items.
+func New[T any](cfg Config, flush FlushFunc[T], opts ...Option[T]) (*Writer[T], error) {
+	if flush == nil {
+		return nil, errNilFlushFunc
+	}
+	if err := cfg.validate(); err != nil {
+		return nil, err
+	}
+
+	w := &Writer[T]{
+		cfg:   cfg,
+		flush: flush,
+		done:  make(chan struct{}),
+	}
+	for _, opt := range opts {
+		opt(w)
+	}
+	return w, nil
+}
+
+// Start launches the background worker. It is safe to call at most once.
+func (w *Writer[T]) Start(parent context.Context) {
+	w.startOnce.Do(func() {
+		w.mu.Lock()
+		defer w.mu.Unlock()
+
+		w.ch = make(chan T, w.cfg.QueueSize)
+		w.workerCtx = context.WithoutCancel(parent)
+		go w.run()
+	})
+}
+
+// Stop closes the queue and waits until the worker drains pending items and exits.
+func (w *Writer[T]) Stop(ctx context.Context) error {
+	w.mu.RLock()
+	ch := w.ch
+	done := w.done
+	w.mu.RUnlock()
+
+	if ch == nil {
+		return nil
+	}
+
+	w.stopOnce.Do(func() {
+		close(ch)
+	})
+
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// Running reports whether Start has been called and Stop has not completed.
+func (w *Writer[T]) Running() bool {
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	if w.ch == nil {
+		return false
+	}
+	select {
+	case <-w.done:
+		return false
+	default:
+		return true
+	}
+}
+
+// TryEnqueue adds one item without blocking. It returns false when the writer is not
+// running or the queue is full.
+func (w *Writer[T]) TryEnqueue(item T) bool {
+	w.mu.RLock()
+	ch := w.ch
+	w.mu.RUnlock()
+	if ch == nil {
+		w.notifyDrop(item)
+		return false
+	}
+
+	select {
+	case ch <- item:
+		return true
+	default:
+		w.notifyDrop(item)
+		return false
+	}
+}
+
+// IsFull reports whether the queue has no remaining capacity.
+func (w *Writer[T]) IsFull() bool {
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	if w.ch == nil {
+		return false
+	}
+	return len(w.ch) >= cap(w.ch)
+}
+
+// Len returns the current queue depth.
+func (w *Writer[T]) Len() int {
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	if w.ch == nil {
+		return 0
+	}
+	return len(w.ch)
+}
+
+// Cap returns the queue capacity.
+func (w *Writer[T]) Cap() int {
+	return w.cfg.QueueSize
+}
+
+// Stats returns a point-in-time snapshot of queue depth and failure counters.
+func (w *Writer[T]) Stats() Stats {
+	return Stats{
+		Name:        w.cfg.Name,
+		Depth:       w.Len(),
+		Cap:         w.Cap(),
+		Drops:       w.drops.Load(),
+		FlushErrors: w.flushErrors.Load(),
+		Running:     w.Running(),
+	}
+}
+
+func (w *Writer[T]) run() {
+	ticker := time.NewTicker(w.cfg.FlushInterval)
+	defer ticker.Stop()
+
+	batch := make([]T, 0, w.cfg.MaxBatchSize)
+	var batchStartedAt time.Time
+	flush := func() {
+		if len(batch) == 0 {
+			return
+		}
+		items := append([]T(nil), batch...)
+		if err := w.flush(w.workerCtx, items); err != nil {
+			w.flushErrors.Add(1)
+			if w.onFlushError != nil {
+				w.onFlushError(w.workerCtx, items, err)
+			}
+		}
+		batch = batch[:0]
+		batchStartedAt = time.Time{}
+	}
+
+	defer func() {
+		flush()
+		close(w.done)
+	}()
+
+	for {
+		select {
+		case item, ok := <-w.ch:
+			if !ok {
+				return
+			}
+			if len(batch) == 0 {
+				batchStartedAt = time.Now()
+			}
+			batch = append(batch, item)
+			if len(batch) >= w.cfg.MaxBatchSize {
+				flush()
+			}
+		case <-ticker.C:
+			if w.shouldFlushOnInterval(len(batch), batchStartedAt, time.Now()) {
+				flush()
+			}
+		}
+	}
+}
+
+func (w *Writer[T]) shouldFlushOnInterval(batchLen int, batchStartedAt time.Time, now time.Time) bool {
+	if batchLen == 0 {
+		return false
+	}
+	if w.cfg.MinBatchSize == 0 || batchLen >= w.cfg.MinBatchSize {
+		return true
+	}
+	if w.cfg.MaxFlushWait <= 0 || batchStartedAt.IsZero() {
+		return false
+	}
+	return !now.Before(batchStartedAt.Add(w.cfg.MaxFlushWait))
+}
+
+func (w *Writer[T]) notifyDrop(item T) {
+	w.drops.Add(1)
+	if w.onDrop == nil {
+		return
+	}
+	w.onDrop(item)
+}
@@ -0,0 +1,317 @@
+// Copyright 2026 Arctel.net
+// SPDX-License-Identifier: Apache-2.0
+
+package batchwriter
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type testEvent struct {
+	ID   int
+	Data string
+}
+
+func testConfig() Config {
+	return Config{
+		Name:          "test-writer",
+		QueueSize:     100,
+		MaxBatchSize:  5,
+		FlushInterval: 20 * time.Millisecond,
+	}
+}
+
+func TestWriter_BatchSizeFlush(t *testing.T) {
+	var (
+		mu      sync.Mutex
+		batches [][]testEvent
+		flushWg sync.WaitGroup
+	)
+	flushWg.Add(1)
+
+	cfg := testConfig()
+	cfg.FlushInterval = time.Hour
+
+	w, err := New(cfg, func(_ context.Context, items []testEvent) error {
+		mu.Lock()
+		defer mu.Unlock()
+		batches = append(batches, items)
+		if len(items) == 5 {
+			flushWg.Done()
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	w.Start(context.Background())
+	defer func() { _ = w.Stop(context.Background()) }()
+
+	for i := 1; i <= 5; i++ {
+		ok := w.TryEnqueue(testEvent{ID: i, Data: "payload"})
+		assert.True(t, ok)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		flushWg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for batch flush")
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Len(t, batches, 1)
+	assert.Len(t, batches[0], 5)
+	for i, item := range batches[0] {
+		assert.Equal(t, i+1, item.ID)
+	}
+}
+
+func TestWriter_IntervalFlush(t *testing.T) {
+	var (
+		mu      sync.Mutex
+		flushed []testEvent
+		done    = make(chan struct{})
+	)
+
+	cfg := testConfig()
+	cfg.MaxBatchSize = 100
+	cfg.FlushInterval = 30 * time.Millisecond
+
+	w, err := New(cfg, func(_ context.Context, items []testEvent) error {
+		mu.Lock()
+		defer mu.Unlock()
+		flushed = append(flushed, items...)
+		if len(flushed) == 2 {
+			select {
+			case <-done:
+			default:
+				close(done)
+			}
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	w.Start(context.Background())
+	defer func() { _ = w.Stop(context.Background()) }()
+
+	assert.True(t, w.TryEnqueue(testEvent{ID: 1}))
+	assert.True(t, w.TryEnqueue(testEvent{ID: 2}))
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for interval flush")
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Len(t, flushed, 2)
+}
+
+func TestWriter_MinBatchSizeThreshold(t *testing.T) {
+	var (
+		mu      sync.Mutex
+		flushed []testEvent
+		done    = make(chan struct{})
+	)
+
+	cfg := testConfig()
+	cfg.MaxBatchSize = 100
+	cfg.MinBatchSize = 3
+	cfg.FlushInterval = 20 * time.Millisecond
+	cfg.MaxFlushWait = 60 * time.Millisecond
+
+	w, err := New(cfg, func(_ context.Context, items []testEvent) error {
+		mu.Lock()
+		defer mu.Unlock()
+		flushed = append(flushed, items...)
+		if len(flushed) == 2 {
+			select {
+			case <-done:
+			default:
+				close(done)
+			}
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	w.Start(context.Background())
+	defer func() { _ = w.Stop(context.Background()) }()
+
+	assert.True(t, w.TryEnqueue(testEvent{ID: 1}))
+	assert.True(t, w.TryEnqueue(testEvent{ID: 2}))
+
+	time.Sleep(30 * time.Millisecond)
+	mu.Lock()
+	assert.Empty(t, flushed, "items should wait until MinBatchSize or MaxFlushWait")
+	mu.Unlock()
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for forced max wait flush")
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Len(t, flushed, 2)
+}
+
+func TestWriter_StopDrainsRemaining(t *testing.T) {
+	var (
+		mu      sync.Mutex
+		flushed []testEvent
+	)
+
+	cfg := testConfig()
+	cfg.FlushInterval = time.Hour
+
+	w, err := New(cfg, func(_ context.Context, items []testEvent) error {
+		mu.Lock()
+		defer mu.Unlock()
+		flushed = append(flushed, items...)
+		return nil
+	})
+	require.NoError(t, err)
+
+	w.Start(context.Background())
+	for i := 1; i <= 3; i++ {
+		assert.True(t, w.TryEnqueue(testEvent{ID: i}))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	require.NoError(t, w.Stop(ctx))
+
+	assert.False(t, w.Running())
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Len(t, flushed, 3)
+}
+
+func TestWriter_DropWhenFull(t *testing.T) {
+	var (
+		dropped atomic.Int64
+		blockCh = make(chan struct{})
+	)
+
+	cfg := Config{
+		QueueSize:     2,
+		MaxBatchSize:  1,
+		FlushInterval: time.Hour,
+	}
+
+	entered := make(chan struct{})
+	w, err := New(cfg, func(_ context.Context, _ []testEvent) error {
+		select {
+		case entered <- struct{}{}:
+		default:
+		}
+		<-blockCh
+		return nil
+	}, WithDropHandler(func(_ testEvent) {
+		dropped.Add(1)
+	}))
+	require.NoError(t, err)
+
+	w.Start(context.Background())
+	defer func() {
+		close(blockCh)
+		_ = w.Stop(context.Background())
+	}()
+
+	// 1. 推入 1 个 item 触发 flush 并阻塞在 blockCh
+	w.ch <- testEvent{ID: 1}
+	<-entered
+
+	// 2. 此时 worker 阻塞，填满 channel
+	w.ch <- testEvent{ID: 2}
+	w.ch <- testEvent{ID: 3}
+
+	assert.False(t, w.TryEnqueue(testEvent{ID: 4}))
+	assert.Equal(t, int64(1), dropped.Load())
+	assert.Equal(t, int64(1), w.Stats().Drops)
+}
+
+func TestWriter_FlushErrorCallback(t *testing.T) {
+	var (
+		called   atomic.Bool
+		flushErr = errors.New("clickhouse write timeout")
+		done     = make(chan struct{})
+	)
+
+	cfg := testConfig()
+	cfg.MaxBatchSize = 1
+	cfg.FlushInterval = time.Hour
+
+	w, err := New(cfg, func(_ context.Context, _ []testEvent) error {
+		return flushErr
+	}, WithFlushErrorHandler(func(_ context.Context, items []testEvent, err error) {
+		called.Store(true)
+		assert.Equal(t, flushErr, err)
+		assert.Len(t, items, 1)
+		close(done)
+	}))
+	require.NoError(t, err)
+
+	w.Start(context.Background())
+	defer func() { _ = w.Stop(context.Background()) }()
+
+	assert.True(t, w.TryEnqueue(testEvent{ID: 1}))
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for error callback")
+	}
+
+	assert.True(t, called.Load())
+	assert.Equal(t, int64(1), w.Stats().FlushErrors)
+}
+
+func TestWriter_ValidateConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     Config
+		wantErr bool
+	}{
+		{"valid", DefaultConfig(), false},
+		{"zero queue", Config{QueueSize: 0, MaxBatchSize: 10, FlushInterval: time.Second}, true},
+		{"zero max batch", Config{QueueSize: 10, MaxBatchSize: 0, FlushInterval: time.Second}, true},
+		{"negative min batch", Config{QueueSize: 10, MaxBatchSize: 10, MinBatchSize: -1, FlushInterval: time.Second}, true},
+		{"zero flush interval", Config{QueueSize: 10, MaxBatchSize: 10, FlushInterval: 0}, true},
+		{"negative max flush wait", Config{QueueSize: 10, MaxBatchSize: 10, FlushInterval: time.Second, MaxFlushWait: -1}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := New(tt.cfg, func(_ context.Context, _ []testEvent) error { return nil })
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestWriter_NilFlushFunc(t *testing.T) {
+	_, err := New[testEvent](DefaultConfig(), nil)
+	assert.ErrorIs(t, err, errNilFlushFunc)
+}