mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 07:26:36 +08:00
refactor(layout): consolidate backend codebase into backend/ package and clean root directory
- Moved cmd/, core/, plugins/, pkg/, downstream/, and main.go into backend/ directory - Batch updated all Go source files to import github.com/Rain-kl/Wavelet/backend/... - Updated Makefile, scripts/swagger.sh, architecture guards, and platform skills - Passed all quality gates (100% tests, 0 lint issues, clean build)
This commit is contained in:
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package cache
|
||||
|
||||
const (
|
||||
errRedisHashSetFailed = "failed to set redis hash: %w"
|
||||
errRedisHashDeleteFailed = "failed to delete redis hash field: %w"
|
||||
errUnmarshalDataFailed = "failed to unmarshal data: %w"
|
||||
errMarshalDataFailed = "failed to marshal data: %w"
|
||||
errRedisKeySetFailed = "failed to set redis key: %w"
|
||||
)
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package cache provides the multi-tier caching infrastructure plugin for Cordis.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/cache/ram"
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/util"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultRAMCapacity = 10000
|
||||
defaultPubSubChannel = "wavelet:cache:invalidation"
|
||||
)
|
||||
|
||||
type ramEntry struct {
|
||||
data []byte
|
||||
expireAt time.Time
|
||||
}
|
||||
|
||||
// Option configures the cache plugin.
|
||||
type Option func(*Plugin)
|
||||
|
||||
// WithRedis sets an explicit Redis client instance.
|
||||
func WithRedis(client redis.UniversalClient) Option {
|
||||
return func(p *Plugin) {
|
||||
p.redisClient = client
|
||||
}
|
||||
}
|
||||
|
||||
// WithKeyPrefix sets a custom Redis key prefix.
|
||||
func WithKeyPrefix(prefix string) Option {
|
||||
return func(p *Plugin) {
|
||||
p.keyPrefix = prefix
|
||||
}
|
||||
}
|
||||
|
||||
// WithRAMCapacity sets the maximum capacity for the L1 RAM cache.
|
||||
func WithRAMCapacity(capacity int) Option {
|
||||
return func(p *Plugin) {
|
||||
p.ramCapacity = capacity
|
||||
}
|
||||
}
|
||||
|
||||
// Plugin implements core.Plugin to provide contracts.CacheService.
|
||||
type Plugin struct {
|
||||
redisClient redis.UniversalClient
|
||||
keyPrefix string
|
||||
ramCapacity int
|
||||
}
|
||||
|
||||
// New creates a new cache infrastructure plugin.
|
||||
func New(opts ...Option) *Plugin {
|
||||
p := &Plugin{
|
||||
ramCapacity: defaultRAMCapacity,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(p)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Name returns the unique identifier of the cache plugin.
|
||||
func (p *Plugin) Name() string {
|
||||
return "cache"
|
||||
}
|
||||
|
||||
// Apply mounts the multi-layer cache service into the Context.
|
||||
func (p *Plugin) Apply(ctx *core.Context) error {
|
||||
redisClient := p.redisClient
|
||||
if redisClient == nil {
|
||||
redisClient = Redis
|
||||
}
|
||||
|
||||
ramCache, err := ram.New[string, ramEntry](ram.Options{
|
||||
MaximumSize: p.ramCapacity,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
svc := &cacheServiceImpl{
|
||||
ramCache: ramCache,
|
||||
redisClient: redisClient,
|
||||
keyPrefix: p.keyPrefix,
|
||||
pubSubChannel: defaultPubSubChannel,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
if redisClient != nil {
|
||||
svc.startPubSubListener()
|
||||
ctx.OnDispose(func() error {
|
||||
svc.stopPubSubListener()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
core.Provide[contracts.CacheService](ctx, svc)
|
||||
return nil
|
||||
}
|
||||
|
||||
type cacheServiceImpl struct {
|
||||
ramCache *ram.Cache[string, ramEntry]
|
||||
redisClient redis.UniversalClient
|
||||
keyPrefix string
|
||||
pubSubChannel string
|
||||
|
||||
subOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
stopCh chan struct{}
|
||||
pubsub *redis.PubSub
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) prefixedKey(key string) string {
|
||||
if s.keyPrefix != "" {
|
||||
return s.keyPrefix + key
|
||||
}
|
||||
return PrefixedKey(key)
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) startPubSubListener() {
|
||||
if s.redisClient == nil {
|
||||
return
|
||||
}
|
||||
|
||||
s.subOnce.Do(func() {
|
||||
pubsub := s.redisClient.Subscribe(context.Background(), s.pubSubChannel)
|
||||
s.pubsub = pubsub
|
||||
|
||||
util.Go(func() {
|
||||
ch := pubsub.Channel()
|
||||
for {
|
||||
select {
|
||||
case <-s.stopCh:
|
||||
return
|
||||
case msg, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if msg != nil && msg.Payload != "" {
|
||||
s.ramCache.Invalidate(msg.Payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) stopPubSubListener() {
|
||||
s.stopOnce.Do(func() {
|
||||
close(s.stopCh)
|
||||
if s.pubsub != nil {
|
||||
_ = s.pubsub.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) Get(ctx context.Context, key string, target any) error {
|
||||
// 1. Check L1 RAM cache
|
||||
if entry, ok := s.ramCache.GetIfPresent(key); ok {
|
||||
if entry.expireAt.IsZero() || time.Now().Before(entry.expireAt) {
|
||||
return json.Unmarshal(entry.data, target)
|
||||
}
|
||||
// Expired in L1 RAM
|
||||
s.ramCache.Invalidate(key)
|
||||
}
|
||||
|
||||
// 2. Check L2 Redis cache
|
||||
if s.redisClient != nil {
|
||||
data, err := s.redisClient.Get(ctx, s.prefixedKey(key)).Bytes()
|
||||
if err == nil {
|
||||
// Backfill L1 RAM cache
|
||||
s.ramCache.Set(key, ramEntry{
|
||||
data: data,
|
||||
})
|
||||
return json.Unmarshal(data, target)
|
||||
} else if !errors.Is(err, redis.Nil) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return contracts.ErrCacheMiss
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) Set(ctx context.Context, key string, value any, ttl time.Duration) error {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var expireAt time.Time
|
||||
if ttl > 0 {
|
||||
expireAt = time.Now().Add(ttl)
|
||||
}
|
||||
|
||||
// 1. Write L1 RAM cache
|
||||
s.ramCache.Set(key, ramEntry{
|
||||
data: data,
|
||||
expireAt: expireAt,
|
||||
})
|
||||
|
||||
// 2. Write L2 Redis cache
|
||||
if s.redisClient != nil {
|
||||
if err := s.redisClient.Set(ctx, s.prefixedKey(key), data, ttl).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) Delete(ctx context.Context, key string) error {
|
||||
// 1. Evict L1 RAM
|
||||
s.ramCache.Invalidate(key)
|
||||
|
||||
// 2. Evict L2 Redis and broadcast invalidation to cluster nodes
|
||||
if s.redisClient != nil {
|
||||
if err := s.redisClient.Del(ctx, s.prefixedKey(key)).Err(); err != nil && !errors.Is(err, redis.Nil) {
|
||||
return err
|
||||
}
|
||||
_ = s.redisClient.Publish(ctx, s.pubSubChannel, key).Err()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) Invalidate(ctx context.Context, key string) error {
|
||||
return s.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (s *cacheServiceImpl) GetOrSet(ctx context.Context, key string, target any, ttl time.Duration, loader func() (any, error)) error {
|
||||
err := s.Get(ctx, key, target)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, contracts.ErrCacheMiss) {
|
||||
return err
|
||||
}
|
||||
|
||||
val, err := loader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.Set(ctx, key, val, ttl); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data, err := json.Marshal(val)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return json.Unmarshal(data, target)
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package cache_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
|
||||
)
|
||||
|
||||
func TestCachePluginOperations(t *testing.T) {
|
||||
mr, err := miniredis.Run()
|
||||
require.NoError(t, err)
|
||||
defer mr.Close()
|
||||
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: mr.Addr(),
|
||||
})
|
||||
defer func() { _ = rdb.Close() }()
|
||||
|
||||
p := cache.New(
|
||||
cache.WithRedis(rdb),
|
||||
cache.WithKeyPrefix("app:"),
|
||||
cache.WithRAMCapacity(500),
|
||||
)
|
||||
assert.Equal(t, "cache", p.Name())
|
||||
|
||||
ctx := core.NewContext(context.Background())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
svc, err := core.Inject[contracts.CacheService](ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
type Data struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
testCtx := context.Background()
|
||||
|
||||
// 1. ErrCacheMiss
|
||||
var out Data
|
||||
err = svc.Get(testCtx, "missing", &out)
|
||||
assert.ErrorIs(t, err, contracts.ErrCacheMiss)
|
||||
|
||||
// 2. Set & Get
|
||||
in := Data{Value: "hello"}
|
||||
require.NoError(t, svc.Set(testCtx, "key1", in, 5*time.Minute))
|
||||
|
||||
require.NoError(t, svc.Get(testCtx, "key1", &out))
|
||||
assert.Equal(t, "hello", out.Value)
|
||||
|
||||
// 3. GetOrSet
|
||||
var target Data
|
||||
err = svc.GetOrSet(testCtx, "key1", &target, time.Minute, func() (any, error) {
|
||||
return Data{Value: "from_loader"}, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello", target.Value)
|
||||
|
||||
err = svc.GetOrSet(testCtx, "key2", &target, time.Minute, func() (any, error) {
|
||||
return Data{Value: "from_loader"}, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "from_loader", target.Value)
|
||||
|
||||
// 4. Invalidate / Delete
|
||||
require.NoError(t, svc.Invalidate(testCtx, "key1"))
|
||||
err = svc.Get(testCtx, "key1", &out)
|
||||
assert.ErrorIs(t, err, contracts.ErrCacheMiss)
|
||||
|
||||
require.NoError(t, ctx.Dispose())
|
||||
}
|
||||
Vendored
+198
@@ -0,0 +1,198 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/config"
|
||||
"github.com/redis/go-redis/extra/redisotel/v9"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
)
|
||||
|
||||
var (
|
||||
// Redis 全局 Redis 客户端实例
|
||||
Redis redis.UniversalClient
|
||||
)
|
||||
|
||||
func init() {
|
||||
cfg := config.Config.Redis
|
||||
|
||||
if !cfg.Enabled {
|
||||
log.Println("[Redis] is disabled, skipping Redis initialization")
|
||||
return
|
||||
}
|
||||
|
||||
if cfg.ClusterMode {
|
||||
// Cluster 模式
|
||||
Redis = redis.NewClusterClient(&redis.ClusterOptions{
|
||||
Addrs: cfg.Addrs,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
PoolSize: cfg.PoolSize,
|
||||
MinIdleConns: cfg.MinIdleConn,
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
|
||||
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
|
||||
MaxRetries: cfg.MaxRetries,
|
||||
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
|
||||
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
|
||||
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
|
||||
})
|
||||
log.Println("[Redis] initialized in Cluster mode")
|
||||
} else {
|
||||
// Standalone 或 Sentinel 模式
|
||||
options := &redis.UniversalOptions{
|
||||
Addrs: cfg.Addrs,
|
||||
MasterName: cfg.MasterName, // 非空时启用 Sentinel
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
DB: cfg.DB,
|
||||
PoolSize: cfg.PoolSize,
|
||||
MinIdleConns: cfg.MinIdleConn,
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
|
||||
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
|
||||
MaxRetries: cfg.MaxRetries,
|
||||
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
|
||||
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
|
||||
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
|
||||
}
|
||||
if cfg.MasterName != "" {
|
||||
client := redis.NewFailoverClient(options.Failover())
|
||||
// FailoverOptions 暂不暴露该配置,在首次建连前写入客户端选项。
|
||||
client.Options().MaintNotificationsConfig = redisMaintNotificationsConfig(cfg.MaintNotifications)
|
||||
Redis = client
|
||||
log.Println("[Redis] initialized in Sentinel mode")
|
||||
} else {
|
||||
Redis = redis.NewUniversalClient(options)
|
||||
log.Println("[Redis] initialized in Standalone mode")
|
||||
}
|
||||
}
|
||||
|
||||
// OpenTelemetry 追踪(UniversalClient 兼容)
|
||||
if err := redisotel.InstrumentTracing(
|
||||
Redis,
|
||||
redisotel.WithAttributes(
|
||||
attribute.String("db.instance", fmt.Sprintf("%v", cfg.DB)),
|
||||
attribute.String("db.ip", strings.Join(cfg.Addrs, ",")),
|
||||
attribute.String("db.system", "Redis"),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[Redis] failed to init trace: %v\n", err)
|
||||
}
|
||||
|
||||
// 测试连接
|
||||
_, err := Redis.Ping(context.Background()).Result()
|
||||
if err != nil {
|
||||
log.Fatalf("[Redis] failed to connect to redis: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func redisMaintNotificationsConfig(enabled bool) *maintnotifications.Config {
|
||||
mode := maintnotifications.ModeDisabled
|
||||
if enabled {
|
||||
mode = maintnotifications.ModeAuto
|
||||
}
|
||||
return &maintnotifications.Config{Mode: mode}
|
||||
}
|
||||
|
||||
// PrefixedKey 返回带前缀的 Key
|
||||
func PrefixedKey(key string) string {
|
||||
prefix := config.Config.Redis.KeyPrefix
|
||||
if prefix == "" {
|
||||
return key
|
||||
}
|
||||
return prefix + key
|
||||
}
|
||||
|
||||
// HSetJSON 将泛型数据序列化为 JSON 并设置到 Redis Hash
|
||||
// ctx: 上下文
|
||||
// hashKey: Redis Hash key
|
||||
// fieldKey: Hash field key
|
||||
// data: 要存储的数据(泛型)
|
||||
func HSetJSON[T any](ctx context.Context, hashKey, fieldKey string, data T) error {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := Redis.HSet(ctx, PrefixedKey(hashKey), fieldKey, jsonData).Err(); err != nil {
|
||||
return fmt.Errorf(errRedisHashSetFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HDel removes one or more fields from a Redis Hash.
|
||||
func HDel(ctx context.Context, hashKey string, fieldKeys ...string) error {
|
||||
if Redis == nil || len(fieldKeys) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := Redis.HDel(ctx, PrefixedKey(hashKey), fieldKeys...).Err(); err != nil {
|
||||
return fmt.Errorf(errRedisHashDeleteFailed, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HGetJSON 从 Redis Hash 获取数据并反序列化为泛型类型
|
||||
// ctx: 上下文
|
||||
// hashKey: Redis Hash key
|
||||
// fieldKey: Hash field key
|
||||
// data: 用于接收数据的指针(泛型)
|
||||
func HGetJSON[T any](ctx context.Context, hashKey, fieldKey string, data *T) error {
|
||||
val, err := Redis.HGet(ctx, PrefixedKey(hashKey), fieldKey).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(val), data); err != nil {
|
||||
return fmt.Errorf(errUnmarshalDataFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetJSON 从Redis获取数据并反序列化为泛型类型
|
||||
// ctx: 上下文
|
||||
// key: Redis key
|
||||
// data: 用于接收数据的指针(泛型)
|
||||
func GetJSON[T any](ctx context.Context, key string, data *T) error {
|
||||
val, err := Redis.Get(ctx, PrefixedKey(key)).Bytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(val, data); err != nil {
|
||||
return fmt.Errorf(errUnmarshalDataFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetJSON 将泛型数据序列化为JSON并设置到Redis
|
||||
// ctx: 上下文
|
||||
// key: Redis key
|
||||
// data: 要存储的数据(泛型)
|
||||
// expiration: 过期时间
|
||||
func SetJSON[T any](ctx context.Context, key string, data T, expiration time.Duration) error {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf(errMarshalDataFailed, err)
|
||||
}
|
||||
|
||||
if err := Redis.Set(ctx, PrefixedKey(key), jsonData, expiration).Err(); err != nil {
|
||||
return fmt.Errorf(errRedisKeySetFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
)
|
||||
|
||||
func TestRedisMaintNotificationsConfig(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
enabled bool
|
||||
want maintnotifications.Mode
|
||||
}{
|
||||
{name: "disabled by default", enabled: false, want: maintnotifications.ModeDisabled},
|
||||
{name: "auto when enabled", enabled: true, want: maintnotifications.ModeAuto},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := redisMaintNotificationsConfig(test.enabled)
|
||||
if cfg.Mode != test.want {
|
||||
t.Fatalf("maintenance notifications mode = %v, want %v", cfg.Mode, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package database 提供数据库连接与基础设施
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/config"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
clickhouseDriver "gorm.io/driver/clickhouse"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/plugin/opentelemetry/tracing"
|
||||
)
|
||||
|
||||
const (
|
||||
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
|
||||
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
|
||||
)
|
||||
|
||||
var (
|
||||
// ChConn ClickHouse 原生连接实例,用于批量写入
|
||||
ChConn driver.Conn
|
||||
|
||||
chDB *gorm.DB
|
||||
)
|
||||
|
||||
func init() {
|
||||
if !config.Config.ClickHouse.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
cfg := config.Config.ClickHouse
|
||||
if cfg.Database == "" {
|
||||
log.Fatalf("[ClickHouse] database name is required (expected: wavelet)\n")
|
||||
}
|
||||
|
||||
opts := buildClickHouseOptions()
|
||||
|
||||
var err error
|
||||
ChConn, err = clickhouse.Open(opts)
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = ChConn.Ping(context.Background()); err != nil {
|
||||
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
|
||||
}
|
||||
|
||||
chDB, err = gorm.Open(clickhouseDriver.New(clickhouseDriver.Config{
|
||||
DSN: buildClickHouseDSN(),
|
||||
}), &gorm.Config{
|
||||
SkipDefaultTransaction: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init gorm connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = chDB.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", cfg.Database),
|
||||
attribute.String("db.system", "ClickHouse"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[ClickHouse] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB, err := chDB.DB()
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] load sql db failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB.SetMaxIdleConns(cfg.MaxIdleConn)
|
||||
sqlDB.SetMaxOpenConns(cfg.MaxOpenConn)
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(cfg.ConnMaxLifetime) * time.Second)
|
||||
|
||||
log.Println("[ClickHouse] connection established successfully")
|
||||
}
|
||||
|
||||
func buildClickHouseOptions() *clickhouse.Options {
|
||||
cfg := config.Config.ClickHouse
|
||||
|
||||
return &clickhouse.Options{
|
||||
Addr: cfg.Hosts,
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
Settings: clickhouse.Settings{
|
||||
"max_execution_time": clickhouseMaxExecTime,
|
||||
},
|
||||
Compression: &clickhouse.Compression{
|
||||
Method: clickhouse.CompressionLZ4,
|
||||
},
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
MaxOpenConns: cfg.MaxOpenConn,
|
||||
MaxIdleConns: cfg.MaxIdleConn,
|
||||
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
|
||||
BlockBufferSize: cfg.BlockBufferSize,
|
||||
}
|
||||
}
|
||||
|
||||
func buildClickHouseDSN() string {
|
||||
cfg := config.Config.ClickHouse
|
||||
|
||||
chURL := &url.URL{
|
||||
Scheme: "clickhouse",
|
||||
Host: strings.Join(cfg.Hosts, ","),
|
||||
Path: "/" + cfg.Database,
|
||||
}
|
||||
if cfg.Username != "" || cfg.Password != "" {
|
||||
chURL.User = url.UserPassword(cfg.Username, cfg.Password)
|
||||
}
|
||||
|
||||
query := chURL.Query()
|
||||
query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout))
|
||||
query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor))
|
||||
query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime))
|
||||
chURL.RawQuery = query.Encode()
|
||||
|
||||
return chURL.String()
|
||||
}
|
||||
|
||||
// ChDB returns a context-aware GORM ClickHouse instance.
|
||||
func ChDB(ctx context.Context) *gorm.DB {
|
||||
if chDB == nil {
|
||||
return nil
|
||||
}
|
||||
return chDB.WithContext(ctx)
|
||||
}
|
||||
|
||||
// SetChDBForTest sets the package-level ClickHouse GORM instance for testing.
|
||||
func SetChDBForTest(d *gorm.DB) {
|
||||
chDB = d
|
||||
}
|
||||
|
||||
// SetChConnForTest sets the package-level native ClickHouse connection for testing.
|
||||
func SetChConnForTest(c driver.Conn) {
|
||||
ChConn = c
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package database provides the relational database infrastructure plugin for Cordis.
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Option configures the database plugin.
|
||||
type Option func(*Plugin)
|
||||
|
||||
// WithDB configures an explicit *gorm.DB instance for the plugin.
|
||||
func WithDB(d *gorm.DB) Option {
|
||||
return func(p *Plugin) {
|
||||
p.db = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithNamedDB registers a named secondary database connection.
|
||||
func WithNamedDB(name string, d *gorm.DB) Option {
|
||||
return func(p *Plugin) {
|
||||
if p.namedDBs == nil {
|
||||
p.namedDBs = make(map[string]*gorm.DB)
|
||||
}
|
||||
p.namedDBs[name] = d
|
||||
}
|
||||
}
|
||||
|
||||
// Plugin implements core.Plugin to provide contracts.DBService into the Cordis micro-kernel.
|
||||
type Plugin struct {
|
||||
db *gorm.DB
|
||||
namedDBs map[string]*gorm.DB
|
||||
}
|
||||
|
||||
// New creates a new database infrastructure plugin.
|
||||
func New(opts ...Option) *Plugin {
|
||||
p := &Plugin{
|
||||
namedDBs: make(map[string]*gorm.DB),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(p)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Name returns the unique identifier of the database plugin.
|
||||
func (p *Plugin) Name() string {
|
||||
return "database"
|
||||
}
|
||||
|
||||
// Apply mounts the database service into the Context.
|
||||
func (p *Plugin) Apply(ctx *core.Context) error {
|
||||
targetDB := p.db
|
||||
if targetDB == nil {
|
||||
targetDB = DB(context.Background())
|
||||
}
|
||||
|
||||
svc := &dbServiceImpl{
|
||||
primary: targetDB,
|
||||
namedDBs: p.namedDBs,
|
||||
}
|
||||
|
||||
core.Provide[contracts.DBService](ctx, svc)
|
||||
return nil
|
||||
}
|
||||
|
||||
type dbServiceImpl struct {
|
||||
primary *gorm.DB
|
||||
namedDBs map[string]*gorm.DB
|
||||
}
|
||||
|
||||
func (s *dbServiceImpl) GORM() *gorm.DB {
|
||||
if s.primary != nil {
|
||||
return s.primary
|
||||
}
|
||||
return DB(context.Background())
|
||||
}
|
||||
|
||||
func (s *dbServiceImpl) DB(ctx context.Context) *gorm.DB {
|
||||
if s.primary != nil {
|
||||
return s.primary.WithContext(ctx)
|
||||
}
|
||||
return DB(ctx)
|
||||
}
|
||||
|
||||
func (s *dbServiceImpl) Named(name string) *gorm.DB {
|
||||
if s.namedDBs != nil {
|
||||
if d, ok := s.namedDBs[name]; ok && d != nil {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return s.GORM()
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/database"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
ID uint64 `gorm:"primaryKey"`
|
||||
Name string
|
||||
}
|
||||
|
||||
func TestDatabasePlugin(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "db_test.db")
|
||||
gdb, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gdb.AutoMigrate(&Item{}))
|
||||
|
||||
namedPath := filepath.Join(t.TempDir(), "named_test.db")
|
||||
namedDB, err := gorm.Open(sqlite.Open(namedPath), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, namedDB.AutoMigrate(&Item{}))
|
||||
|
||||
p := database.New(
|
||||
database.WithDB(gdb),
|
||||
database.WithNamedDB("analytics", namedDB),
|
||||
)
|
||||
assert.Equal(t, "database", p.Name())
|
||||
|
||||
ctx := core.NewContext(context.Background())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
svc, err := core.Inject[contracts.DBService](ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, svc)
|
||||
|
||||
assert.Equal(t, gdb, svc.GORM())
|
||||
assert.NotNil(t, svc.DB(context.Background()))
|
||||
assert.Equal(t, namedDB, svc.Named("analytics"))
|
||||
assert.Equal(t, gdb, svc.Named("non_existent"))
|
||||
|
||||
// Verify DB write
|
||||
item := Item{ID: 1, Name: "TestItem"}
|
||||
require.NoError(t, svc.DB(context.Background()).Create(&item).Error)
|
||||
|
||||
var retrieved Item
|
||||
require.NoError(t, svc.GORM().First(&retrieved, 1).Error)
|
||||
assert.Equal(t, "TestItem", retrieved.Name)
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/config"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/plugin/dbresolver"
|
||||
"gorm.io/plugin/opentelemetry/tracing"
|
||||
)
|
||||
|
||||
var (
|
||||
db *gorm.DB
|
||||
)
|
||||
|
||||
func init() {
|
||||
if !config.Config.Database.Enabled {
|
||||
// PostgreSQL 禁用,使用 SQLite
|
||||
initSQLite()
|
||||
return
|
||||
}
|
||||
|
||||
initPostgres()
|
||||
}
|
||||
|
||||
// initSQLite 初始化 SQLite 数据库(PostgreSQL 禁用时的后备方案)
|
||||
func initSQLite() {
|
||||
sqlitePath := config.Config.Database.SQLitePath
|
||||
if sqlitePath == "" {
|
||||
sqlitePath = "./data/wavelet.db"
|
||||
}
|
||||
|
||||
var err error
|
||||
db, err = gorm.Open(sqlite.Open(sqlitePath), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: &gormZapLogger{
|
||||
logLevel: parseLogLevel(config.Config.Database.LogLevel),
|
||||
slowThreshold: config.Config.Database.SlowThreshold,
|
||||
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[SQLite] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
// Trace 注入
|
||||
if err = db.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", sqlitePath),
|
||||
attribute.String("db.system", "SQLite"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[SQLite] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
log.Printf("[SQLite] initialized (path: %s)\n", sqlitePath)
|
||||
}
|
||||
|
||||
// initPostgres 初始化 PostgreSQL 数据库
|
||||
func initPostgres() {
|
||||
var err error
|
||||
dbConfig := config.Config.Database
|
||||
|
||||
// 构建主库 DSN 并连接
|
||||
primaryDSN := buildDSN(dbConfig.Host, dbConfig.Port, dbConfig.Username, dbConfig.Password)
|
||||
|
||||
pgConfig := postgres.Config{
|
||||
DSN: primaryDSN,
|
||||
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
|
||||
}
|
||||
|
||||
db, err = gorm.Open(postgres.New(pgConfig), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: &gormZapLogger{
|
||||
logLevel: parseLogLevel(config.Config.Database.LogLevel),
|
||||
slowThreshold: config.Config.Database.SlowThreshold,
|
||||
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[PostgreSQL] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
// Trace 注入
|
||||
if err = db.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", dbConfig.Database),
|
||||
attribute.String("db.ip", dbConfig.Host),
|
||||
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
|
||||
attribute.String("db.system", "PostgreSQL"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[PostgreSQL] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
if len(dbConfig.Replicas) > 0 {
|
||||
var replicaDialectors []gorm.Dialector
|
||||
for _, replica := range dbConfig.Replicas {
|
||||
username := replica.Username
|
||||
if username == "" {
|
||||
username = dbConfig.Username
|
||||
}
|
||||
password := replica.Password
|
||||
if password == "" {
|
||||
password = dbConfig.Password
|
||||
}
|
||||
replicaDSN := buildDSN(replica.Host, replica.Port, username, password)
|
||||
replicaDialectors = append(replicaDialectors, postgres.New(postgres.Config{
|
||||
DSN: replicaDSN,
|
||||
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
|
||||
}))
|
||||
}
|
||||
|
||||
resolver := dbresolver.Register(dbresolver.Config{
|
||||
Replicas: replicaDialectors,
|
||||
Policy: dbresolver.RandomPolicy{},
|
||||
})
|
||||
|
||||
resolver.SetMaxIdleConns(dbConfig.MaxIdleConn).
|
||||
SetMaxOpenConns(dbConfig.MaxOpenConn).
|
||||
SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second).
|
||||
SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
|
||||
|
||||
if err = db.Use(resolver); err != nil {
|
||||
log.Fatalf("[PostgreSQL] init dbresolver failed: %v\n", err)
|
||||
}
|
||||
log.Printf("[PostgreSQL] initialized in Primary-Replica mode (%d replicas)\n", len(dbConfig.Replicas))
|
||||
} else {
|
||||
log.Println("[PostgreSQL] initialized in Standalone mode")
|
||||
}
|
||||
|
||||
// 获取通用数据库对象设置连接池
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
log.Fatalf("[PostgreSQL] load sql db failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB.SetMaxIdleConns(dbConfig.MaxIdleConn)
|
||||
sqlDB.SetMaxOpenConns(dbConfig.MaxOpenConn)
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second)
|
||||
sqlDB.SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
|
||||
}
|
||||
|
||||
// buildDSN 构建 PostgreSQL DSN
|
||||
func buildDSN(host string, port int, username, password string) string {
|
||||
cfg := config.Config.Database
|
||||
pqURL := &url.URL{
|
||||
Scheme: "postgres",
|
||||
Host: net.JoinHostPort(host, strconv.Itoa(port)),
|
||||
Path: cfg.Database,
|
||||
}
|
||||
if username != "" {
|
||||
pqURL.User = url.UserPassword(username, password)
|
||||
}
|
||||
|
||||
query := pqURL.Query()
|
||||
sslMode := cfg.SSLMode
|
||||
if sslMode == "" {
|
||||
sslMode = "disable"
|
||||
}
|
||||
query.Set("sslmode", sslMode)
|
||||
if cfg.ApplicationName != "" {
|
||||
query.Set("application_name", cfg.ApplicationName)
|
||||
}
|
||||
if cfg.SearchPath != "" {
|
||||
query.Set("search_path", cfg.SearchPath)
|
||||
}
|
||||
if cfg.DefaultQueryExecMode != "" {
|
||||
query.Set("default_query_exec_mode", cfg.DefaultQueryExecMode)
|
||||
}
|
||||
if cfg.StatementCacheCapacity > 0 {
|
||||
query.Set("statement_cache_capacity", strconv.Itoa(cfg.StatementCacheCapacity))
|
||||
}
|
||||
|
||||
rawQuery := query.Encode()
|
||||
if cfg.TimeZone != "" {
|
||||
if rawQuery != "" {
|
||||
rawQuery += "&"
|
||||
}
|
||||
rawQuery += "TimeZone=" + cfg.TimeZone
|
||||
}
|
||||
pqURL.RawQuery = rawQuery
|
||||
|
||||
return pqURL.String()
|
||||
}
|
||||
|
||||
// DB 返回带上下文追踪的 GORM 数据库实例
|
||||
func DB(ctx context.Context) *gorm.DB {
|
||||
if db == nil {
|
||||
return nil
|
||||
}
|
||||
return db.WithContext(ctx)
|
||||
}
|
||||
|
||||
// SetDB sets the package-level database instance for testing.
|
||||
func SetDB(d *gorm.DB) {
|
||||
db = d
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/logger"
|
||||
"gorm.io/gorm"
|
||||
gormLogger "gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// nanoToMilli 纳秒转毫秒的除数
|
||||
const nanoToMilli = 1e6
|
||||
|
||||
type gormZapLogger struct {
|
||||
logLevel gormLogger.LogLevel
|
||||
ignoreRecordNotFoundError bool
|
||||
slowThreshold time.Duration
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) LogMode(level gormLogger.LogLevel) gormLogger.Interface {
|
||||
clone := *l
|
||||
clone.logLevel = level
|
||||
return &clone
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Info(ctx context.Context, fmt string, args ...interface{}) {
|
||||
if l.logLevel >= gormLogger.Info {
|
||||
logger.InfoF(ctx, fmt, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Warn(ctx context.Context, fmt string, args ...interface{}) {
|
||||
if l.logLevel >= gormLogger.Warn {
|
||||
logger.WarnF(ctx, fmt, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Error(ctx context.Context, fmt string, args ...interface{}) {
|
||||
if l.logLevel >= gormLogger.Error {
|
||||
logger.ErrorF(ctx, fmt, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
|
||||
elapsed := time.Since(begin)
|
||||
switch {
|
||||
case err != nil && l.logLevel >= gormLogger.Error && (!errors.Is(err, gorm.ErrRecordNotFound) || !l.ignoreRecordNotFoundError):
|
||||
_, rows := fc()
|
||||
logger.ErrorF(ctx, "database query failed: %s [%.3fms] [rows:%v]", err, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
|
||||
case elapsed > l.slowThreshold && l.slowThreshold != 0 && l.logLevel >= gormLogger.Warn:
|
||||
_, rows := fc()
|
||||
slowLog := fmt.Sprintf("SLOW SQL >= %v", l.slowThreshold)
|
||||
logger.WarnF(ctx, "%s [%.3fms] [rows:%v]", slowLog, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
|
||||
case l.logLevel == gormLogger.Info:
|
||||
sql, rows := fc()
|
||||
logger.DebugF(ctx, "[%.3fms] [rows:%v] %s", float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows), sql)
|
||||
}
|
||||
}
|
||||
|
||||
func formatRows(rows int64) interface{} {
|
||||
if rows == -1 {
|
||||
return "-"
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func parseLogLevel(level string) gormLogger.LogLevel {
|
||||
level = strings.ToLower(level)
|
||||
switch level {
|
||||
case "silent":
|
||||
return gormLogger.Silent
|
||||
case "error":
|
||||
return gormLogger.Error
|
||||
case "warn":
|
||||
return gormLogger.Warn
|
||||
case "info":
|
||||
return gormLogger.Info
|
||||
case "debug":
|
||||
return gormLogger.Info
|
||||
default:
|
||||
return gormLogger.Info
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
gormLogger "gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestParseLogLevel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
configuredLevel string
|
||||
want gormLogger.LogLevel
|
||||
}{
|
||||
{
|
||||
name: "debug enables SQL trace processing",
|
||||
configuredLevel: "debug",
|
||||
want: gormLogger.Info,
|
||||
},
|
||||
{
|
||||
name: "development preserves configured level",
|
||||
configuredLevel: "warn",
|
||||
want: gormLogger.Warn,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := parseLogLevel(tt.configuredLevel); got != tt.want {
|
||||
t.Fatalf("parseLogLevel() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package infra_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/database"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/logger"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage/objectstore"
|
||||
)
|
||||
|
||||
type TestUser struct {
|
||||
ID uint64 `gorm:"primaryKey"`
|
||||
Name string
|
||||
}
|
||||
|
||||
func setupTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.AutoMigrate(&TestUser{}))
|
||||
return db
|
||||
}
|
||||
|
||||
func TestDatabasePlugin(t *testing.T) {
|
||||
ctx := core.NewContext(context.Background())
|
||||
testDB := setupTestDB(t)
|
||||
|
||||
p := database.New(database.WithDB(testDB))
|
||||
require.Equal(t, "database", p.Name())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
dbSvc, err := core.Inject[contracts.DBService](ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, dbSvc)
|
||||
|
||||
// Verify GORM and DB methods
|
||||
assert.NotNil(t, dbSvc.GORM())
|
||||
assert.NotNil(t, dbSvc.DB(context.Background()))
|
||||
|
||||
// Test CRUD via service
|
||||
user := TestUser{ID: 1, Name: "Alice"}
|
||||
require.NoError(t, dbSvc.DB(context.Background()).Create(&user).Error)
|
||||
|
||||
var fetched TestUser
|
||||
require.NoError(t, dbSvc.GORM().First(&fetched, 1).Error)
|
||||
assert.Equal(t, "Alice", fetched.Name)
|
||||
|
||||
// Test NamedDB fallback
|
||||
assert.NotNil(t, dbSvc.Named("replica"))
|
||||
}
|
||||
|
||||
func TestCachePluginRAMOnly(t *testing.T) {
|
||||
ctx := core.NewContext(context.Background())
|
||||
p := cache.New()
|
||||
require.Equal(t, "cache", p.Name())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
cacheSvc, err := core.Inject[contracts.CacheService](ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cacheSvc)
|
||||
|
||||
type CacheItem struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
testCtx := context.Background()
|
||||
|
||||
// 1. Get non-existing key
|
||||
var notFound CacheItem
|
||||
err = cacheSvc.Get(testCtx, "missing:key", ¬Found)
|
||||
assert.ErrorIs(t, err, contracts.ErrCacheMiss)
|
||||
|
||||
// 2. Set and Get
|
||||
item := CacheItem{Name: "item1", Count: 42}
|
||||
require.NoError(t, cacheSvc.Set(testCtx, "test:item1", item, time.Minute))
|
||||
|
||||
var retrieved CacheItem
|
||||
require.NoError(t, cacheSvc.Get(testCtx, "test:item1", &retrieved))
|
||||
assert.Equal(t, item, retrieved)
|
||||
|
||||
// 3. GetOrSet
|
||||
var getOrSetTarget CacheItem
|
||||
var loaderCalled bool
|
||||
err = cacheSvc.GetOrSet(testCtx, "test:item1", &getOrSetTarget, time.Minute, func() (any, error) {
|
||||
loaderCalled = true
|
||||
return CacheItem{Name: "never_called", Count: 0}, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, loaderCalled)
|
||||
assert.Equal(t, item, getOrSetTarget)
|
||||
|
||||
// GetOrSet with cache miss
|
||||
var newItem CacheItem
|
||||
err = cacheSvc.GetOrSet(testCtx, "test:item2", &newItem, time.Minute, func() (any, error) {
|
||||
loaderCalled = true
|
||||
return CacheItem{Name: "loaded", Count: 99}, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, loaderCalled)
|
||||
assert.Equal(t, "loaded", newItem.Name)
|
||||
assert.Equal(t, 99, newItem.Count)
|
||||
|
||||
// 4. Delete
|
||||
require.NoError(t, cacheSvc.Delete(testCtx, "test:item1"))
|
||||
err = cacheSvc.Get(testCtx, "test:item1", &retrieved)
|
||||
assert.ErrorIs(t, err, contracts.ErrCacheMiss)
|
||||
|
||||
// 5. Invalidate alias
|
||||
require.NoError(t, cacheSvc.Invalidate(testCtx, "test:item2"))
|
||||
err = cacheSvc.Get(testCtx, "test:item2", &retrieved)
|
||||
assert.ErrorIs(t, err, contracts.ErrCacheMiss)
|
||||
}
|
||||
|
||||
func TestCachePluginWithRedisAndPubSub(t *testing.T) {
|
||||
mr, err := miniredis.Run()
|
||||
require.NoError(t, err)
|
||||
defer mr.Close()
|
||||
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: mr.Addr(),
|
||||
})
|
||||
defer func() { _ = rdb.Close() }()
|
||||
|
||||
ctx1 := core.NewContext(context.Background())
|
||||
ctx2 := core.NewContext(context.Background())
|
||||
|
||||
p1 := cache.New(cache.WithRedis(rdb), cache.WithKeyPrefix("test:"))
|
||||
p2 := cache.New(cache.WithRedis(rdb), cache.WithKeyPrefix("test:"))
|
||||
|
||||
require.NoError(t, p1.Apply(ctx1))
|
||||
require.NoError(t, p2.Apply(ctx2))
|
||||
|
||||
cache1, err := core.Inject[contracts.CacheService](ctx1)
|
||||
require.NoError(t, err)
|
||||
cache2, err := core.Inject[contracts.CacheService](ctx2)
|
||||
require.NoError(t, err)
|
||||
|
||||
testCtx := context.Background()
|
||||
|
||||
// Node 1 writes to cache
|
||||
type UserCache struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
require.NoError(t, cache1.Set(testCtx, "user:100", UserCache{Name: "Bob"}, 10*time.Minute))
|
||||
|
||||
// Node 2 reads from cache (misses Node 2's RAM, hits Redis, backfills Node 2's RAM)
|
||||
var u2 UserCache
|
||||
require.NoError(t, cache2.Get(testCtx, "user:100", &u2))
|
||||
assert.Equal(t, "Bob", u2.Name)
|
||||
|
||||
// Node 1 deletes cache (evicts Node 1 RAM, Redis, and broadcasts to Node 2)
|
||||
require.NoError(t, cache1.Delete(testCtx, "user:100"))
|
||||
|
||||
// Verify Redis is deleted
|
||||
var uRedis UserCache
|
||||
err = cache1.Get(testCtx, "user:100", &uRedis)
|
||||
assert.ErrorIs(t, err, contracts.ErrCacheMiss)
|
||||
|
||||
// Clean up contexts
|
||||
require.NoError(t, ctx1.Dispose())
|
||||
require.NoError(t, ctx2.Dispose())
|
||||
}
|
||||
|
||||
func TestLoggerPlugin(t *testing.T) {
|
||||
ctx := core.NewContext(context.Background())
|
||||
p := logger.New()
|
||||
require.Equal(t, "logger", p.Name())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
logSvc, err := core.Inject[contracts.LoggerService](ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, logSvc)
|
||||
|
||||
testCtx := context.Background()
|
||||
|
||||
// Should not panic on any log call
|
||||
logSvc.Debug(testCtx, "debug message", "key1", "val1")
|
||||
logSvc.Info(testCtx, "info message", "userID", 123)
|
||||
logSvc.Warn(testCtx, "warn message", "warning", true)
|
||||
logSvc.Error(testCtx, "error message", "err", "something broke")
|
||||
|
||||
logSvc.Debugf(testCtx, "formatted debug %d", 1)
|
||||
logSvc.Infof(testCtx, "formatted info %s", "test")
|
||||
logSvc.Warnf(testCtx, "formatted warn %v", map[string]int{"a": 1})
|
||||
logSvc.Errorf(testCtx, "formatted error %s", "fatal")
|
||||
|
||||
childLog := logSvc.With("module", "test_module")
|
||||
require.NotNil(t, childLog)
|
||||
childLog.Info(testCtx, "child log message", "action", "run")
|
||||
}
|
||||
|
||||
type memoryBackend struct {
|
||||
mu sync.RWMutex
|
||||
storage map[string][]byte
|
||||
}
|
||||
|
||||
func newMemoryBackend() *memoryBackend {
|
||||
return &memoryBackend{
|
||||
storage: make(map[string][]byte),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *memoryBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (objectstore.PutResult, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
data, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
return objectstore.PutResult{}, err
|
||||
}
|
||||
m.storage[key] = data
|
||||
return objectstore.PutResult{Key: key, Bucket: "test-bucket"}, nil
|
||||
}
|
||||
|
||||
func (m *memoryBackend) Get(ctx context.Context, key string) (*objectstore.Object, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
data, ok := m.storage[key]
|
||||
if !ok {
|
||||
return nil, gorm.ErrRecordNotFound
|
||||
}
|
||||
return &objectstore.Object{
|
||||
Body: io.NopCloser(bytes.NewReader(data)),
|
||||
ContentLength: int64(len(data)),
|
||||
ContentType: "application/octet-stream",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *memoryBackend) Delete(ctx context.Context, key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.storage, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memoryBackend) Test(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestStoragePlugin(t *testing.T) {
|
||||
ctx := core.NewContext(context.Background())
|
||||
backend := newMemoryBackend()
|
||||
|
||||
p := storage.New(storage.WithBackend(backend))
|
||||
require.Equal(t, "storage", p.Name())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
storageSvc, err := core.Inject[contracts.StorageService](ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, storageSvc)
|
||||
|
||||
testCtx := context.Background()
|
||||
|
||||
// 1. Put
|
||||
content := []byte("Hello, Wavelet Storage Plugin!")
|
||||
putRes, err := storageSvc.Put(testCtx, "uploads/hello.txt", bytes.NewReader(content), int64(len(content)), "text/plain")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "uploads/hello.txt", putRes.Key)
|
||||
assert.Equal(t, "test-bucket", putRes.Bucket)
|
||||
|
||||
// 2. Get
|
||||
obj, err := storageSvc.Get(testCtx, "uploads/hello.txt")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, obj)
|
||||
data, err := io.ReadAll(obj.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
assert.Equal(t, int64(len(content)), obj.ContentLength)
|
||||
|
||||
// 3. Delete
|
||||
require.NoError(t, storageSvc.Delete(testCtx, "uploads/hello.txt"))
|
||||
_, err = storageSvc.Get(testCtx, "uploads/hello.txt")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestAllInfraPluginsCombined(t *testing.T) {
|
||||
ctx := core.NewContext(context.Background())
|
||||
testDB := setupTestDB(t)
|
||||
memBackend := newMemoryBackend()
|
||||
|
||||
require.NoError(t, database.New(database.WithDB(testDB)).Apply(ctx))
|
||||
require.NoError(t, cache.New().Apply(ctx))
|
||||
require.NoError(t, logger.New().Apply(ctx))
|
||||
require.NoError(t, storage.New(storage.WithBackend(memBackend)).Apply(ctx))
|
||||
|
||||
// Using3 to resolve dependencies concurrently
|
||||
var resolved bool
|
||||
err := core.Using3(ctx, func(db contracts.DBService, c contracts.CacheService, l contracts.LoggerService) {
|
||||
resolved = true
|
||||
assert.NotNil(t, db)
|
||||
assert.NotNil(t, c)
|
||||
assert.NotNil(t, l)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, resolved)
|
||||
|
||||
// Using storage
|
||||
err = core.Using(ctx, func(s contracts.StorageService) {
|
||||
assert.NotNil(t, s)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package logger provides the structured logging infrastructure plugin for Cordis.
|
||||
package logger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/logger"
|
||||
)
|
||||
|
||||
// Plugin implements core.Plugin to provide contracts.LoggerService.
|
||||
type Plugin struct{}
|
||||
|
||||
// New creates a new logger infrastructure plugin.
|
||||
func New() *Plugin {
|
||||
return &Plugin{}
|
||||
}
|
||||
|
||||
// Name returns the unique identifier of the logger plugin.
|
||||
func (p *Plugin) Name() string {
|
||||
return "logger"
|
||||
}
|
||||
|
||||
// Apply mounts the logger service into the Context.
|
||||
func (p *Plugin) Apply(ctx *core.Context) error {
|
||||
svc := &loggerServiceImpl{}
|
||||
core.Provide[contracts.LoggerService](ctx, svc)
|
||||
return nil
|
||||
}
|
||||
|
||||
type loggerServiceImpl struct {
|
||||
extraFields []any
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) formatMsg(msg string, keysAndValues ...any) string {
|
||||
allFields := make([]any, 0, len(s.extraFields)+len(keysAndValues))
|
||||
allFields = append(allFields, s.extraFields...)
|
||||
allFields = append(allFields, keysAndValues...)
|
||||
|
||||
if len(allFields) == 0 {
|
||||
return msg
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(msg)
|
||||
sb.WriteString(" [")
|
||||
for i := 0; i < len(allFields); i += 2 {
|
||||
if i > 0 {
|
||||
sb.WriteString(" ")
|
||||
}
|
||||
if i+1 < len(allFields) {
|
||||
fmt.Fprintf(&sb, "%v=%v", allFields[i], allFields[i+1])
|
||||
} else {
|
||||
fmt.Fprintf(&sb, "%v", allFields[i])
|
||||
}
|
||||
}
|
||||
sb.WriteString("]")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Debug(ctx context.Context, msg string, keysAndValues ...any) {
|
||||
logger.DebugF(ctx, "%s", s.formatMsg(msg, keysAndValues...))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Info(ctx context.Context, msg string, keysAndValues ...any) {
|
||||
logger.InfoF(ctx, "%s", s.formatMsg(msg, keysAndValues...))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Warn(ctx context.Context, msg string, keysAndValues ...any) {
|
||||
logger.WarnF(ctx, "%s", s.formatMsg(msg, keysAndValues...))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Error(ctx context.Context, msg string, keysAndValues ...any) {
|
||||
logger.ErrorF(ctx, "%s", s.formatMsg(msg, keysAndValues...))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Debugf(ctx context.Context, format string, args ...any) {
|
||||
msg := fmt.Sprintf(format, args...)
|
||||
logger.DebugF(ctx, "%s", s.formatMsg(msg))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Infof(ctx context.Context, format string, args ...any) {
|
||||
msg := fmt.Sprintf(format, args...)
|
||||
logger.InfoF(ctx, "%s", s.formatMsg(msg))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Warnf(ctx context.Context, format string, args ...any) {
|
||||
msg := fmt.Sprintf(format, args...)
|
||||
logger.WarnF(ctx, "%s", s.formatMsg(msg))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) Errorf(ctx context.Context, format string, args ...any) {
|
||||
msg := fmt.Sprintf(format, args...)
|
||||
logger.ErrorF(ctx, "%s", s.formatMsg(msg))
|
||||
}
|
||||
|
||||
func (s *loggerServiceImpl) With(keysAndValues ...any) contracts.LoggerService {
|
||||
merged := make([]any, 0, len(s.extraFields)+len(keysAndValues))
|
||||
merged = append(merged, s.extraFields...)
|
||||
merged = append(merged, keysAndValues...)
|
||||
return &loggerServiceImpl{
|
||||
extraFields: merged,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logger_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/logger"
|
||||
)
|
||||
|
||||
func TestLoggerPluginOperations(t *testing.T) {
|
||||
p := logger.New()
|
||||
assert.Equal(t, "logger", p.Name())
|
||||
|
||||
ctx := core.NewContext(context.Background())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
svc, err := core.Inject[contracts.LoggerService](ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, svc)
|
||||
|
||||
testCtx := context.Background()
|
||||
|
||||
svc.Debug(testCtx, "debug msg", "key", "val")
|
||||
svc.Info(testCtx, "info msg", "user", 1)
|
||||
svc.Warn(testCtx, "warn msg", "odd_field")
|
||||
svc.Error(testCtx, "error msg")
|
||||
|
||||
svc.Debugf(testCtx, "debug %s", "formatted")
|
||||
svc.Infof(testCtx, "info %d", 100)
|
||||
svc.Warnf(testCtx, "warn %v", true)
|
||||
svc.Errorf(testCtx, "error %s", "fail")
|
||||
|
||||
withLogger := svc.With("trace", "t-123", "span", "s-456")
|
||||
require.NotNil(t, withLogger)
|
||||
withLogger.Info(testCtx, "enriched message")
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package diskcache wraps the generic pkg/cache/disk to provide database configuration integration.
|
||||
package diskcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
pkgcache "github.com/Rain-kl/Wavelet/backend/pkg/cache/disk"
|
||||
database "github.com/Rain-kl/Wavelet/backend/plugins/infra/database"
|
||||
)
|
||||
|
||||
// Status represents the runtime cache statistics.
|
||||
type Status = pkgcache.Status
|
||||
|
||||
const (
|
||||
defaultCacheDir = "uploads/diskcache"
|
||||
defaultMaxSizeMB = 100
|
||||
defaultTTLMinutes = 60
|
||||
defaultCleanupInterval = 10
|
||||
|
||||
// DefaultExpiration applies the cache-wide default TTL.
|
||||
DefaultExpiration = pkgcache.DefaultExpiration
|
||||
// NoExpiration stores the item without a TTL. Size limits and LRU eviction still apply.
|
||||
NoExpiration = pkgcache.NoExpiration
|
||||
)
|
||||
|
||||
// ErrCacheMiss represents a cache miss.
|
||||
var ErrCacheMiss = pkgcache.ErrCacheMiss
|
||||
|
||||
// DiskCache is a wrapper around the generic pkg/diskcache that integrates with the DB for configs.
|
||||
type DiskCache struct {
|
||||
*pkgcache.Cache
|
||||
}
|
||||
|
||||
var (
|
||||
globalCache *DiskCache
|
||||
globalCacheOnce sync.Once
|
||||
)
|
||||
|
||||
// GetGlobalCache returns the global singleton DiskCache instance.
|
||||
func GetGlobalCache() *DiskCache {
|
||||
globalCacheOnce.Do(func() {
|
||||
pureCache := pkgcache.New(defaultCacheDir)
|
||||
globalCache = &DiskCache{pureCache}
|
||||
// Load initial configs from database
|
||||
globalCache.ReloadConfig(context.Background())
|
||||
// Start background routine to clean expired items every 10 minutes
|
||||
go globalCache.StartCleanupWorker(defaultCleanupInterval * time.Minute)
|
||||
})
|
||||
return globalCache
|
||||
}
|
||||
|
||||
// New creates a new DiskCache wrapper.
|
||||
func New(basePath string) *DiskCache {
|
||||
return &DiskCache{pkgcache.New(basePath)}
|
||||
}
|
||||
|
||||
// ReloadConfig reloads policies from database configs dynamically.
|
||||
func (c *DiskCache) ReloadConfig(ctx context.Context) {
|
||||
// Ensure DB is initialized before querying
|
||||
if database.DB(ctx) == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Max Size
|
||||
maxSizeMB := int64(defaultMaxSizeMB)
|
||||
var maxVal string
|
||||
if err := database.DB(ctx).Table("w_system_configs").Where("key = ?", "disk_cache_max_size_mb").Pluck("value", &maxVal).Error; err == nil && maxVal != "" {
|
||||
if val, err := strconv.ParseInt(maxVal, 10, 64); err == nil && val > 0 {
|
||||
maxSizeMB = val
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Default TTL
|
||||
ttlMinutes := int64(defaultTTLMinutes)
|
||||
var ttlVal string
|
||||
if err := database.DB(ctx).Table("w_system_configs").Where("key = ?", "disk_cache_ttl_minutes").Pluck("value", &ttlVal).Error; err == nil && ttlVal != "" {
|
||||
if val, err := strconv.ParseInt(ttlVal, 10, 64); err == nil && val >= 0 {
|
||||
ttlMinutes = val
|
||||
}
|
||||
}
|
||||
|
||||
// 3. LRU Enabled
|
||||
lruEnabled := true
|
||||
var lruVal string
|
||||
if err := database.DB(ctx).Table("w_system_configs").Where("key = ?", "disk_cache_lru_enabled").Pluck("value", &lruVal).Error; err == nil && lruVal != "" {
|
||||
if val, err := strconv.ParseBool(lruVal); err == nil {
|
||||
lruEnabled = val
|
||||
}
|
||||
}
|
||||
|
||||
c.UpdatePolicy(maxSizeMB, ttlMinutes, lruEnabled)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package diskcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/testhelper"
|
||||
cache "github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
|
||||
)
|
||||
|
||||
func TestDiskCacheReloadConfig(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
testDir := "uploads/test_diskcache_reload"
|
||||
defer func() { _ = os.RemoveAll(testDir) }()
|
||||
_ = os.RemoveAll(testDir)
|
||||
|
||||
c := New(testDir)
|
||||
defer func() { _ = c.Clear() }()
|
||||
|
||||
// Update DB config values
|
||||
dbConn.Table("w_system_configs").Where("key = ?", "disk_cache_max_size_mb").Update("value", "250")
|
||||
dbConn.Table("w_system_configs").Where("key = ?", "disk_cache_ttl_minutes").Update("value", "120")
|
||||
dbConn.Table("w_system_configs").Where("key = ?", "disk_cache_lru_enabled").Update("value", "false")
|
||||
|
||||
// Invalidate Redis config cache to force DB reload
|
||||
if cache.Redis != nil {
|
||||
cache.Redis.Del(context.Background(), cache.PrefixedKey("system_configs"))
|
||||
}
|
||||
|
||||
// Reload config
|
||||
c.ReloadConfig(context.Background())
|
||||
|
||||
status := c.Status()
|
||||
if status.MaxSizeMB != 250 {
|
||||
t.Errorf("expected MaxSizeMB to be 250, got %d", status.MaxSizeMB)
|
||||
}
|
||||
if status.TTLMinutes != 120 {
|
||||
t.Errorf("expected TTLMinutes to be 120, got %d", status.TTLMinutes)
|
||||
}
|
||||
if status.LRUEnabled != false {
|
||||
t.Errorf("expected LRUEnabled to be false, got %v", status.LRUEnabled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package objectstore provides dynamically configured file storage backends.
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
database "github.com/Rain-kl/Wavelet/backend/plugins/infra/database"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Driver identifies a supported storage backend.
|
||||
type Driver string
|
||||
|
||||
const (
|
||||
// DriverLocal stores files on the local filesystem.
|
||||
DriverLocal Driver = "local"
|
||||
// DriverS3 stores files in an S3-compatible object store.
|
||||
DriverS3 Driver = "s3"
|
||||
// DriverR2 stores files in Cloudflare R2.
|
||||
DriverR2 Driver = "r2"
|
||||
// DriverMinIO stores files in MinIO.
|
||||
DriverMinIO Driver = "minio"
|
||||
// DriverOSS stores files in Aliyun OSS.
|
||||
DriverOSS Driver = "oss"
|
||||
// DriverWebDAV stores files through WebDAV.
|
||||
DriverWebDAV Driver = "webdav"
|
||||
|
||||
// ConfigMask replaces secrets returned to the frontend.
|
||||
ConfigMask = "******"
|
||||
)
|
||||
|
||||
// LocalConfig configures local filesystem storage.
|
||||
type LocalConfig struct {
|
||||
Root string `json:"root"`
|
||||
}
|
||||
|
||||
// ObjectConfig configures S3-compatible or OSS object storage.
|
||||
type ObjectConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
AccountID string `json:"account_id,omitempty"`
|
||||
PathStyle bool `json:"path_style"`
|
||||
KeyPrefix string `json:"key_prefix"`
|
||||
CDNURL string `json:"cdn_url"`
|
||||
}
|
||||
|
||||
// WebDAVConfig configures WebDAV storage.
|
||||
type WebDAVConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
BasePath string `json:"base_path"`
|
||||
}
|
||||
|
||||
// Config contains all storage backends and the currently active driver.
|
||||
type Config struct {
|
||||
Driver Driver `json:"driver"`
|
||||
Local LocalConfig `json:"local"`
|
||||
S3 ObjectConfig `json:"s3"`
|
||||
R2 ObjectConfig `json:"r2"`
|
||||
MinIO ObjectConfig `json:"minio"`
|
||||
OSS ObjectConfig `json:"oss"`
|
||||
WebDAV WebDAVConfig `json:"webdav"`
|
||||
}
|
||||
|
||||
// DefaultConfig returns the local-storage default configuration.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Driver: DriverLocal,
|
||||
Local: LocalConfig{Root: "."},
|
||||
S3: ObjectConfig{Region: "us-east-1"},
|
||||
R2: ObjectConfig{Region: "auto"},
|
||||
MinIO: ObjectConfig{Region: "us-east-1", PathStyle: true},
|
||||
}
|
||||
}
|
||||
|
||||
// LoadConfig loads the active storage configuration.
|
||||
func LoadConfig(ctx context.Context) (Config, error) {
|
||||
pubSubOnce.Do(startPubSubListener)
|
||||
|
||||
cacheMutex.RLock()
|
||||
isCacheValid := time.Since(lastChecked) < 5*time.Second && activeConfigJSON != ""
|
||||
configJSON := activeConfigJSON
|
||||
cacheMutex.RUnlock()
|
||||
|
||||
if isCacheValid {
|
||||
cfg := DefaultConfig()
|
||||
if strings.TrimSpace(configJSON) != "" {
|
||||
if err := json.Unmarshal([]byte(configJSON), &cfg); err != nil {
|
||||
return Config{}, fmt.Errorf("parse storage config from cache: %w", err)
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
return loadConfigByKey(ctx, "storage_config", DefaultConfig())
|
||||
}
|
||||
|
||||
func loadConfigByKey(ctx context.Context, key string, fallback Config) (Config, error) {
|
||||
var val string
|
||||
err := database.DB(ctx).Table("w_system_configs").Where("key = ?", key).Pluck("value", &val).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return fallback, nil
|
||||
}
|
||||
return Config{}, err
|
||||
}
|
||||
if strings.TrimSpace(val) == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
if err := json.Unmarshal([]byte(val), &fallback); err != nil {
|
||||
return Config{}, fmt.Errorf("parse %s: %w", key, err)
|
||||
}
|
||||
return fallback, nil
|
||||
}
|
||||
|
||||
// ValidateConfig validates the selected backend configuration.
|
||||
func ValidateConfig(cfg Config) error {
|
||||
switch cfg.Driver {
|
||||
case DriverLocal:
|
||||
if strings.TrimSpace(cfg.Local.Root) == "" {
|
||||
return errors.New("local root is required")
|
||||
}
|
||||
case DriverS3:
|
||||
return validateObjectConfig(cfg.S3, false)
|
||||
case DriverR2:
|
||||
if strings.TrimSpace(cfg.R2.AccountID) == "" && strings.TrimSpace(cfg.R2.Endpoint) == "" {
|
||||
return errors.New("R2 account ID or endpoint is required")
|
||||
}
|
||||
return validateObjectConfig(cfg.R2, false)
|
||||
case DriverMinIO:
|
||||
if strings.TrimSpace(cfg.MinIO.Endpoint) == "" {
|
||||
return errors.New("MinIO endpoint is required")
|
||||
}
|
||||
return validateObjectConfig(cfg.MinIO, true)
|
||||
case DriverOSS:
|
||||
if strings.TrimSpace(cfg.OSS.Endpoint) == "" {
|
||||
return errors.New("OSS endpoint is required")
|
||||
}
|
||||
return validateObjectConfig(cfg.OSS, true)
|
||||
case DriverWebDAV:
|
||||
if strings.TrimSpace(cfg.WebDAV.Endpoint) == "" {
|
||||
return errors.New("WebDAV endpoint is required")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported storage driver %q", cfg.Driver)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateObjectConfig(cfg ObjectConfig, endpointRequired bool) error {
|
||||
if endpointRequired && strings.TrimSpace(cfg.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Region) == "" {
|
||||
return errors.New("region is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bucket) == "" {
|
||||
return errors.New("bucket is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.AccessKeyID) == "" || strings.TrimSpace(cfg.SecretAccessKey) == "" {
|
||||
return errors.New("access key ID and secret access key are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveActiveConfig persists the active storage configuration.
|
||||
func SaveActiveConfig(ctx context.Context, cfg Config) error {
|
||||
return saveSystemConfig(ctx, "storage_config", cfg, "文件存储驱动及连接配置(JSON)")
|
||||
}
|
||||
|
||||
func saveSystemConfig(ctx context.Context, key string, value any, description string) error {
|
||||
err := database.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
return upsertSystemConfig(ctx, tx, key, value, description)
|
||||
})
|
||||
if err == nil && key == "storage_config" {
|
||||
ResetCache()
|
||||
PublishCacheInvalidation(ctx)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func upsertSystemConfig(_ context.Context, tx *gorm.DB, key string, value any, description string) error {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal %s: %w", key, err)
|
||||
}
|
||||
type sysConfig struct {
|
||||
Key string `gorm:"primaryKey"`
|
||||
Value string
|
||||
Type string
|
||||
Visibility string
|
||||
Description string
|
||||
}
|
||||
sc := sysConfig{
|
||||
Key: key,
|
||||
Value: string(data),
|
||||
Type: "system",
|
||||
Visibility: "hidden",
|
||||
Description: description,
|
||||
}
|
||||
return tx.Table("w_system_configs").Where("key = ?", key).
|
||||
Assign(map[string]any{"value": sc.Value, "description": description, "visibility": "hidden"}).
|
||||
FirstOrCreate(&sc).Error
|
||||
}
|
||||
|
||||
// MergeMaskedSecrets restores unchanged secrets from the current configuration.
|
||||
func MergeMaskedSecrets(next, current Config) Config {
|
||||
mergeObjectSecret := func(dst *ObjectConfig, src ObjectConfig) {
|
||||
if dst.AccessKeyID == ConfigMask {
|
||||
dst.AccessKeyID = src.AccessKeyID
|
||||
}
|
||||
if dst.SecretAccessKey == ConfigMask {
|
||||
dst.SecretAccessKey = src.SecretAccessKey
|
||||
}
|
||||
}
|
||||
mergeObjectSecret(&next.S3, current.S3)
|
||||
mergeObjectSecret(&next.R2, current.R2)
|
||||
mergeObjectSecret(&next.MinIO, current.MinIO)
|
||||
mergeObjectSecret(&next.OSS, current.OSS)
|
||||
if next.WebDAV.Password == ConfigMask {
|
||||
next.WebDAV.Password = current.WebDAV.Password
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// MaskSecrets replaces stored credentials with placeholders for API responses.
|
||||
func MaskSecrets(cfg Config) Config {
|
||||
maskObject := func(value *ObjectConfig) {
|
||||
if value.AccessKeyID != "" {
|
||||
value.AccessKeyID = ConfigMask
|
||||
}
|
||||
if value.SecretAccessKey != "" {
|
||||
value.SecretAccessKey = ConfigMask
|
||||
}
|
||||
}
|
||||
maskObject(&cfg.S3)
|
||||
maskObject(&cfg.R2)
|
||||
maskObject(&cfg.MinIO)
|
||||
maskObject(&cfg.OSS)
|
||||
if cfg.WebDAV.Password != "" {
|
||||
cfg.WebDAV.Password = ConfigMask
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/httppool"
|
||||
)
|
||||
|
||||
func getHTTPObject(ctx context.Context, baseURL, key string) (*Object, error) {
|
||||
objectURL, err := url.JoinPath(baseURL, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build CDN object URL: %w", err)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, objectURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create CDN request: %w", err)
|
||||
}
|
||||
const cdnRequestTimeout = 30 * time.Second
|
||||
client := httppool.NewClient(cdnRequestTimeout)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get CDN object: %w", err)
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
_ = response.Body.Close()
|
||||
return nil, fmt.Errorf("get CDN object: unexpected status %d", response.StatusCode)
|
||||
}
|
||||
contentType := response.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = defaultContentType
|
||||
}
|
||||
return &Object{
|
||||
Body: response.Body,
|
||||
ContentLength: response.ContentLength,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type localBackend struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func newLocalBackend(cfg LocalConfig) (*localBackend, error) {
|
||||
root := filepath.Clean(cfg.Root)
|
||||
if root == "" {
|
||||
return nil, fmt.Errorf("local root is required")
|
||||
}
|
||||
return &localBackend{root: root}, nil
|
||||
}
|
||||
|
||||
func (b *localBackend) Put(_ context.Context, key string, body io.Reader, _ int64, _ string) (PutResult, error) {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), storageDirPerm); err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
file, err := os.OpenFile( //nolint:gosec // path is constrained to the configured storage root.
|
||||
path,
|
||||
os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
|
||||
storageFilePerm,
|
||||
)
|
||||
if err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
if _, err := io.Copy(file, body); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(path)
|
||||
return PutResult{}, err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
_ = os.Remove(path)
|
||||
return PutResult{}, err
|
||||
}
|
||||
return PutResult{Key: filepath.ToSlash(key)}, nil
|
||||
}
|
||||
|
||||
func (b *localBackend) Get(_ context.Context, key string) (*Object, error) {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file, err := os.Open(path) //nolint:gosec // path is constrained to the configured storage root.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, err
|
||||
}
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
contentType = defaultContentType
|
||||
}
|
||||
return &Object{Body: file, ContentLength: info.Size(), ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *localBackend) Delete(_ context.Context, key string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Remove(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *localBackend) Test(_ context.Context) error {
|
||||
return os.MkdirAll(b.root, storageDirPerm)
|
||||
}
|
||||
|
||||
func (b *localBackend) path(key string) (string, error) {
|
||||
if filepath.IsAbs(key) {
|
||||
cleanPath := filepath.Clean(key)
|
||||
absRoot, err := filepath.Abs(b.root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
absPath, err := filepath.Abs(cleanPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, err := filepath.Rel(absRoot, absPath)
|
||||
if err != nil || strings.HasPrefix(rel, "..") {
|
||||
return "", fmt.Errorf("storage key escapes local root")
|
||||
}
|
||||
return cleanPath, nil
|
||||
}
|
||||
cleanKey := filepath.Clean(filepath.FromSlash(strings.TrimPrefix(key, "/")))
|
||||
if cleanKey == "." || cleanKey == "" || strings.HasPrefix(cleanKey, "..") {
|
||||
return "", fmt.Errorf("invalid local storage key %q", key)
|
||||
}
|
||||
path := filepath.Join(b.root, cleanKey)
|
||||
rel, err := filepath.Rel(b.root, path)
|
||||
if err != nil || strings.HasPrefix(rel, "..") {
|
||||
return "", fmt.Errorf("storage key escapes local root")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocalBackendRoundTrip(t *testing.T) {
|
||||
backend, err := newLocalBackend(LocalConfig{Root: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatalf("newLocalBackend() returned error: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
const key = "uploads/2026/06/13/test.txt"
|
||||
const content = "wavelet storage"
|
||||
|
||||
storedResult, err := backend.Put(ctx, key, bytes.NewBufferString(content), int64(len(content)), "text/plain")
|
||||
if err != nil {
|
||||
t.Fatalf("Put(%q) returned error: %v", key, err)
|
||||
}
|
||||
if storedResult.Key != key {
|
||||
t.Errorf("Put(%q) key = %q, want %q", key, storedResult.Key, key)
|
||||
}
|
||||
|
||||
object, err := backend.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%q) returned error: %v", key, err)
|
||||
}
|
||||
got, err := io.ReadAll(object.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll(Get(%q)) returned error: %v", key, err)
|
||||
}
|
||||
if err := object.Body.Close(); err != nil {
|
||||
t.Fatalf("Close(Get(%q)) returned error: %v", key, err)
|
||||
}
|
||||
if string(got) != content {
|
||||
t.Errorf("Get(%q) content = %q, want %q", key, got, content)
|
||||
}
|
||||
|
||||
if err := backend.Delete(ctx, key); err != nil {
|
||||
t.Fatalf("Delete(%q) returned error: %v", key, err)
|
||||
}
|
||||
if _, err := backend.Get(ctx, key); err == nil {
|
||||
t.Errorf("Get(%q) after Delete() returned nil error", key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
|
||||
)
|
||||
|
||||
type ossBackend struct {
|
||||
client *oss.Client
|
||||
bucket string
|
||||
keyPrefix string
|
||||
cdnURL string
|
||||
}
|
||||
|
||||
func newOSSBackend(cfg ObjectConfig) (*ossBackend, error) {
|
||||
options := oss.LoadDefaultConfig().
|
||||
WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey)).
|
||||
WithRegion(cfg.Region)
|
||||
if cfg.Endpoint != "" {
|
||||
options.WithEndpoint(cfg.Endpoint)
|
||||
}
|
||||
return &ossBackend{
|
||||
client: oss.NewClient(options),
|
||||
bucket: cfg.Bucket,
|
||||
keyPrefix: strings.Trim(cfg.KeyPrefix, "/"),
|
||||
cdnURL: strings.TrimRight(cfg.CDNURL, "/"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) (PutResult, error) {
|
||||
key = b.key(key)
|
||||
_, err := b.client.PutObject(ctx, &oss.PutObjectRequest{
|
||||
Bucket: oss.Ptr(b.bucket),
|
||||
Key: oss.Ptr(key),
|
||||
Body: body,
|
||||
})
|
||||
if err != nil {
|
||||
return PutResult{}, fmt.Errorf("put OSS object: %w", err)
|
||||
}
|
||||
return PutResult{Key: key, Bucket: b.bucket}, nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
key = b.key(key)
|
||||
if b.cdnURL != "" {
|
||||
return getHTTPObject(ctx, b.cdnURL, key)
|
||||
}
|
||||
output, err := b.client.GetObject(ctx, &oss.GetObjectRequest{
|
||||
Bucket: oss.Ptr(b.bucket),
|
||||
Key: oss.Ptr(key),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get OSS object: %w", err)
|
||||
}
|
||||
contentType := defaultContentType
|
||||
if output.ContentType != nil {
|
||||
contentType = *output.ContentType
|
||||
}
|
||||
return &Object{Body: output.Body, ContentLength: output.ContentLength, ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Delete(ctx context.Context, key string) error {
|
||||
_, err := b.client.DeleteObject(ctx, &oss.DeleteObjectRequest{
|
||||
Bucket: oss.Ptr(b.bucket),
|
||||
Key: oss.Ptr(b.key(key)),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete OSS object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Test(ctx context.Context) error {
|
||||
ok, err := b.client.IsBucketExist(ctx, b.bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("access OSS bucket: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("OSS bucket %q does not exist", b.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) key(key string) string {
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if b.keyPrefix == "" || strings.HasPrefix(key, b.keyPrefix+"/") {
|
||||
return key
|
||||
}
|
||||
return b.keyPrefix + "/" + key
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type s3Backend struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
keyPrefix string
|
||||
cdnURL string
|
||||
}
|
||||
|
||||
func newS3Backend(ctx context.Context, cfg ObjectConfig) (*s3Backend, error) {
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
|
||||
awsconfig.WithRegion(cfg.Region),
|
||||
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
cfg.AccessKeyID,
|
||||
cfg.SecretAccessKey,
|
||||
"",
|
||||
)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load S3 config: %w", err)
|
||||
}
|
||||
client := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
|
||||
if cfg.Endpoint != "" {
|
||||
options.BaseEndpoint = aws.String(strings.TrimRight(cfg.Endpoint, "/"))
|
||||
}
|
||||
options.UsePathStyle = cfg.PathStyle
|
||||
})
|
||||
return &s3Backend{
|
||||
client: client,
|
||||
bucket: cfg.Bucket,
|
||||
keyPrefix: strings.Trim(cfg.KeyPrefix, "/"),
|
||||
cdnURL: strings.TrimRight(cfg.CDNURL, "/"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newR2Backend(ctx context.Context, cfg ObjectConfig) (*s3Backend, error) {
|
||||
if cfg.Endpoint == "" {
|
||||
cfg.Endpoint = fmt.Sprintf("https://%s.r2.cloudflarestorage.com", cfg.AccountID)
|
||||
}
|
||||
cfg.Region = "auto"
|
||||
return newS3Backend(ctx, cfg)
|
||||
}
|
||||
|
||||
func (b *s3Backend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error) {
|
||||
key = b.key(key)
|
||||
_, err := b.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: body,
|
||||
ContentLength: aws.Int64(size),
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return PutResult{}, fmt.Errorf("put S3 object: %w", err)
|
||||
}
|
||||
return PutResult{Key: key, Bucket: b.bucket}, nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
key = b.key(key)
|
||||
if b.cdnURL != "" {
|
||||
return getHTTPObject(ctx, b.cdnURL, key)
|
||||
}
|
||||
output, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get S3 object: %w", err)
|
||||
}
|
||||
contentType := defaultContentType
|
||||
if output.ContentType != nil {
|
||||
contentType = *output.ContentType
|
||||
}
|
||||
var size int64
|
||||
if output.ContentLength != nil {
|
||||
size = *output.ContentLength
|
||||
}
|
||||
return &Object{Body: output.Body, ContentLength: size, ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) Delete(ctx context.Context, key string) error {
|
||||
_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(b.key(key)),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete S3 object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) Test(ctx context.Context) error {
|
||||
_, err := b.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(b.bucket)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("access S3 bucket: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) key(key string) string {
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if b.keyPrefix == "" || strings.HasPrefix(key, b.keyPrefix+"/") {
|
||||
return key
|
||||
}
|
||||
return b.keyPrefix + "/" + key
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
cache "github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
|
||||
database "github.com/Rain-kl/Wavelet/backend/plugins/infra/database"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/util"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultContentType = "application/octet-stream"
|
||||
storageDirPerm = 0o750
|
||||
storageFilePerm = 0o600
|
||||
)
|
||||
|
||||
// Object describes a readable stored object.
|
||||
type Object struct {
|
||||
CachePath string
|
||||
Body io.ReadCloser
|
||||
ContentLength int64
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// PutResult describes the result of a successful Put operation.
|
||||
type PutResult struct {
|
||||
Key string
|
||||
Bucket string
|
||||
}
|
||||
|
||||
// Backend defines storage operations used by the upload domain.
|
||||
type Backend interface {
|
||||
Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error)
|
||||
Get(ctx context.Context, key string) (*Object, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
Test(ctx context.Context) error
|
||||
}
|
||||
|
||||
var (
|
||||
// IsEnabledFunc preserves the legacy S3 test hook while tests migrate to backend injection.
|
||||
IsEnabledFunc = func() bool { return false }
|
||||
mockBackend Backend
|
||||
|
||||
activeBackend Backend
|
||||
activeDriver Driver
|
||||
activeConfigJSON string
|
||||
lastChecked time.Time
|
||||
cacheMutex sync.RWMutex
|
||||
)
|
||||
|
||||
// ConfigInvalidationChannel is the Redis pub/sub channel used to evict storage caches cluster-wide.
|
||||
const ConfigInvalidationChannel = "storage:config_invalidation"
|
||||
|
||||
var pubSubOnce sync.Once
|
||||
|
||||
// ResetCache clears the local cache for storage configuration and client singletons.
|
||||
func ResetCache() {
|
||||
cacheMutex.Lock()
|
||||
defer cacheMutex.Unlock()
|
||||
activeBackend = nil
|
||||
activeDriver = ""
|
||||
activeConfigJSON = ""
|
||||
lastChecked = time.Time{}
|
||||
}
|
||||
|
||||
// PublishCacheInvalidation broadcasts cache eviction to all nodes in the cluster via Redis.
|
||||
func PublishCacheInvalidation(ctx context.Context) {
|
||||
if cache.Redis != nil {
|
||||
_ = cache.Redis.Publish(ctx, ConfigInvalidationChannel, "reset").Err()
|
||||
}
|
||||
}
|
||||
|
||||
// startPubSubListener starts the background subscriber for cache invalidations.
|
||||
func startPubSubListener() {
|
||||
rdb := cache.Redis
|
||||
if rdb == nil {
|
||||
return
|
||||
}
|
||||
util.Go(func() {
|
||||
pubsub := rdb.Subscribe(context.Background(), ConfigInvalidationChannel)
|
||||
defer func() {
|
||||
_ = pubsub.Close()
|
||||
}()
|
||||
|
||||
ch := pubsub.Channel()
|
||||
for range ch {
|
||||
ResetCache()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Active returns the configured active driver and backend, using an in-memory cache with 5s TTL.
|
||||
func Active(ctx context.Context) (Driver, Backend, error) {
|
||||
if IsEnabledFunc() && mockBackend != nil {
|
||||
return DriverS3, mockBackend, nil
|
||||
}
|
||||
|
||||
pubSubOnce.Do(startPubSubListener)
|
||||
|
||||
cacheMutex.RLock()
|
||||
isCacheValid := time.Since(lastChecked) < 5*time.Second && activeBackend != nil
|
||||
if isCacheValid {
|
||||
d, b := activeDriver, activeBackend
|
||||
cacheMutex.RUnlock()
|
||||
return d, b, nil
|
||||
}
|
||||
cacheMutex.RUnlock()
|
||||
|
||||
cacheMutex.Lock()
|
||||
defer cacheMutex.Unlock()
|
||||
|
||||
// Double-check under write lock
|
||||
if time.Since(lastChecked) < 5*time.Second && activeBackend != nil {
|
||||
return activeDriver, activeBackend, nil
|
||||
}
|
||||
|
||||
var val string
|
||||
err := database.DB(ctx).Table("w_system_configs").Where("key = ?", "storage_config").Pluck("value", &val).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return "", nil, err
|
||||
}
|
||||
sc := struct{ Value string }{Value: val}
|
||||
|
||||
lastChecked = time.Now()
|
||||
|
||||
// Reuse existing backend client singleton if configuration JSON matches
|
||||
if sc.Value == activeConfigJSON && activeBackend != nil {
|
||||
return activeDriver, activeBackend, nil
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
if strings.TrimSpace(sc.Value) != "" {
|
||||
if err := json.Unmarshal([]byte(sc.Value), &cfg); err != nil {
|
||||
return "", nil, fmt.Errorf("parse storage config: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
backend, err := NewBackend(ctx, cfg, cfg.Driver)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
activeDriver = cfg.Driver
|
||||
activeBackend = backend
|
||||
activeConfigJSON = sc.Value
|
||||
|
||||
return activeDriver, activeBackend, nil
|
||||
}
|
||||
|
||||
type functionBackend struct {
|
||||
put func(context.Context, string, io.Reader, int64, string) error
|
||||
get func(context.Context, string) (*Object, error)
|
||||
delete func(context.Context, string) error
|
||||
}
|
||||
|
||||
func (b *functionBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error) {
|
||||
if err := b.put(ctx, key, body, size, contentType); err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
return PutResult{Key: key}, nil
|
||||
}
|
||||
|
||||
func (b *functionBackend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
return b.get(ctx, key)
|
||||
}
|
||||
|
||||
func (b *functionBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b *functionBackend) Test(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// MockStorage replaces object operations for package tests and returns a restore function.
|
||||
func MockStorage(
|
||||
put func(context.Context, string, io.Reader, int64, string) error,
|
||||
get func(context.Context, string) (*Object, error),
|
||||
deleteObject func(context.Context, string) error,
|
||||
) func() {
|
||||
previous := mockBackend
|
||||
mockBackend = &functionBackend{put: put, get: get, delete: deleteObject}
|
||||
return func() {
|
||||
mockBackend = previous
|
||||
}
|
||||
}
|
||||
|
||||
// NewBackend constructs a concrete backend from configuration.
|
||||
func NewBackend(ctx context.Context, cfg Config, driver Driver) (Backend, error) {
|
||||
if driver == DriverS3 && mockBackend != nil {
|
||||
return mockBackend, nil
|
||||
}
|
||||
switch driver {
|
||||
case DriverLocal:
|
||||
return newLocalBackend(cfg.Local)
|
||||
case DriverS3:
|
||||
return newS3Backend(ctx, cfg.S3)
|
||||
case DriverR2:
|
||||
return newR2Backend(ctx, cfg.R2)
|
||||
case DriverMinIO:
|
||||
return newS3Backend(ctx, cfg.MinIO)
|
||||
case DriverOSS:
|
||||
return newOSSBackend(cfg.OSS)
|
||||
case DriverWebDAV:
|
||||
return newWebDAVBackend(cfg.WebDAV)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage driver %q", driver)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cache "github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func TestStorageCache(t *testing.T) {
|
||||
// 1. Reset cache
|
||||
ResetCache()
|
||||
|
||||
if activeConfigJSON != "" || activeDriver != "" || activeBackend != nil || !lastChecked.IsZero() {
|
||||
t.Fatal("ResetCache did not clear cache variables")
|
||||
}
|
||||
|
||||
// 2. Set up cache manually
|
||||
expectedConfig := Config{
|
||||
Driver: DriverLocal,
|
||||
Local: LocalConfig{Root: "/tmp/wavelet-test"},
|
||||
}
|
||||
cfgJSON, err := json.Marshal(expectedConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal config failed: %v", err)
|
||||
}
|
||||
|
||||
cacheMutex.Lock()
|
||||
activeConfigJSON = string(cfgJSON)
|
||||
lastChecked = time.Now()
|
||||
cacheMutex.Unlock()
|
||||
|
||||
// 3. Call LoadConfig and verify it loads from cache (doesn't hit database, which would fail/panic because DB is not initialized)
|
||||
ctx := context.Background()
|
||||
loadedCfg, err := LoadConfig(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig failed: %v", err)
|
||||
}
|
||||
|
||||
if loadedCfg.Driver != expectedConfig.Driver || loadedCfg.Local.Root != expectedConfig.Local.Root {
|
||||
t.Errorf("Loaded config %+v, expected %+v", loadedCfg, expectedConfig)
|
||||
}
|
||||
|
||||
// 4. Test Active() returns cached driver and backend
|
||||
mockBnd := &functionBackend{
|
||||
put: func(context.Context, string, io.Reader, int64, string) error { return nil },
|
||||
get: func(context.Context, string) (*Object, error) { return nil, nil },
|
||||
delete: func(context.Context, string) error { return nil },
|
||||
}
|
||||
|
||||
cacheMutex.Lock()
|
||||
activeBackend = mockBnd
|
||||
activeDriver = DriverLocal
|
||||
cacheMutex.Unlock()
|
||||
|
||||
drv, bnd, err := Active(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Active failed: %v", err)
|
||||
}
|
||||
if drv != DriverLocal || bnd != mockBnd {
|
||||
t.Errorf("Active returned driver %v, backend %v; expected %v, %v", drv, bnd, DriverLocal, mockBnd)
|
||||
}
|
||||
|
||||
// 5. Test ResetCache again
|
||||
ResetCache()
|
||||
if activeConfigJSON != "" || activeDriver != "" || activeBackend != nil || !lastChecked.IsZero() {
|
||||
t.Fatal("ResetCache did not clear cache variables after setting them")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageCachePubSub(t *testing.T) {
|
||||
// 1. Start miniredis
|
||||
mr, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to run miniredis: %v", err)
|
||||
}
|
||||
defer mr.Close()
|
||||
|
||||
// 2. Initialize Redis client
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: mr.Addr(),
|
||||
})
|
||||
defer rdb.Close()
|
||||
|
||||
// 3. Set cache.Redis to our client
|
||||
oldRedis := cache.Redis
|
||||
cache.Redis = rdb
|
||||
defer func() {
|
||||
cache.Redis = oldRedis
|
||||
}()
|
||||
|
||||
// Reset cache and set some cached config
|
||||
ResetCache()
|
||||
cacheMutex.Lock()
|
||||
activeConfigJSON = "some_config"
|
||||
lastChecked = time.Now()
|
||||
cacheMutex.Unlock()
|
||||
|
||||
// 4. Force trigger lazy initialization of subscription
|
||||
// Reset the once guard so it runs the listener
|
||||
pubSubOnce = sync.Once{}
|
||||
ctx := context.Background()
|
||||
|
||||
// Create mock backend for Active call
|
||||
mockBnd := &functionBackend{
|
||||
put: func(context.Context, string, io.Reader, int64, string) error { return nil },
|
||||
get: func(context.Context, string) (*Object, error) { return nil, nil },
|
||||
delete: func(context.Context, string) error { return nil },
|
||||
}
|
||||
cacheMutex.Lock()
|
||||
activeBackend = mockBnd
|
||||
activeDriver = DriverLocal
|
||||
cacheMutex.Unlock()
|
||||
|
||||
_, _, _ = Active(ctx) // This calls startPubSubListener()
|
||||
|
||||
// Allow some time for subscriber connection
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// 5. Publish cache invalidation
|
||||
PublishCacheInvalidation(ctx)
|
||||
|
||||
// Allow message propagation
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// 6. Verify cache was cleared
|
||||
cacheMutex.RLock()
|
||||
configJSON := activeConfigJSON
|
||||
cacheMutex.RUnlock()
|
||||
|
||||
if configJSON != "" {
|
||||
t.Error("Memory cache was not cleared after Redis Pub/Sub broadcast")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/pkg/httppool"
|
||||
"github.com/studio-b12/gowebdav"
|
||||
)
|
||||
|
||||
type contextTransport struct {
|
||||
ctx context.Context
|
||||
parent http.RoundTripper
|
||||
}
|
||||
|
||||
func (t *contextTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return t.parent.RoundTrip(req.WithContext(t.ctx))
|
||||
}
|
||||
|
||||
type webDAVBackend struct {
|
||||
endpoint string
|
||||
username string
|
||||
password string
|
||||
basePath string
|
||||
}
|
||||
|
||||
func newWebDAVBackend(cfg WebDAVConfig) (*webDAVBackend, error) {
|
||||
return &webDAVBackend{
|
||||
endpoint: strings.TrimRight(cfg.Endpoint, "/"),
|
||||
username: cfg.Username,
|
||||
password: cfg.Password,
|
||||
basePath: strings.Trim(cfg.BasePath, "/"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) newClient(ctx context.Context) *gowebdav.Client {
|
||||
client := gowebdav.NewClient(b.endpoint, b.username, b.password)
|
||||
client.SetTransport(&contextTransport{
|
||||
ctx: ctx,
|
||||
parent: httppool.DefaultTransport(),
|
||||
})
|
||||
return client
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Put(ctx context.Context, key string, body io.Reader, size int64, _ string) (PutResult, error) {
|
||||
key = b.key(key)
|
||||
client := b.newClient(ctx)
|
||||
if dir := path.Dir(key); dir != "." && dir != "/" {
|
||||
if err := client.MkdirAll(dir, storageDirPerm); err != nil {
|
||||
return PutResult{}, fmt.Errorf("create WebDAV directory: %w", err)
|
||||
}
|
||||
}
|
||||
if err := client.WriteStreamWithLength(key, body, size, storageFilePerm); err != nil {
|
||||
return PutResult{}, fmt.Errorf("put WebDAV object: %w", err)
|
||||
}
|
||||
return PutResult{Key: key}, nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
key = b.key(key)
|
||||
client := b.newClient(ctx)
|
||||
info, err := client.Stat(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat WebDAV object: %w", err)
|
||||
}
|
||||
body, err := client.ReadStream(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get WebDAV object: %w", err)
|
||||
}
|
||||
contentType := defaultContentType
|
||||
if typed, ok := info.(interface{ ContentType() string }); ok && typed.ContentType() != "" {
|
||||
contentType = typed.ContentType()
|
||||
}
|
||||
return &Object{Body: body, ContentLength: info.Size(), ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Delete(ctx context.Context, key string) error {
|
||||
client := b.newClient(ctx)
|
||||
if err := client.Remove(b.key(key)); err != nil {
|
||||
return fmt.Errorf("delete WebDAV object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Test(ctx context.Context) error {
|
||||
client := b.newClient(ctx)
|
||||
if err := client.Connect(); err != nil {
|
||||
return fmt.Errorf("connect WebDAV: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) key(key string) string {
|
||||
return "/" + path.Join(b.basePath, strings.TrimLeft(key, "/"))
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package storage provides the object storage and ingestion infrastructure plugin for Cordis.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/ingest"
|
||||
uploadmodels "github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/models"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage/objectstore"
|
||||
)
|
||||
|
||||
// Option configures the storage plugin.
|
||||
type Option func(*Plugin)
|
||||
|
||||
// WithBackend sets an explicit storage backend instance (useful for testing or custom engines).
|
||||
func WithBackend(b objectstore.Backend) Option {
|
||||
return func(p *Plugin) {
|
||||
p.backend = b
|
||||
}
|
||||
}
|
||||
|
||||
// Plugin implements core.Plugin to provide contracts.StorageService.
|
||||
type Plugin struct {
|
||||
backend objectstore.Backend
|
||||
}
|
||||
|
||||
// New creates a new storage infrastructure plugin.
|
||||
func New(opts ...Option) *Plugin {
|
||||
p := &Plugin{}
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(p)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Name returns the unique identifier of the storage plugin.
|
||||
func (p *Plugin) Name() string {
|
||||
return "storage"
|
||||
}
|
||||
|
||||
// Apply mounts the storage service into the Context.
|
||||
func (p *Plugin) Apply(ctx *core.Context) error {
|
||||
svc := &storageServiceImpl{
|
||||
backend: p.backend,
|
||||
}
|
||||
core.Provide[contracts.StorageService](ctx, svc)
|
||||
return nil
|
||||
}
|
||||
|
||||
type storageServiceImpl struct {
|
||||
backend objectstore.Backend
|
||||
}
|
||||
|
||||
func (s *storageServiceImpl) getBackend(ctx context.Context) (objectstore.Backend, error) {
|
||||
if s.backend != nil {
|
||||
return s.backend, nil
|
||||
}
|
||||
_, b, err := objectstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: get active backend failed: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (s *storageServiceImpl) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (contracts.StoragePutResult, error) {
|
||||
b, err := s.getBackend(ctx)
|
||||
if err != nil {
|
||||
return contracts.StoragePutResult{}, err
|
||||
}
|
||||
|
||||
res, err := b.Put(ctx, key, body, size, contentType)
|
||||
if err != nil {
|
||||
return contracts.StoragePutResult{}, err
|
||||
}
|
||||
|
||||
return contracts.StoragePutResult{
|
||||
Key: res.Key,
|
||||
Bucket: res.Bucket,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *storageServiceImpl) Get(ctx context.Context, key string) (*contracts.StorageObject, error) {
|
||||
b, err := s.getBackend(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
obj, err := b.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &contracts.StorageObject{
|
||||
Key: key,
|
||||
CachePath: obj.CachePath,
|
||||
Body: obj.Body,
|
||||
ContentLength: obj.ContentLength,
|
||||
ContentType: obj.ContentType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *storageServiceImpl) Delete(ctx context.Context, key string) error {
|
||||
b, err := s.getBackend(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (s *storageServiceImpl) Ingest(ctx context.Context, reader io.Reader, opts contracts.IngestOptions) (*contracts.IngestResult, error) {
|
||||
meta := uploadmodels.UploadMetadata{
|
||||
Extra: opts.Metadata,
|
||||
}
|
||||
|
||||
req := ingest.Request{
|
||||
UserID: opts.UserID,
|
||||
Type: opts.Type,
|
||||
FileName: opts.FileName,
|
||||
MimeType: opts.MimeType,
|
||||
Extension: opts.Extension,
|
||||
Size: opts.Size,
|
||||
Reader: reader,
|
||||
Policy: ingest.Policy(opts.Policy),
|
||||
Metadata: meta,
|
||||
}
|
||||
|
||||
res, err := ingest.Ingest(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &contracts.IngestResult{
|
||||
ID: res.Upload.ID,
|
||||
Key: res.Upload.FilePath,
|
||||
Created: res.Created,
|
||||
Stored: res.Stored,
|
||||
Resolved: res.Resolved,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package storage_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/backend/core"
|
||||
"github.com/Rain-kl/Wavelet/backend/core/contracts"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage"
|
||||
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage/objectstore"
|
||||
)
|
||||
|
||||
type mockStorageBackend struct {
|
||||
mu sync.RWMutex
|
||||
files map[string][]byte
|
||||
}
|
||||
|
||||
func newMockStorageBackend() *mockStorageBackend {
|
||||
return &mockStorageBackend{files: make(map[string][]byte)}
|
||||
}
|
||||
|
||||
func (m *mockStorageBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (objectstore.PutResult, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
data, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
return objectstore.PutResult{}, err
|
||||
}
|
||||
m.files[key] = data
|
||||
return objectstore.PutResult{Key: key, Bucket: "mock-bucket"}, nil
|
||||
}
|
||||
|
||||
func (m *mockStorageBackend) Get(ctx context.Context, key string) (*objectstore.Object, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
data, ok := m.files[key]
|
||||
if !ok {
|
||||
return nil, errors.New("file not found")
|
||||
}
|
||||
return &objectstore.Object{
|
||||
Body: io.NopCloser(bytes.NewReader(data)),
|
||||
ContentLength: int64(len(data)),
|
||||
ContentType: "application/octet-stream",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *mockStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.files, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockStorageBackend) Test(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestStoragePluginOperations(t *testing.T) {
|
||||
backend := newMockStorageBackend()
|
||||
p := storage.New(storage.WithBackend(backend))
|
||||
assert.Equal(t, "storage", p.Name())
|
||||
|
||||
ctx := core.NewContext(context.Background())
|
||||
require.NoError(t, p.Apply(ctx))
|
||||
|
||||
svc, err := core.Inject[contracts.StorageService](ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, svc)
|
||||
|
||||
testCtx := context.Background()
|
||||
|
||||
// Put
|
||||
content := []byte("wavelet storage content")
|
||||
putRes, err := svc.Put(testCtx, "avatar/user1.png", bytes.NewReader(content), int64(len(content)), "image/png")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "avatar/user1.png", putRes.Key)
|
||||
assert.Equal(t, "mock-bucket", putRes.Bucket)
|
||||
|
||||
// Get
|
||||
obj, err := svc.Get(testCtx, "avatar/user1.png")
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(obj.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
|
||||
// Delete
|
||||
require.NoError(t, svc.Delete(testCtx, "avatar/user1.png"))
|
||||
_, err = svc.Get(testCtx, "avatar/user1.png")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
Reference in New Issue
Block a user