mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-06 23:56:37 +08:00
缓存框架
This commit is contained in:
Vendored
+135
@@ -0,0 +1,135 @@
|
||||
// Package cache provides HTTP handlers for managing disk cache.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/diskcache"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/util"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type updateCacheConfigRequest struct {
|
||||
MaxSizeMB int64 `json:"max_size_mb" binding:"required,min=1"`
|
||||
TTLMinutes int64 `json:"ttl_minutes" binding:"required,min=0"`
|
||||
LRUEnabled bool `json:"lru_enabled"`
|
||||
}
|
||||
|
||||
// GetCacheStatus 获取磁盘缓存状态与当前统计数据
|
||||
// @Summary 获取缓存状态
|
||||
// @Description 获取当前系统磁盘缓存的使用情况(已占用字节、Key 数量等)与策略配置
|
||||
// @Tags admin
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny{data=diskcache.Status} "获取成功"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Failure 403 {object} util.ResponseAny "无管理员权限"
|
||||
// @Failure 500 {object} util.ResponseAny "内部错误"
|
||||
// @Router /api/v1/admin/cache/status [get]
|
||||
func GetCacheStatus(c *gin.Context) {
|
||||
status := diskcache.GetGlobalCache().Status()
|
||||
c.JSON(http.StatusOK, util.OK(status))
|
||||
}
|
||||
|
||||
// UpdateCacheConfig 更新磁盘缓存策略配置
|
||||
// @Summary 更新缓存配置
|
||||
// @Description 更改磁盘缓存最大容量限制、文件生存时间(TTL)以及是否启用 LRU 淘汰淘汰算法,并进行热更新
|
||||
// @Tags admin
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body cache.updateCacheConfigRequest true "缓存配置请求体"
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny "更新成功"
|
||||
// @Failure 400 {object} util.ResponseAny "参数错误"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Failure 403 {object} util.ResponseAny "无管理员权限"
|
||||
// @Failure 500 {object} util.ResponseAny "服务内部错误"
|
||||
// @Router /api/v1/admin/cache/config [post]
|
||||
func UpdateCacheConfig(c *gin.Context) {
|
||||
var req updateCacheConfigRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
ctx := c.Request.Context()
|
||||
|
||||
// Update Max Size
|
||||
if err := saveOrUpdateConfig(ctx, model.ConfigKeyDiskCacheMaxSizeMB, strconv.FormatInt(req.MaxSizeMB, 10)); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// Update Default TTL
|
||||
if err := saveOrUpdateConfig(ctx, model.ConfigKeyDiskCacheTTLMinutes, strconv.FormatInt(req.TTLMinutes, 10)); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// Update LRU Enabled
|
||||
if err := saveOrUpdateConfig(ctx, model.ConfigKeyDiskCacheLRUEnabled, strconv.FormatBool(req.LRUEnabled)); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// Trigger hot reloading in global cache
|
||||
diskcache.GetGlobalCache().ReloadConfig(ctx)
|
||||
|
||||
c.JSON(http.StatusOK, util.OKNil())
|
||||
}
|
||||
|
||||
// ClearCache 一键清空所有磁盘缓存数据
|
||||
// @Summary 清空缓存
|
||||
// @Description 清除系统磁盘缓存目录中的所有临时文件,并重置缓存容量和 Key 追踪数据
|
||||
// @Tags admin
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny "清理成功"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Failure 403 {object} util.ResponseAny "无管理员权限"
|
||||
// @Failure 500 {object} util.ResponseAny "服务内部错误"
|
||||
// @Router /api/v1/admin/cache/clear [post]
|
||||
func ClearCache(c *gin.Context) {
|
||||
if err := diskcache.GetGlobalCache().Clear(); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, util.OKNil())
|
||||
}
|
||||
|
||||
func saveOrUpdateConfig(ctx context.Context, key string, value string) error {
|
||||
var sc model.SystemConfig
|
||||
err := db.DB(ctx).Where("key = ?", key).First(&sc).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
sc = model.SystemConfig{
|
||||
Key: key,
|
||||
Value: value,
|
||||
Type: "system",
|
||||
Visibility: 0,
|
||||
}
|
||||
if err := db.DB(ctx).Create(&sc).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
sc.Value = value
|
||||
if err := db.DB(ctx).Save(&sc).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Sync config value in Redis cache
|
||||
if db.Redis != nil {
|
||||
_ = db.HSetJSON(ctx, model.SystemConfigRedisHashKey, key, &sc)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -137,9 +137,9 @@ func TestListSystemConfigs(t *testing.T) {
|
||||
var configs []model.SystemConfig
|
||||
_ = json.Unmarshal(dataBytes, &configs)
|
||||
|
||||
// Defaults seed 24 configurations
|
||||
if len(configs) != 24 {
|
||||
t.Errorf("expected 24 default configs, got %d", len(configs))
|
||||
// Defaults seed 27 configurations
|
||||
if len(configs) != 27 {
|
||||
t.Errorf("expected 27 default configs, got %d", len(configs))
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES
|
||||
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
|
||||
@@ -0,0 +1,10 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES
|
||||
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
|
||||
@@ -38,8 +38,8 @@ func TestMigrateInitializesSQLiteDatabase(t *testing.T) {
|
||||
if err := sqliteDB.Table("w_system_configs").Count(&systemConfigCount).Error; err != nil {
|
||||
t.Fatalf("Migrate() count w_system_configs error = %v", err)
|
||||
}
|
||||
if systemConfigCount != 24 {
|
||||
t.Errorf("Migrate() w_system_configs count = %d, want %d", systemConfigCount, 24)
|
||||
if systemConfigCount != 27 {
|
||||
t.Errorf("Migrate() w_system_configs count = %d, want %d", systemConfigCount, 27)
|
||||
}
|
||||
|
||||
var adminCount int64
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
// Package diskcache implements a platform-level disk-backed cache with size limit, TTL, and LRU eviction.
|
||||
package diskcache
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/peterbourgon/diskv/v3"
|
||||
)
|
||||
|
||||
// ErrCacheMiss represents a cache miss.
|
||||
var ErrCacheMiss = errors.New("cache miss")
|
||||
|
||||
// Constants for disk cache configuration and sizing
|
||||
const (
|
||||
defaultCacheDir = "uploads/diskcache"
|
||||
headerSize = 8 // 8 bytes metadata prefix for expiration UnixNano timestamp
|
||||
defaultMaxSizeMB = 100
|
||||
defaultTTLMinutes = 60
|
||||
defaultCleanupInterval = 10
|
||||
)
|
||||
|
||||
var (
|
||||
globalCache *DiskCache
|
||||
globalCacheOnce sync.Once
|
||||
)
|
||||
|
||||
// Status represents the runtime cache statistics.
|
||||
type Status struct {
|
||||
TotalSize int64 `json:"total_size"`
|
||||
KeysCount int `json:"keys_count"`
|
||||
MaxSizeMB int64 `json:"max_size_mb"`
|
||||
TTLMinutes int64 `json:"ttl_minutes"`
|
||||
LRUEnabled bool `json:"lru_enabled"`
|
||||
BasePath string `json:"base_path"`
|
||||
}
|
||||
|
||||
// DiskCache implements the disk-backed cache with size limits, TTL, and LRU eviction.
|
||||
type DiskCache struct {
|
||||
mu sync.RWMutex
|
||||
d *diskv.Diskv
|
||||
basePath string
|
||||
maxSize int64 // in bytes
|
||||
defaultTTL time.Duration
|
||||
lruEnabled bool
|
||||
|
||||
// LRU and Size tracking
|
||||
currentSize int64
|
||||
items map[string]*list.Element
|
||||
evictList *list.List
|
||||
}
|
||||
|
||||
type cacheItem struct {
|
||||
key string
|
||||
size int64
|
||||
expiredAt time.Time
|
||||
}
|
||||
|
||||
// GetGlobalCache returns the global singleton DiskCache instance.
|
||||
func GetGlobalCache() *DiskCache {
|
||||
globalCacheOnce.Do(func() {
|
||||
globalCache = New(defaultCacheDir)
|
||||
// Load initial configs from database
|
||||
globalCache.ReloadConfig(context.Background())
|
||||
// Start background routine to clean expired items every 10 minutes
|
||||
go globalCache.startCleanupWorker(defaultCleanupInterval * time.Minute)
|
||||
})
|
||||
return globalCache
|
||||
}
|
||||
|
||||
// New creates a new DiskCache instance.
|
||||
func New(basePath string) *DiskCache {
|
||||
d := diskv.New(diskv.Options{
|
||||
BasePath: basePath,
|
||||
Transform: func(_ string) []string { return []string{} }, // flat structure for easy walk
|
||||
CacheSizeMax: 1024 * 1024, // 1MB in-memory cache size for diskv itself
|
||||
})
|
||||
|
||||
c := &DiskCache{
|
||||
d: d,
|
||||
basePath: basePath,
|
||||
maxSize: defaultMaxSizeMB * 1024 * 1024, // 100MB default
|
||||
defaultTTL: defaultTTLMinutes * time.Minute, // 60 minutes default
|
||||
lruEnabled: true,
|
||||
items: make(map[string]*list.Element),
|
||||
evictList: list.New(),
|
||||
}
|
||||
|
||||
// Scan directory on startup to rebuild LRU and size tracking
|
||||
_ = c.loadTracker()
|
||||
return c
|
||||
}
|
||||
|
||||
// Set stores a key-value pair in the cache with a specific TTL.
|
||||
// If ttl <= 0, the default cache TTL is used.
|
||||
func (c *DiskCache) Set(key string, value []byte, ttl time.Duration) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if ttl <= 0 {
|
||||
ttl = c.defaultTTL
|
||||
}
|
||||
|
||||
var expiredAt time.Time
|
||||
if ttl > 0 {
|
||||
expiredAt = time.Now().Add(ttl)
|
||||
}
|
||||
|
||||
// Prepare data layout: 8 bytes expiration timestamp + raw payload
|
||||
buf := make([]byte, headerSize+len(value))
|
||||
var expNano int64
|
||||
if !expiredAt.IsZero() {
|
||||
expNano = expiredAt.UnixNano()
|
||||
}
|
||||
binary.BigEndian.PutUint64(buf[0:headerSize], uint64(expNano))
|
||||
copy(buf[headerSize:], value)
|
||||
|
||||
// Write to diskv
|
||||
if err := c.d.Write(key, buf); err != nil {
|
||||
return fmt.Errorf("failed to write key to disk: %w", err)
|
||||
}
|
||||
|
||||
// Get file size on disk (approximate)
|
||||
size := int64(len(buf))
|
||||
|
||||
// Update memory tracker
|
||||
if elem, ok := c.items[key]; ok {
|
||||
item := elem.Value.(*cacheItem)
|
||||
c.currentSize += size - item.size
|
||||
item.size = size
|
||||
item.expiredAt = expiredAt
|
||||
c.evictList.MoveToFront(elem)
|
||||
} else {
|
||||
item := &cacheItem{
|
||||
key: key,
|
||||
size: size,
|
||||
expiredAt: expiredAt,
|
||||
}
|
||||
elem := c.evictList.PushFront(item)
|
||||
c.items[key] = elem
|
||||
c.currentSize += size
|
||||
}
|
||||
|
||||
// Evict items if size limit exceeded and LRU is enabled
|
||||
c.evict()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get retrieves a key's value from the cache.
|
||||
func (c *DiskCache) Get(key string) ([]byte, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
elem, ok := c.items[key]
|
||||
if !ok {
|
||||
return nil, ErrCacheMiss
|
||||
}
|
||||
|
||||
item := elem.Value.(*cacheItem)
|
||||
|
||||
// Check expiration
|
||||
if !item.expiredAt.IsZero() && time.Now().After(item.expiredAt) {
|
||||
// Lazily delete expired item
|
||||
_ = c.deleteUnlocked(key)
|
||||
return nil, ErrCacheMiss
|
||||
}
|
||||
|
||||
// Read from diskv
|
||||
data, err := c.d.Read(key)
|
||||
if err != nil {
|
||||
// Key exists in memory but not on disk, sync state
|
||||
_ = c.deleteUnlocked(key)
|
||||
return nil, ErrCacheMiss
|
||||
}
|
||||
|
||||
if len(data) < headerSize {
|
||||
_ = c.deleteUnlocked(key)
|
||||
return nil, ErrCacheMiss
|
||||
}
|
||||
|
||||
// Update LRU access order
|
||||
c.evictList.MoveToFront(elem)
|
||||
|
||||
// Slice off the metadata header
|
||||
return data[headerSize:], nil
|
||||
}
|
||||
|
||||
// Delete removes a key-value pair from the cache.
|
||||
func (c *DiskCache) Delete(key string) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.deleteUnlocked(key)
|
||||
}
|
||||
|
||||
func (c *DiskCache) deleteUnlocked(key string) error {
|
||||
if elem, ok := c.items[key]; ok {
|
||||
item := elem.Value.(*cacheItem)
|
||||
c.currentSize -= item.size
|
||||
c.evictList.Remove(elem)
|
||||
delete(c.items, key)
|
||||
}
|
||||
return c.d.Erase(key)
|
||||
}
|
||||
|
||||
// Clear flushes all cached elements.
|
||||
func (c *DiskCache) Clear() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.currentSize = 0
|
||||
c.items = make(map[string]*list.Element)
|
||||
c.evictList.Init()
|
||||
|
||||
return c.d.EraseAll()
|
||||
}
|
||||
|
||||
// Status returns the cache status.
|
||||
func (c *DiskCache) Status() Status {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
return Status{
|
||||
TotalSize: c.currentSize,
|
||||
KeysCount: len(c.items),
|
||||
MaxSizeMB: c.maxSize / (1024 * 1024),
|
||||
TTLMinutes: int64(c.defaultTTL.Minutes()),
|
||||
LRUEnabled: c.lruEnabled,
|
||||
BasePath: c.basePath,
|
||||
}
|
||||
}
|
||||
|
||||
// ReloadConfig reloads policies from database configs dynamically.
|
||||
func (c *DiskCache) ReloadConfig(ctx context.Context) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Ensure DB is initialized before querying
|
||||
if db.DB(ctx) == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Max Size
|
||||
var scMaxSize model.SystemConfig
|
||||
maxSizeMB := int64(defaultMaxSizeMB)
|
||||
if err := scMaxSize.GetByKey(ctx, model.ConfigKeyDiskCacheMaxSizeMB); err == nil && scMaxSize.Value != "" {
|
||||
if val, err := strconv.ParseInt(scMaxSize.Value, 10, 64); err == nil && val > 0 {
|
||||
maxSizeMB = val
|
||||
}
|
||||
}
|
||||
c.maxSize = maxSizeMB * 1024 * 1024
|
||||
|
||||
// 2. Default TTL
|
||||
var scTTL model.SystemConfig
|
||||
ttlMinutes := int64(defaultTTLMinutes)
|
||||
if err := scTTL.GetByKey(ctx, model.ConfigKeyDiskCacheTTLMinutes); err == nil && scTTL.Value != "" {
|
||||
if val, err := strconv.ParseInt(scTTL.Value, 10, 64); err == nil && val >= 0 {
|
||||
ttlMinutes = val
|
||||
}
|
||||
}
|
||||
c.defaultTTL = time.Duration(ttlMinutes) * time.Minute
|
||||
|
||||
// 3. LRU Enabled
|
||||
var scLRU model.SystemConfig
|
||||
lruEnabled := true
|
||||
if err := scLRU.GetByKey(ctx, model.ConfigKeyDiskCacheLRUEnabled); err == nil && scLRU.Value != "" {
|
||||
if val, err := strconv.ParseBool(scLRU.Value); err == nil {
|
||||
lruEnabled = val
|
||||
}
|
||||
}
|
||||
c.lruEnabled = lruEnabled
|
||||
|
||||
// Apply eviction immediately under new configs
|
||||
c.evict()
|
||||
}
|
||||
|
||||
// evict evicts oldest items if current size exceeds maxSize and LRU is enabled.
|
||||
func (c *DiskCache) evict() {
|
||||
if !c.lruEnabled {
|
||||
return
|
||||
}
|
||||
|
||||
for c.currentSize > c.maxSize && c.evictList.Len() > 0 {
|
||||
elem := c.evictList.Back()
|
||||
if elem == nil {
|
||||
break
|
||||
}
|
||||
item := elem.Value.(*cacheItem)
|
||||
key := item.key
|
||||
|
||||
// Remove from memory
|
||||
c.currentSize -= item.size
|
||||
c.evictList.Remove(elem)
|
||||
delete(c.items, key)
|
||||
|
||||
// Delete from disk
|
||||
_ = c.d.Erase(key)
|
||||
}
|
||||
}
|
||||
|
||||
// loadTracker walks the directory to rebuild the LRU and size tracking structures.
|
||||
func (c *DiskCache) loadTracker() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.currentSize = 0
|
||||
c.items = make(map[string]*list.Element)
|
||||
c.evictList = list.New()
|
||||
|
||||
files, err := os.ReadDir(c.basePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type tempItem struct {
|
||||
key string
|
||||
size int64
|
||||
expiredAt time.Time
|
||||
mtime time.Time
|
||||
}
|
||||
var loadedItems []tempItem
|
||||
|
||||
for _, file := range files {
|
||||
if file.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := file.Name()
|
||||
// Skip temporary files
|
||||
if strings.HasPrefix(name, ".") || strings.Contains(name, "temp") {
|
||||
continue
|
||||
}
|
||||
|
||||
info, err := file.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
filePath := filepath.Join(c.basePath, name)
|
||||
// #nosec G304
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var expiredAt time.Time
|
||||
var expNano int64
|
||||
err = binary.Read(f, binary.BigEndian, &expNano)
|
||||
_ = f.Close()
|
||||
if err != nil {
|
||||
// Corrupt metadata header: delete file
|
||||
_ = os.Remove(filePath)
|
||||
continue
|
||||
}
|
||||
|
||||
if expNano > 0 {
|
||||
expiredAt = time.Unix(0, expNano)
|
||||
// Expired: delete file
|
||||
if time.Now().After(expiredAt) {
|
||||
_ = os.Remove(filePath)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
loadedItems = append(loadedItems, tempItem{
|
||||
key: name,
|
||||
size: info.Size(),
|
||||
expiredAt: expiredAt,
|
||||
mtime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
|
||||
// Sort loaded items by modification time ascending (oldest first)
|
||||
sort.Slice(loadedItems, func(i, j int) bool {
|
||||
return loadedItems[i].mtime.Before(loadedItems[j].mtime)
|
||||
})
|
||||
|
||||
// Populate LRU (PushFront so that newest items are at the front, oldest at the back)
|
||||
for _, item := range loadedItems {
|
||||
entry := &cacheItem{
|
||||
key: item.key,
|
||||
size: item.size,
|
||||
expiredAt: item.expiredAt,
|
||||
}
|
||||
element := c.evictList.PushFront(entry)
|
||||
c.items[item.key] = element
|
||||
c.currentSize += item.size
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// startCleanupWorker periodically cleans up expired cache items.
|
||||
func (c *DiskCache) startCleanupWorker(interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
for range ticker.C {
|
||||
c.cleanExpired()
|
||||
}
|
||||
}
|
||||
|
||||
// cleanExpired scans memory for expired items and removes them.
|
||||
func (c *DiskCache) cleanExpired() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
for key, elem := range c.items {
|
||||
item := elem.Value.(*cacheItem)
|
||||
if !item.expiredAt.IsZero() && now.After(item.expiredAt) {
|
||||
c.currentSize -= item.size
|
||||
c.evictList.Remove(elem)
|
||||
delete(c.items, key)
|
||||
_ = c.d.Erase(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package diskcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
)
|
||||
|
||||
func TestDiskCacheBasic(t *testing.T) {
|
||||
testDir := "uploads/test_diskcache_basic"
|
||||
defer func() { _ = os.RemoveAll(testDir) }()
|
||||
_ = os.RemoveAll(testDir)
|
||||
|
||||
c := New(testDir)
|
||||
defer func() { _ = c.Clear() }()
|
||||
|
||||
key := "key1"
|
||||
val := []byte("value1")
|
||||
|
||||
// Get non-existent
|
||||
_, err := c.Get(key)
|
||||
if err != ErrCacheMiss {
|
||||
t.Fatalf("expected ErrCacheMiss, got %v", err)
|
||||
}
|
||||
|
||||
// Set & Get
|
||||
err = c.Set(key, val, 10*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set cache: %v", err)
|
||||
}
|
||||
|
||||
got, err := c.Get(key)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get cache: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(got, val) {
|
||||
t.Errorf("expected %s, got %s", val, got)
|
||||
}
|
||||
|
||||
// Delete
|
||||
err = c.Delete(key)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to delete: %v", err)
|
||||
}
|
||||
|
||||
_, err = c.Get(key)
|
||||
if err != ErrCacheMiss {
|
||||
t.Errorf("expected ErrCacheMiss after delete, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheTTL(t *testing.T) {
|
||||
testDir := "uploads/test_diskcache_ttl"
|
||||
defer func() { _ = os.RemoveAll(testDir) }()
|
||||
_ = os.RemoveAll(testDir)
|
||||
|
||||
c := New(testDir)
|
||||
defer func() { _ = c.Clear() }()
|
||||
|
||||
key := "ttlkey"
|
||||
val := []byte("ttlval")
|
||||
|
||||
// Set with 200ms TTL
|
||||
err := c.Set(key, val, 200*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set: %v", err)
|
||||
}
|
||||
|
||||
// Immediate Get should succeed
|
||||
got, err := c.Get(key)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, val) {
|
||||
t.Errorf("expected %s, got %s", val, got)
|
||||
}
|
||||
|
||||
// Sleep 250ms to expire
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
|
||||
// Get should fail with cache miss
|
||||
_, err = c.Get(key)
|
||||
if err != ErrCacheMiss {
|
||||
t.Errorf("expected ErrCacheMiss after TTL expiration, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheLRUEviction(t *testing.T) {
|
||||
testDir := "uploads/test_diskcache_lru"
|
||||
defer func() { _ = os.RemoveAll(testDir) }()
|
||||
_ = os.RemoveAll(testDir)
|
||||
|
||||
c := New(testDir)
|
||||
defer func() { _ = c.Clear() }()
|
||||
|
||||
// Force a very small max size of 20 bytes for testing (8 bytes header + payload)
|
||||
// So 2 items of 2 bytes payload = 2 * (8 + 2) = 20 bytes max.
|
||||
c.maxSize = 20
|
||||
c.lruEnabled = true
|
||||
|
||||
// Write item 1: 8 + 2 = 10 bytes
|
||||
err := c.Set("k1", []byte("v1"), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set k1: %v", err)
|
||||
}
|
||||
|
||||
// Write item 2: 8 + 2 = 10 bytes
|
||||
err = c.Set("k2", []byte("v2"), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set k2: %v", err)
|
||||
}
|
||||
|
||||
// Both should exist
|
||||
if _, err := c.Get("k1"); err != nil {
|
||||
t.Errorf("k1 should exist: %v", err)
|
||||
}
|
||||
if _, err := c.Get("k2"); err != nil {
|
||||
t.Errorf("k2 should exist: %v", err)
|
||||
}
|
||||
|
||||
// Write item 3: 8 + 2 = 10 bytes -> total size would be 30, exceeding 20.
|
||||
// This should evict the oldest item. Since k1 was accessed, but then k2 was accessed,
|
||||
// wait, let's access k1 again to make it the most recently used, so k2 becomes oldest!
|
||||
_, _ = c.Get("k1") // k1 is now MRU, k2 is LRU
|
||||
|
||||
err = c.Set("k3", []byte("v3"), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set k3: %v", err)
|
||||
}
|
||||
|
||||
// k2 should be evicted, k1 and k3 should exist
|
||||
_, err = c.Get("k2")
|
||||
if err != ErrCacheMiss {
|
||||
t.Errorf("expected k2 to be evicted, got error %v", err)
|
||||
}
|
||||
|
||||
if _, err := c.Get("k1"); err != nil {
|
||||
t.Errorf("k1 should still exist: %v", err)
|
||||
}
|
||||
|
||||
if _, err := c.Get("k3"); err != nil {
|
||||
t.Errorf("k3 should exist: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheReloadConfig(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
testDir := "uploads/test_diskcache_reload"
|
||||
defer func() { _ = os.RemoveAll(testDir) }()
|
||||
_ = os.RemoveAll(testDir)
|
||||
|
||||
c := New(testDir)
|
||||
defer func() { _ = c.Clear() }()
|
||||
|
||||
// Update DB config values
|
||||
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheMaxSizeMB).Update("value", "250")
|
||||
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheTTLMinutes).Update("value", "120")
|
||||
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheLRUEnabled).Update("value", "false")
|
||||
|
||||
// Invalidate Redis config cache to force DB reload
|
||||
if db.Redis != nil {
|
||||
db.Redis.Del(context.Background(), db.PrefixedKey(model.SystemConfigRedisHashKey))
|
||||
}
|
||||
|
||||
// Reload config
|
||||
c.ReloadConfig(context.Background())
|
||||
|
||||
status := c.Status()
|
||||
if status.MaxSizeMB != 250 {
|
||||
t.Errorf("expected MaxSizeMB to be 250, got %d", status.MaxSizeMB)
|
||||
}
|
||||
if status.TTLMinutes != 120 {
|
||||
t.Errorf("expected TTLMinutes to be 120, got %d", status.TTLMinutes)
|
||||
}
|
||||
if status.LRUEnabled != false {
|
||||
t.Errorf("expected LRUEnabled to be false, got %t", status.LRUEnabled)
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,9 @@ const (
|
||||
ConfigKeyMenuDisplayConfig = "menu_display_config" // 目录显示配置 (JSON 字符串)
|
||||
ConfigKeySearchEngineIndexingEnabled = "search_engine_indexing_enabled" // 是否允许搜索引擎检索
|
||||
ConfigKeyFileAccessWhitelist = "file_access_whitelist" // 免登录访问的文件业务类型白名单 (JSON 数组格式)
|
||||
ConfigKeyDiskCacheMaxSizeMB = "disk_cache_max_size_mb" // 磁盘缓存最大空间大小 (MB)
|
||||
ConfigKeyDiskCacheTTLMinutes = "disk_cache_ttl_minutes" // 磁盘缓存默认有效期 (分钟)
|
||||
ConfigKeyDiskCacheLRUEnabled = "disk_cache_lru_enabled" // 是否启用 LRU 淘汰机制
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/admin"
|
||||
admin_auth_source "github.com/Rain-kl/Wavelet/internal/apps/admin/auth_source"
|
||||
admin_cache "github.com/Rain-kl/Wavelet/internal/apps/admin/cache"
|
||||
admin_db_manage "github.com/Rain-kl/Wavelet/internal/apps/admin/db_manage"
|
||||
admin_logs "github.com/Rain-kl/Wavelet/internal/apps/admin/logs"
|
||||
admin_status "github.com/Rain-kl/Wavelet/internal/apps/admin/status"
|
||||
@@ -229,6 +230,11 @@ func registerRoutes(r *gin.Engine) {
|
||||
adminRouter.GET("/db-manage/table-data", admin_db_manage.GetDBTableData)
|
||||
adminRouter.POST("/db-manage/query", admin_db_manage.ExecuteSQL)
|
||||
|
||||
// Cache management
|
||||
adminRouter.GET("/cache/status", admin_cache.GetCacheStatus)
|
||||
adminRouter.POST("/cache/config", admin_cache.UpdateCacheConfig)
|
||||
adminRouter.POST("/cache/clear", admin_cache.ClearCache)
|
||||
|
||||
// System logs
|
||||
adminRouter.GET("/logs", admin_logs.GetLogs)
|
||||
adminRouter.GET("/logs/access", admin_logs.GetAccessLogs)
|
||||
|
||||
@@ -74,8 +74,8 @@ func SetupTestEnvironment(t *testing.T) (*gorm.DB, *miniredis.Miniredis, func())
|
||||
return sqliteDB, mr, cleanup
|
||||
}
|
||||
|
||||
func seedDefaultConfigs(t *testing.T, tx *gorm.DB) {
|
||||
defaultConfigs := []model.SystemConfig{
|
||||
func getSeedConfigsPart1() []model.SystemConfig {
|
||||
return []model.SystemConfig{
|
||||
{
|
||||
Key: model.ConfigKeyUploadAllowedExtensions,
|
||||
Value: "jpg,png,webp",
|
||||
@@ -160,6 +160,11 @@ func seedDefaultConfigs(t *testing.T, tx *gorm.DB) {
|
||||
Type: "system",
|
||||
Description: "人机验证兑换凭证有效时间(秒)",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getSeedConfigsPart2() []model.SystemConfig {
|
||||
return []model.SystemConfig{
|
||||
{
|
||||
Key: model.ConfigKeyServerAddress,
|
||||
Value: "",
|
||||
@@ -220,7 +225,29 @@ func seedDefaultConfigs(t *testing.T, tx *gorm.DB) {
|
||||
Type: "system",
|
||||
Description: "免登录访问的文件业务类型白名单",
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyDiskCacheMaxSizeMB,
|
||||
Value: "100",
|
||||
Type: "system",
|
||||
Description: "磁盘缓存最大空间大小 (MB)",
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyDiskCacheTTLMinutes,
|
||||
Value: "60",
|
||||
Type: "system",
|
||||
Description: "磁盘缓存默认有效期 (分钟)",
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyDiskCacheLRUEnabled,
|
||||
Value: "true",
|
||||
Type: "system",
|
||||
Description: "是否启用 LRU 淘汰机制",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func seedDefaultConfigs(t *testing.T, tx *gorm.DB) {
|
||||
defaultConfigs := append(getSeedConfigsPart1(), getSeedConfigsPart2()...)
|
||||
|
||||
if err := tx.Create(&defaultConfigs).Error; err != nil {
|
||||
t.Fatalf("failed to seed default system configs: %v", err)
|
||||
|
||||
Reference in New Issue
Block a user