mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-07 08:06:37 +08:00
压缩历史至 95081aff
This commit is contained in:
ryan
@@ -0,0 +1,211 @@
|
||||
/*
|
||||
Copyright 2025 linux.do
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/linux-do/credit/internal/config"
|
||||
"github.com/linux-do/credit/internal/logger"
|
||||
"github.com/linux-do/credit/internal/otel_trace"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
var localCacheEnabled = false
|
||||
var localCacheDir = ""
|
||||
var cacheFilePath = "%s/%s"
|
||||
var cacheMetaFilePath = "%s/%s.meta"
|
||||
var group singleflight.Group
|
||||
|
||||
type metaInfo struct {
|
||||
ContentType string `json:"content_type"`
|
||||
ContentLength int64 `json:"content_length"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
cfg := config.Config.S3.LocalCache
|
||||
localCacheEnabled = cfg.Enabled && cfg.CacheDir != ""
|
||||
localCacheDir = strings.TrimSuffix(cfg.CacheDir, "/")
|
||||
if localCacheEnabled {
|
||||
if err := os.MkdirAll(cfg.CacheDir, 0755); err != nil {
|
||||
log.Fatalf("[Storage] failed to create local cache directory: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func GetObjectViaCache(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||
// 没有开启本地缓存
|
||||
if !localCacheEnabled {
|
||||
return GetObjectViaProxy(ctx, key)
|
||||
}
|
||||
|
||||
// 初始化 Trace
|
||||
ctx, span := otel_trace.Start(ctx, "S3.GetObjectViaCache", trace.WithSpanKind(trace.SpanKindClient))
|
||||
defer span.End()
|
||||
|
||||
// 检查本地缓存
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
localPath := fmt.Sprintf(cacheFilePath, localCacheDir, key)
|
||||
metaPath := fmt.Sprintf(cacheMetaFilePath, localCacheDir, key)
|
||||
objInfo, err := getLocalCacheFile(ctx, localPath, metaPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if objInfo != nil {
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// 使用 singleflight 确保同一时间只有一个请求会触发 CDN 获取和本地缓存保存
|
||||
_, err, _ = group.Do(key, func() (interface{}, error) {
|
||||
ctx := context.WithoutCancel(ctx)
|
||||
|
||||
// 没有缓存，通过 CDN 获取
|
||||
objInfo, err := GetObjectViaProxy(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 保存到本地
|
||||
if err := saveToLocalCache(ctx, localPath, metaPath, objInfo); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "Failed to get object via singleflight for key %s: %v", key, err)
|
||||
return nil, LocalCacheError{}
|
||||
}
|
||||
|
||||
return GetObjectViaCache(ctx, key)
|
||||
}
|
||||
|
||||
func getLocalCacheFile(ctx context.Context, localPath, metaPath string) (*ObjectInfo, error) {
|
||||
ctx, span := otel_trace.Start(ctx, "S3.GetLocalCacheFile", trace.WithSpanKind(trace.SpanKindClient))
|
||||
defer span.End()
|
||||
|
||||
// 尝试打开本地缓存文件
|
||||
file, err := os.Open(localPath)
|
||||
defer file.Close()
|
||||
|
||||
// 文件不存在
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// 判断是否为其他异常
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 读取元信息
|
||||
metaData, err := os.ReadFile(metaPath)
|
||||
|
||||
// 文件不存在
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// 判断是否为其他异常
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 解析元信息
|
||||
meta := &metaInfo{}
|
||||
if err := json.Unmarshal(metaData, meta); err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ObjectInfo{CachePath: localPath, ContentLength: meta.ContentLength, ContentType: meta.ContentType}, nil
|
||||
}
|
||||
|
||||
func saveToLocalCache(ctx context.Context, localPath, metaPath string, objInfo *ObjectInfo) error {
|
||||
ctx, span := otel_trace.Start(ctx, "S3.SaveToLocalCache", trace.WithSpanKind(trace.SpanKindClient))
|
||||
defer span.End()
|
||||
|
||||
// 创建目录
|
||||
localDir := filepath.Dir(localPath)
|
||||
if err := os.MkdirAll(localDir, 0755); err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// 创建文件
|
||||
if err := saveFile(localPath, objInfo.Body); err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// 创建元信息文件
|
||||
meta := &metaInfo{ContentType: objInfo.ContentType, ContentLength: objInfo.ContentLength}
|
||||
metaData, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
if err := saveFile(metaPath, bytes.NewReader(metaData)); err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func saveFile(localPath string, data io.Reader) error {
|
||||
// 创建临时文件
|
||||
tempFile, err := os.CreateTemp(filepath.Dir(localPath), "cache_temp_*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tempFile.Name())
|
||||
|
||||
// 将内容写入临时文件
|
||||
if _, err := tempFile.ReadFrom(data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 确保数据写入磁盘
|
||||
if err := tempFile.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 关闭临时文件
|
||||
if err := tempFile.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 将临时文件重命名为最终文件
|
||||
if err := os.Rename(tempFile.Name(), localPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
Copyright 2025 linux.do
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
type ErrS3InitializationFailed struct{}
|
||||
|
||||
func (e ErrS3InitializationFailed) Error() string {
|
||||
return "S3存储初始化失败"
|
||||
}
|
||||
|
||||
type LocalCacheError struct{}
|
||||
|
||||
func (e LocalCacheError) Error() string {
|
||||
return "本地缓存错误"
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
/*
|
||||
Copyright 2025 linux.do
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/linux-do/credit/internal/config"
|
||||
"github.com/linux-do/credit/internal/otel_trace"
|
||||
"github.com/linux-do/credit/internal/util"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var (
|
||||
client *s3.Client
|
||||
bucket string
|
||||
keyPrefix string
|
||||
cdnURL string
|
||||
)
|
||||
|
||||
func init() {
|
||||
cfg := config.Config.S3
|
||||
if !cfg.Enabled {
|
||||
log.Println("[Storage] S3 storage disabled")
|
||||
return
|
||||
}
|
||||
|
||||
bucket = cfg.Bucket
|
||||
keyPrefix = cfg.KeyPrefix
|
||||
cdnURL = strings.TrimRight(cfg.CdnURL, "/")
|
||||
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
|
||||
awsconfig.WithRegion(cfg.Region),
|
||||
awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("[Storage] failed to load AWS config: %v\n", err)
|
||||
}
|
||||
|
||||
client = s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
if cfg.Endpoint != "" {
|
||||
o.BaseEndpoint = aws.String(cfg.Endpoint)
|
||||
}
|
||||
o.UsePathStyle = cfg.PathStyle
|
||||
})
|
||||
|
||||
log.Printf("[Storage] S3 storage initialized (bucket: %s, prefix: %s, cdn: %s)\n", bucket, keyPrefix, cdnURL)
|
||||
}
|
||||
|
||||
func IsEnabled() bool {
|
||||
return client != nil
|
||||
}
|
||||
|
||||
// BuildKey constructs a full S3 object key with the configured prefix.
|
||||
func BuildKey(path string) string {
|
||||
return keyPrefix + path
|
||||
}
|
||||
|
||||
// PutObject uploads a file to S3.
|
||||
func PutObject(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
ctx, span := otel_trace.Start(ctx, "S3.PutObject", trace.WithSpanKind(trace.SpanKindClient))
|
||||
defer span.End()
|
||||
|
||||
span.SetAttributes(
|
||||
attribute.String("s3.key", key),
|
||||
attribute.Int64("s3.content_length", size),
|
||||
attribute.String("s3.content_type", contentType),
|
||||
)
|
||||
|
||||
if !IsEnabled() {
|
||||
span.SetStatus(codes.Error, "S3 not initialized")
|
||||
return ErrS3InitializationFailed{}
|
||||
}
|
||||
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
Body: body,
|
||||
ContentLength: aws.Int64(size),
|
||||
ContentType: aws.String(contentType),
|
||||
}
|
||||
|
||||
_, err := client.PutObject(ctx, input)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("S3 put object failed: %v", err))
|
||||
return fmt.Errorf("s3 put object failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObjectInfo holds metadata about a retrieved object.
|
||||
type ObjectInfo struct {
|
||||
CachePath string
|
||||
Body io.ReadCloser
|
||||
ContentLength int64
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// GetObject retrieves a file directly from S3.
|
||||
func GetObject(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||
ctx, span := otel_trace.Start(ctx, "S3.GetObject", trace.WithSpanKind(trace.SpanKindClient))
|
||||
defer span.End()
|
||||
|
||||
span.SetAttributes(attribute.String("s3.key", key))
|
||||
|
||||
if !IsEnabled() {
|
||||
span.SetStatus(codes.Error, "S3 not initialized")
|
||||
return nil, ErrS3InitializationFailed{}
|
||||
}
|
||||
|
||||
output, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("S3 get object failed: %v", err))
|
||||
return nil, fmt.Errorf("s3 get object failed: %w", err)
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
if output.ContentType != nil {
|
||||
contentType = *output.ContentType
|
||||
}
|
||||
|
||||
var contentLength int64
|
||||
if output.ContentLength != nil {
|
||||
contentLength = *output.ContentLength
|
||||
}
|
||||
|
||||
return &ObjectInfo{
|
||||
Body: output.Body,
|
||||
ContentLength: contentLength,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetObjectViaProxy retrieves a file via CDN if configured, otherwise falls back to S3.
|
||||
func GetObjectViaProxy(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||
ctx, span := otel_trace.Start(ctx, "S3.GetObjectViaProxy", trace.WithSpanKind(trace.SpanKindClient))
|
||||
defer span.End()
|
||||
|
||||
span.SetAttributes(attribute.String("s3.key", key))
|
||||
|
||||
if !IsEnabled() {
|
||||
span.SetStatus(codes.Error, "S3 not initialized")
|
||||
return nil, ErrS3InitializationFailed{}
|
||||
}
|
||||
|
||||
if cdnURL == "" {
|
||||
return GetObject(ctx, key)
|
||||
}
|
||||
|
||||
url := cdnURL + "/" + key
|
||||
span.SetAttributes(attribute.Bool("s3.use_cdn", true))
|
||||
|
||||
resp, err := util.Request(ctx, http.MethodGet, url, nil, nil, nil)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("cdn request failed: %v", err))
|
||||
return nil, fmt.Errorf("cdn request failed: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("cdn returned status %d", resp.StatusCode))
|
||||
return nil, fmt.Errorf("cdn returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
return &ObjectInfo{
|
||||
Body: resp.Body,
|
||||
ContentLength: resp.ContentLength,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeleteObject deletes a file from S3.
|
||||
func DeleteObject(ctx context.Context, key string) error {
|
||||
ctx, span := otel_trace.Start(ctx, "S3.DeleteObject", trace.WithSpanKind(trace.SpanKindClient))
|
||||
defer span.End()
|
||||
|
||||
span.SetAttributes(attribute.String("s3.key", key))
|
||||
|
||||
if !IsEnabled() {
|
||||
return ErrS3InitializationFailed{}
|
||||
}
|
||||
|
||||
_, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("S3 delete object failed: %v", err))
|
||||
return fmt.Errorf("s3 delete object failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user