mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-16 12:54:22 +08:00
fix media event access, cache isolation and indexing
This commit is contained in:
@@ -178,14 +178,6 @@ func (r *MediaRepository) YearRange(ctx context.Context, filter MediaQueryFilter
|
||||
return min, max, nil
|
||||
}
|
||||
|
||||
func (r *MediaRepository) indexMediaBestEffort(ctx context.Context, media model.Media) {
|
||||
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
_ = backend.IndexMedia(ctx, []model.Media{media})
|
||||
}
|
||||
|
||||
// FindByID returns the media row or (nil, nil).
|
||||
func (r *MediaRepository) FindByID(ctx context.Context, id string) (*model.Media, error) {
|
||||
var m model.Media
|
||||
|
||||
@@ -96,15 +96,34 @@ func (r *MediaRepository) UpsertBatchWithAliases(ctx context.Context, items []Me
|
||||
|
||||
// indexByIDBestEffort 在事务提交后按 ID 回读最新行并同步搜索索引。
|
||||
func (r *MediaRepository) indexByIDBestEffort(ctx context.Context, ids []string) {
|
||||
if _, ok := r.searchBackend.(MediaSearchSyncBackend); !ok {
|
||||
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
unique := make([]string, 0, len(ids))
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if fresh, err := r.FindByID(ctx, id); err == nil && fresh != nil {
|
||||
r.indexMediaBestEffort(ctx, *fresh)
|
||||
if _, exists := seen[id]; !exists {
|
||||
seen[id] = struct{}{}
|
||||
unique = append(unique, id)
|
||||
}
|
||||
}
|
||||
// Read committed rows and submit one bulk per bounded batch. Keep network
|
||||
// work outside the transaction and below SQLite's parameter limit.
|
||||
const batchSize = 500
|
||||
for start := 0; start < len(unique); start += batchSize {
|
||||
batch := unique[start:min(start+batchSize, len(unique))]
|
||||
rows, err := r.ListMediaByIDs(ctx, batch)
|
||||
if err == nil {
|
||||
err = backend.IndexMedia(ctx, rows)
|
||||
}
|
||||
if err != nil {
|
||||
// The primary write has committed; do not report a failed upsert and
|
||||
// make scanners repeat it. Surface the separate index failure instead.
|
||||
r.db.Logger.Error(ctx, "media saved but search index sync failed for %d items: %v", len(batch), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -135,14 +137,113 @@ func (b *OpenSearchMediaBackend) EnsureIndex(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Media) error {
|
||||
const batchSize = 500
|
||||
var failures []error
|
||||
for start := 0; start < len(rows); start += batchSize {
|
||||
if err := b.indexMediaBatch(ctx, rows[start:min(start+batchSize, len(rows))]); err != nil {
|
||||
failures = append(failures, err)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
|
||||
type openSearchBulkResponse struct {
|
||||
Errors bool `json:"errors"`
|
||||
Items []struct {
|
||||
Index struct {
|
||||
ID string `json:"_id"`
|
||||
Status int `json:"status"`
|
||||
Error *struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"error"`
|
||||
} `json:"index"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) indexMediaBatch(ctx context.Context, rows []model.Media) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
pending := rows
|
||||
var failures []error
|
||||
const attempts = 3
|
||||
for attempt := 0; attempt < attempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return errors.Join(append(failures, err)...)
|
||||
}
|
||||
if attempt > 0 {
|
||||
timer := time.NewTimer(time.Duration(100*(1<<(attempt-1))) * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return errors.Join(append(failures, ctx.Err())...)
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
response, err := b.submitMediaBulk(ctx, pending)
|
||||
if err != nil {
|
||||
if retryableOpenSearchError(err) && attempt < attempts-1 {
|
||||
continue
|
||||
}
|
||||
return errors.Join(append(failures, err)...)
|
||||
}
|
||||
if len(response.Items) != len(pending) {
|
||||
return errors.Join(append(failures, fmt.Errorf("opensearch bulk returned %d item results for %d documents", len(response.Items), len(pending)))...)
|
||||
}
|
||||
var retry []model.Media
|
||||
failed := false
|
||||
for i, item := range response.Items {
|
||||
result := item.Index
|
||||
if result.Status >= 200 && result.Status < 300 && result.Error == nil {
|
||||
continue
|
||||
}
|
||||
failed = true
|
||||
if retryableOpenSearchStatus(result.Status) && attempt < attempts-1 {
|
||||
retry = append(retry, pending[i])
|
||||
continue
|
||||
}
|
||||
errorType := "unknown"
|
||||
if result.Error != nil {
|
||||
errorType = result.Error.Type
|
||||
}
|
||||
failures = append(failures, fmt.Errorf("opensearch bulk document %q failed: status %d (%s)", pending[i].ID, result.Status, errorType))
|
||||
}
|
||||
if response.Errors && !failed {
|
||||
failures = append(failures, errors.New("opensearch bulk reported errors without failed item results"))
|
||||
}
|
||||
if len(retry) == 0 {
|
||||
break
|
||||
}
|
||||
pending = retry
|
||||
}
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
|
||||
func retryableOpenSearchStatus(status int) bool {
|
||||
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
|
||||
}
|
||||
|
||||
func retryableOpenSearchError(err error) bool {
|
||||
var statusError *openSearchHTTPError
|
||||
if errors.As(err, &statusError) {
|
||||
return retryableOpenSearchStatus(statusError.status)
|
||||
}
|
||||
var networkError net.Error
|
||||
return errors.As(err, &networkError)
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) submitMediaBulk(ctx context.Context, rows []model.Media) (*openSearchBulkResponse, error) {
|
||||
if len(rows) == 0 {
|
||||
return &openSearchBulkResponse{}, nil
|
||||
}
|
||||
var bulk bytes.Buffer
|
||||
enc := json.NewEncoder(&bulk)
|
||||
for _, row := range rows {
|
||||
if err := enc.Encode(map[string]any{"index": map[string]any{"_index": b.index, "_id": row.ID}}); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if err := enc.Encode(map[string]any{
|
||||
"id": row.ID,
|
||||
@@ -155,10 +256,24 @@ func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Me
|
||||
"deleted": row.DeletedAt.Valid,
|
||||
"created_at": row.CreatedAt,
|
||||
}); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", nil)
|
||||
var response openSearchBulkResponse
|
||||
if err := b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", &response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
type openSearchHTTPError struct {
|
||||
method string
|
||||
path string
|
||||
status int
|
||||
}
|
||||
|
||||
func (e *openSearchHTTPError) Error() string {
|
||||
return fmt.Sprintf("opensearch %s %s returned %d", e.method, e.path, e.status)
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) doJSON(ctx context.Context, method, path string, body any, out any) error {
|
||||
@@ -190,7 +305,7 @@ func (b *OpenSearchMediaBackend) do(ctx context.Context, method, path string, bo
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("opensearch %s %s returned %d", method, path, resp.StatusCode)
|
||||
return &openSearchHTTPError{method: method, path: path, status: resp.StatusCode}
|
||||
}
|
||||
if out == nil {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
@@ -1,12 +1,23 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
|
||||
@@ -55,3 +66,156 @@ func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
|
||||
t.Fatalf("ids=%#v total=%d", ids, total)
|
||||
}
|
||||
}
|
||||
|
||||
func bulkRequestIDs(r *http.Request) []string {
|
||||
var ids []string
|
||||
scanner := bufio.NewScanner(r.Body)
|
||||
for scanner.Scan() {
|
||||
var action struct {
|
||||
Index struct {
|
||||
ID string `json:"_id"`
|
||||
} `json:"index"`
|
||||
}
|
||||
_ = json.Unmarshal(scanner.Bytes(), &action)
|
||||
ids = append(ids, action.Index.ID)
|
||||
if !scanner.Scan() {
|
||||
break
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func writeBulkResults(w http.ResponseWriter, ids []string, statuses map[string]int) {
|
||||
items := make([]any, 0, len(ids))
|
||||
failed := false
|
||||
for _, id := range ids {
|
||||
status := statuses[id]
|
||||
if status == 0 {
|
||||
status = http.StatusCreated
|
||||
}
|
||||
result := map[string]any{"_id": id, "status": status}
|
||||
if status >= 400 {
|
||||
failed = true
|
||||
result["error"] = map[string]any{"type": "test_failure"}
|
||||
}
|
||||
items = append(items, map[string]any{"index": result})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"errors": failed, "items": items})
|
||||
}
|
||||
|
||||
func TestOpenSearchBulkRetriesOnlyTransientFailedDocuments(t *testing.T) {
|
||||
var requests [][]string
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ids := bulkRequestIDs(r)
|
||||
requests = append(requests, ids)
|
||||
statuses := map[string]int{}
|
||||
if len(requests) == 1 {
|
||||
statuses["retry"] = http.StatusTooManyRequests
|
||||
statuses["invalid"] = http.StatusBadRequest
|
||||
}
|
||||
writeBulkResults(w, ids, statuses)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
|
||||
rows := []model.Media{{Base: model.Base{ID: "ok"}}, {Base: model.Base{ID: "retry"}}, {Base: model.Base{ID: "invalid"}}}
|
||||
err := backend.IndexMedia(t.Context(), rows)
|
||||
if err == nil || !strings.Contains(err.Error(), `"invalid"`) {
|
||||
t.Fatalf("error = %v, want permanent document failure", err)
|
||||
}
|
||||
if len(requests) != 2 || strings.Join(requests[1], ",") != "retry" {
|
||||
t.Fatalf("requests = %#v; successful/permanent documents must not be retried", requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSearchBulkBoundsRetriesAndRejectsMalformedResponses(t *testing.T) {
|
||||
for _, scenario := range []string{"transient-item", "transient-http", "invalid-http", "missing-results"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
calls := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
switch scenario {
|
||||
case "transient-item":
|
||||
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusServiceUnavailable})
|
||||
case "transient-http":
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
case "invalid-http":
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
case "missing-results":
|
||||
fmt.Fprint(w, `{"errors":false,"items":[]}`)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
|
||||
if err := backend.IndexMedia(t.Context(), []model.Media{{Base: model.Base{ID: "m1"}}}); err == nil {
|
||||
t.Fatal("failed indexing was reported as successful")
|
||||
}
|
||||
want := 1
|
||||
if strings.HasPrefix(scenario, "transient") {
|
||||
want = 3
|
||||
}
|
||||
if calls != want {
|
||||
t.Fatalf("calls = %d, want %d", calls, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSearchBulkHonorsCancellationDuringRetry(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusTooManyRequests})
|
||||
cancel()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
|
||||
err := backend.IndexMedia(ctx, []model.Media{{Base: model.Base{ID: "m1"}}})
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("error = %v, calls = %d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaUpsertBatchIndexesCommittedRowsTogether(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
defer sqlDB.Close()
|
||||
if err := db.AutoMigrate(&model.Media{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := New(db)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
calls := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
ids := bulkRequestIDs(r)
|
||||
if len(ids) != 5 {
|
||||
t.Errorf("bulk contains %d documents, want 5", len(ids))
|
||||
}
|
||||
// With only one DB connection this also detects network work inside the
|
||||
// transaction: a separate read could not observe the committed batch.
|
||||
var count int64
|
||||
if err := db.WithContext(ctx).Model(&model.Media{}).Count(&count).Error; err != nil || count != 5 {
|
||||
t.Errorf("committed rows = %d, error = %v", count, err)
|
||||
}
|
||||
writeBulkResults(w, ids, nil)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
repos.Media.SetSearchBackend(NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL}))
|
||||
var rows []*model.Media
|
||||
for i := 0; i < 5; i++ {
|
||||
rows = append(rows, &model.Media{LibraryID: "lib", Path: fmt.Sprintf("/media/%d.mkv", i)})
|
||||
}
|
||||
if err := repos.Media.UpsertBatch(ctx, rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("index requests = %d, want one bulk", calls)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user