fix media event access, cache isolation and indexing

This commit is contained in:
jason jia
2026-10-20 12:39:13 +00:00
parent 0b68397265
commit 3cf898756b
18 changed files with 926 additions and 141 deletions
-8
View File
@@ -178,14 +178,6 @@ func (r *MediaRepository) YearRange(ctx context.Context, filter MediaQueryFilter
return min, max, nil
}
func (r *MediaRepository) indexMediaBestEffort(ctx context.Context, media model.Media) {
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
if !ok {
return
}
_ = backend.IndexMedia(ctx, []model.Media{media})
}
// FindByID returns the media row or (nil, nil).
func (r *MediaRepository) FindByID(ctx context.Context, id string) (*model.Media, error) {
var m model.Media
+22 -3
View File
@@ -96,15 +96,34 @@ func (r *MediaRepository) UpsertBatchWithAliases(ctx context.Context, items []Me
// indexByIDBestEffort 在事务提交后按 ID 回读最新行并同步搜索索引。
func (r *MediaRepository) indexByIDBestEffort(ctx context.Context, ids []string) {
if _, ok := r.searchBackend.(MediaSearchSyncBackend); !ok {
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
if !ok {
return
}
unique := make([]string, 0, len(ids))
seen := make(map[string]struct{}, len(ids))
for _, id := range ids {
if id == "" {
continue
}
if fresh, err := r.FindByID(ctx, id); err == nil && fresh != nil {
r.indexMediaBestEffort(ctx, *fresh)
if _, exists := seen[id]; !exists {
seen[id] = struct{}{}
unique = append(unique, id)
}
}
// Read committed rows and submit one bulk per bounded batch. Keep network
// work outside the transaction and below SQLite's parameter limit.
const batchSize = 500
for start := 0; start < len(unique); start += batchSize {
batch := unique[start:min(start+batchSize, len(unique))]
rows, err := r.ListMediaByIDs(ctx, batch)
if err == nil {
err = backend.IndexMedia(ctx, rows)
}
if err != nil {
// The primary write has committed; do not report a failed upsert and
// make scanners repeat it. Surface the separate index failure instead.
r.db.Logger.Error(ctx, "media saved but search index sync failed for %d items: %v", len(batch), err)
}
}
}
+119 -4
View File
@@ -4,8 +4,10 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
@@ -135,14 +137,113 @@ func (b *OpenSearchMediaBackend) EnsureIndex(ctx context.Context) error {
}
func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Media) error {
const batchSize = 500
var failures []error
for start := 0; start < len(rows); start += batchSize {
if err := b.indexMediaBatch(ctx, rows[start:min(start+batchSize, len(rows))]); err != nil {
failures = append(failures, err)
if ctx.Err() != nil {
break
}
}
}
return errors.Join(failures...)
}
type openSearchBulkResponse struct {
Errors bool `json:"errors"`
Items []struct {
Index struct {
ID string `json:"_id"`
Status int `json:"status"`
Error *struct {
Type string `json:"type"`
} `json:"error"`
} `json:"index"`
} `json:"items"`
}
func (b *OpenSearchMediaBackend) indexMediaBatch(ctx context.Context, rows []model.Media) error {
if len(rows) == 0 {
return nil
}
pending := rows
var failures []error
const attempts = 3
for attempt := 0; attempt < attempts; attempt++ {
if err := ctx.Err(); err != nil {
return errors.Join(append(failures, err)...)
}
if attempt > 0 {
timer := time.NewTimer(time.Duration(100*(1<<(attempt-1))) * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
return errors.Join(append(failures, ctx.Err())...)
case <-timer.C:
}
}
response, err := b.submitMediaBulk(ctx, pending)
if err != nil {
if retryableOpenSearchError(err) && attempt < attempts-1 {
continue
}
return errors.Join(append(failures, err)...)
}
if len(response.Items) != len(pending) {
return errors.Join(append(failures, fmt.Errorf("opensearch bulk returned %d item results for %d documents", len(response.Items), len(pending)))...)
}
var retry []model.Media
failed := false
for i, item := range response.Items {
result := item.Index
if result.Status >= 200 && result.Status < 300 && result.Error == nil {
continue
}
failed = true
if retryableOpenSearchStatus(result.Status) && attempt < attempts-1 {
retry = append(retry, pending[i])
continue
}
errorType := "unknown"
if result.Error != nil {
errorType = result.Error.Type
}
failures = append(failures, fmt.Errorf("opensearch bulk document %q failed: status %d (%s)", pending[i].ID, result.Status, errorType))
}
if response.Errors && !failed {
failures = append(failures, errors.New("opensearch bulk reported errors without failed item results"))
}
if len(retry) == 0 {
break
}
pending = retry
}
return errors.Join(failures...)
}
func retryableOpenSearchStatus(status int) bool {
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
}
func retryableOpenSearchError(err error) bool {
var statusError *openSearchHTTPError
if errors.As(err, &statusError) {
return retryableOpenSearchStatus(statusError.status)
}
var networkError net.Error
return errors.As(err, &networkError)
}
func (b *OpenSearchMediaBackend) submitMediaBulk(ctx context.Context, rows []model.Media) (*openSearchBulkResponse, error) {
if len(rows) == 0 {
return &openSearchBulkResponse{}, nil
}
var bulk bytes.Buffer
enc := json.NewEncoder(&bulk)
for _, row := range rows {
if err := enc.Encode(map[string]any{"index": map[string]any{"_index": b.index, "_id": row.ID}}); err != nil {
return err
return nil, err
}
if err := enc.Encode(map[string]any{
"id": row.ID,
@@ -155,10 +256,24 @@ func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Me
"deleted": row.DeletedAt.Valid,
"created_at": row.CreatedAt,
}); err != nil {
return err
return nil, err
}
}
return b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", nil)
var response openSearchBulkResponse
if err := b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", &response); err != nil {
return nil, err
}
return &response, nil
}
type openSearchHTTPError struct {
method string
path string
status int
}
func (e *openSearchHTTPError) Error() string {
return fmt.Sprintf("opensearch %s %s returned %d", e.method, e.path, e.status)
}
func (b *OpenSearchMediaBackend) doJSON(ctx context.Context, method, path string, body any, out any) error {
@@ -190,7 +305,7 @@ func (b *OpenSearchMediaBackend) do(ctx context.Context, method, path string, bo
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("opensearch %s %s returned %d", method, path, resp.StatusCode)
return &openSearchHTTPError{method: method, path: path, status: resp.StatusCode}
}
if out == nil {
_, _ = io.Copy(io.Discard, resp.Body)
+164
View File
@@ -1,12 +1,23 @@
package repository
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/model"
)
func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
@@ -55,3 +66,156 @@ func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
t.Fatalf("ids=%#v total=%d", ids, total)
}
}
func bulkRequestIDs(r *http.Request) []string {
var ids []string
scanner := bufio.NewScanner(r.Body)
for scanner.Scan() {
var action struct {
Index struct {
ID string `json:"_id"`
} `json:"index"`
}
_ = json.Unmarshal(scanner.Bytes(), &action)
ids = append(ids, action.Index.ID)
if !scanner.Scan() {
break
}
}
return ids
}
func writeBulkResults(w http.ResponseWriter, ids []string, statuses map[string]int) {
items := make([]any, 0, len(ids))
failed := false
for _, id := range ids {
status := statuses[id]
if status == 0 {
status = http.StatusCreated
}
result := map[string]any{"_id": id, "status": status}
if status >= 400 {
failed = true
result["error"] = map[string]any{"type": "test_failure"}
}
items = append(items, map[string]any{"index": result})
}
_ = json.NewEncoder(w).Encode(map[string]any{"errors": failed, "items": items})
}
func TestOpenSearchBulkRetriesOnlyTransientFailedDocuments(t *testing.T) {
var requests [][]string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ids := bulkRequestIDs(r)
requests = append(requests, ids)
statuses := map[string]int{}
if len(requests) == 1 {
statuses["retry"] = http.StatusTooManyRequests
statuses["invalid"] = http.StatusBadRequest
}
writeBulkResults(w, ids, statuses)
}))
defer upstream.Close()
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
rows := []model.Media{{Base: model.Base{ID: "ok"}}, {Base: model.Base{ID: "retry"}}, {Base: model.Base{ID: "invalid"}}}
err := backend.IndexMedia(t.Context(), rows)
if err == nil || !strings.Contains(err.Error(), `"invalid"`) {
t.Fatalf("error = %v, want permanent document failure", err)
}
if len(requests) != 2 || strings.Join(requests[1], ",") != "retry" {
t.Fatalf("requests = %#v; successful/permanent documents must not be retried", requests)
}
}
func TestOpenSearchBulkBoundsRetriesAndRejectsMalformedResponses(t *testing.T) {
for _, scenario := range []string{"transient-item", "transient-http", "invalid-http", "missing-results"} {
t.Run(scenario, func(t *testing.T) {
calls := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
switch scenario {
case "transient-item":
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusServiceUnavailable})
case "transient-http":
w.WriteHeader(http.StatusServiceUnavailable)
case "invalid-http":
w.WriteHeader(http.StatusBadRequest)
case "missing-results":
fmt.Fprint(w, `{"errors":false,"items":[]}`)
}
}))
defer upstream.Close()
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
if err := backend.IndexMedia(t.Context(), []model.Media{{Base: model.Base{ID: "m1"}}}); err == nil {
t.Fatal("failed indexing was reported as successful")
}
want := 1
if strings.HasPrefix(scenario, "transient") {
want = 3
}
if calls != want {
t.Fatalf("calls = %d, want %d", calls, want)
}
})
}
}
func TestOpenSearchBulkHonorsCancellationDuringRetry(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
calls := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusTooManyRequests})
cancel()
}))
defer upstream.Close()
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
err := backend.IndexMedia(ctx, []model.Media{{Base: model.Base{ID: "m1"}}})
if !errors.Is(err, context.Canceled) || calls != 1 {
t.Fatalf("error = %v, calls = %d", err, calls)
}
}
func TestMediaUpsertBatchIndexesCommittedRowsTogether(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatal(err)
}
sqlDB, _ := db.DB()
sqlDB.SetMaxOpenConns(1)
defer sqlDB.Close()
if err := db.AutoMigrate(&model.Media{}); err != nil {
t.Fatal(err)
}
repos := New(db)
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
calls := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
ids := bulkRequestIDs(r)
if len(ids) != 5 {
t.Errorf("bulk contains %d documents, want 5", len(ids))
}
// With only one DB connection this also detects network work inside the
// transaction: a separate read could not observe the committed batch.
var count int64
if err := db.WithContext(ctx).Model(&model.Media{}).Count(&count).Error; err != nil || count != 5 {
t.Errorf("committed rows = %d, error = %v", count, err)
}
writeBulkResults(w, ids, nil)
}))
defer upstream.Close()
repos.Media.SetSearchBackend(NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL}))
var rows []*model.Media
for i := 0; i < 5; i++ {
rows = append(rows, &model.Media{LibraryID: "lib", Path: fmt.Sprintf("/media/%d.mkv", i)})
}
if err := repos.Media.UpsertBatch(ctx, rows); err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("index requests = %d, want one bulk", calls)
}
}