tcjlj/MeBox
1
0
Fork 0
mirror of https://github.com/truewhile/MeBox.git synced 2026-10-06 13:26:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add configurable scraper throttling

Browse Source
This commit is contained in:
ShukeBta
2026-06-09 17:56:25 +08:00
parent 76fe0dbde9
commit a55fc502ca
5 changed files with 107 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/service/scraper_test.go
+33
View File
@@ -8,6 +8,7 @@ import (
"path/filepath"
"strings"
"testing"
"time"
"github.com/glebarez/sqlite"
"go.uber.org/zap"
@@ -200,6 +201,38 @@ func TestManualEnrichLibraryRetriesNoMatchAndCountsRealMatches(t *testing.T) {
}
}
func TestScrapeDelayUsesSettings(t *testing.T) {
scraper, repos, closeServer := newTestScraper(t)
defer closeServer()
if err := repos.DB.AutoMigrate(&model.Setting{}); err != nil {
t.Fatal(err)
}
if got := scraper.scrapeDelay(t.Context()); got < 250*time.Millisecond || got > 500*time.Millisecond {
t.Fatalf("default scrapeDelay = %s, want 250-500ms", got)
}
if err := repos.Setting.Set(t.Context(), "scrape.delay_min_ms", "0"); err != nil {
t.Fatal(err)
}
if err := repos.Setting.Set(t.Context(), "scrape.delay_max_ms", "0"); err != nil {
t.Fatal(err)
}
if got := scraper.scrapeDelay(t.Context()); got != 0 {
t.Fatalf("disabled scrapeDelay = %s, want 0", got)
}
if err := repos.Setting.Set(t.Context(), "scrape.delay_min_ms", "800"); err != nil {
t.Fatal(err)
}
if err := repos.Setting.Set(t.Context(), "scrape.delay_max_ms", "200"); err != nil {
t.Fatal(err)
}
if got := scraper.scrapeDelay(t.Context()); got != 800*time.Millisecond {
t.Fatalf("normalized scrapeDelay = %s, want 800ms", got)
}
}
func newTestScraper(t *testing.T) (*ScraperService, *repository.Container, func()) {
t.Helper()