diff --git a/internal/service/duplicate.go b/internal/service/duplicate.go index 0097b8a..b39396b 100644 --- a/internal/service/duplicate.go +++ b/internal/service/duplicate.go @@ -16,13 +16,7 @@ package service import ( "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "io" "os" - "sort" "go.uber.org/zap" @@ -30,8 +24,6 @@ import ( "github.com/ShukeBta/MediaStationGo/internal/repository" ) -const sampleSize = 1 << 20 // 1 MiB per sample window - // DuplicateService is the entry point for the duplicate finder. type DuplicateService struct { log *zap.Logger @@ -149,72 +141,6 @@ func (d *DuplicateService) Detect(ctx context.Context, libraryID string) (*Repor return rep, nil } -func (d *DuplicateService) markDuplicateGroup(ctx context.Context, rep *Report, key string, group []model.Media, markedIDs map[string]struct{}) { - primary := pickPrimary(group) - dupes := make([]model.Media, 0, len(group)-1) - for _, m := range group { - if m.ID == primary.ID || m.DuplicateOf == primary.ID { - continue - } - if _, ok := markedIDs[m.ID]; ok { - continue - } - dupes = append(dupes, m) - if err := d.repo.DB.WithContext(ctx). - Model(&model.Media{}). - Where("id = ?", m.ID). - Updates(map[string]any{ - "is_duplicate": true, - "duplicate_of": primary.ID, - }).Error; err != nil { - d.log.Warn("dup mark failed", zap.Error(err)) - continue - } - markedIDs[m.ID] = struct{}{} - rep.ItemsMarked++ - } - if len(dupes) == 0 { - return - } - rep.Groups = append(rep.Groups, Group{ - Hash: key, - Primary: primary, - Duplicates: dupes, - }) -} - -func groupByExternalIdentity(rows []model.Media) map[string][]model.Media { - groups := map[string][]model.Media{} - for _, row := range rows { - key := mediaExternalIdentityKey(row) - if key == "" { - continue - } - groups[key] = append(groups[key], row) - } - return groups -} - -func mediaExternalIdentityKey(row model.Media) string { - var key string - switch { - case row.TMDbID > 0: - key = fmt.Sprintf("tmdb:%d", row.TMDbID) - case row.BangumiID > 0: - key = fmt.Sprintf("bangumi:%d", row.BangumiID) - case row.DoubanID != "": - key = "douban:" + row.DoubanID - case row.TheTVDBID != "": - key = "thetvdb:" + row.TheTVDBID - default: - return "" - } - if row.SeasonNum > 0 || row.EpisodeNum > 0 { - key += fmt.Sprintf(":s%d:e%d", row.SeasonNum, row.EpisodeNum) - } - return key -} - // Current returns duplicate groups already marked in the database. It keeps // the UI useful after a prior scan and avoids requiring POST on page load. func (d *DuplicateService) Current(ctx context.Context, libraryID string) (*Report, error) { @@ -288,67 +214,3 @@ func (d *DuplicateService) Unmark(ctx context.Context, libraryID string) (int64, res := q.Updates(map[string]any{"is_duplicate": false, "duplicate_of": ""}) return res.RowsAffected, res.Error } - -// pickPrimary picks the "best" media row to keep: prefer scraped > size > id. -func pickPrimary(group []model.Media) model.Media { - sort.SliceStable(group, func(i, j int) bool { - ai, aj := group[i].ScrapeStatus == "matched", group[j].ScrapeStatus == "matched" - if ai != aj { - return ai - } - if group[i].SizeBytes != group[j].SizeBytes { - return group[i].SizeBytes > group[j].SizeBytes - } - return group[i].ID < group[j].ID - }) - return group[0] -} - -// SparseFileHash computes the head+mid+tail SHA-256 of a file, suffixed with -// the file size so two files that happen to collide on the sample window -// but differ in length are still distinguishable. -func SparseFileHash(path string) (string, error) { - if path == "" { - return "", errors.New("empty path") - } - f, err := os.Open(path) // #nosec G304 -- path is selected from configured media library files for duplicate detection. - if err != nil { - return "", err - } - defer f.Close() - st, err := f.Stat() - if err != nil { - return "", err - } - size := st.Size() - h := sha256.New() - if size <= int64(sampleSize)*3 { - if _, err := io.Copy(h, f); err != nil { - return "", err - } - return fmt.Sprintf("%s-%d", hex.EncodeToString(h.Sum(nil)), size), nil - } - buf := make([]byte, sampleSize) - // head - if _, err := io.ReadFull(f, buf); err != nil { - return "", err - } - h.Write(buf) - // middle - if _, err := f.Seek(size/2-int64(sampleSize)/2, io.SeekStart); err != nil { - return "", err - } - if _, err := io.ReadFull(f, buf); err != nil { - return "", err - } - h.Write(buf) - // tail - if _, err := f.Seek(size-int64(sampleSize), io.SeekStart); err != nil { - return "", err - } - if _, err := io.ReadFull(f, buf); err != nil { - return "", err - } - h.Write(buf) - return fmt.Sprintf("%s-%d", hex.EncodeToString(h.Sum(nil)), size), nil -} diff --git a/internal/service/duplicate_groups.go b/internal/service/duplicate_groups.go new file mode 100644 index 0000000..3695361 --- /dev/null +++ b/internal/service/duplicate_groups.go @@ -0,0 +1,93 @@ +// Package service — duplicate grouping helpers. +package service + +import ( + "context" + "fmt" + "sort" + + "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func (d *DuplicateService) markDuplicateGroup(ctx context.Context, rep *Report, key string, group []model.Media, markedIDs map[string]struct{}) { + primary := pickPrimary(group) + dupes := make([]model.Media, 0, len(group)-1) + for _, m := range group { + if m.ID == primary.ID || m.DuplicateOf == primary.ID { + continue + } + if _, ok := markedIDs[m.ID]; ok { + continue + } + dupes = append(dupes, m) + if err := d.repo.DB.WithContext(ctx). + Model(&model.Media{}). + Where("id = ?", m.ID). + Updates(map[string]any{ + "is_duplicate": true, + "duplicate_of": primary.ID, + }).Error; err != nil { + d.log.Warn("dup mark failed", zap.Error(err)) + continue + } + markedIDs[m.ID] = struct{}{} + rep.ItemsMarked++ + } + if len(dupes) == 0 { + return + } + rep.Groups = append(rep.Groups, Group{ + Hash: key, + Primary: primary, + Duplicates: dupes, + }) +} + +func groupByExternalIdentity(rows []model.Media) map[string][]model.Media { + groups := map[string][]model.Media{} + for _, row := range rows { + key := mediaExternalIdentityKey(row) + if key == "" { + continue + } + groups[key] = append(groups[key], row) + } + return groups +} + +func mediaExternalIdentityKey(row model.Media) string { + var key string + switch { + case row.TMDbID > 0: + key = fmt.Sprintf("tmdb:%d", row.TMDbID) + case row.BangumiID > 0: + key = fmt.Sprintf("bangumi:%d", row.BangumiID) + case row.DoubanID != "": + key = "douban:" + row.DoubanID + case row.TheTVDBID != "": + key = "thetvdb:" + row.TheTVDBID + default: + return "" + } + if row.SeasonNum > 0 || row.EpisodeNum > 0 { + key += fmt.Sprintf(":s%d:e%d", row.SeasonNum, row.EpisodeNum) + } + return key +} + +// pickPrimary picks the "best" media row to keep: prefer scraped > size > id. +func pickPrimary(group []model.Media) model.Media { + sort.SliceStable(group, func(i, j int) bool { + ai, aj := group[i].ScrapeStatus == "matched", group[j].ScrapeStatus == "matched" + if ai != aj { + return ai + } + if group[i].SizeBytes != group[j].SizeBytes { + return group[i].SizeBytes > group[j].SizeBytes + } + return group[i].ID < group[j].ID + }) + return group[0] +} diff --git a/internal/service/duplicate_hash.go b/internal/service/duplicate_hash.go new file mode 100644 index 0000000..9847fda --- /dev/null +++ b/internal/service/duplicate_hash.go @@ -0,0 +1,62 @@ +// Package service — duplicate sparse file hashing. +package service + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" +) + +const sampleSize = 1 << 20 // 1 MiB per sample window + +// SparseFileHash computes the head+mid+tail SHA-256 of a file, suffixed with +// the file size so two files that happen to collide on the sample window +// but differ in length are still distinguishable. +func SparseFileHash(path string) (string, error) { + if path == "" { + return "", errors.New("empty path") + } + f, err := os.Open(path) // #nosec G304 -- path is selected from configured media library files for duplicate detection. + if err != nil { + return "", err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return "", err + } + size := st.Size() + h := sha256.New() + if size <= int64(sampleSize)*3 { + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%s-%d", hex.EncodeToString(h.Sum(nil)), size), nil + } + buf := make([]byte, sampleSize) + // head + if _, err := io.ReadFull(f, buf); err != nil { + return "", err + } + h.Write(buf) + // middle + if _, err := f.Seek(size/2-int64(sampleSize)/2, io.SeekStart); err != nil { + return "", err + } + if _, err := io.ReadFull(f, buf); err != nil { + return "", err + } + h.Write(buf) + // tail + if _, err := f.Seek(size-int64(sampleSize), io.SeekStart); err != nil { + return "", err + } + if _, err := io.ReadFull(f, buf); err != nil { + return "", err + } + h.Write(buf) + return fmt.Sprintf("%s-%d", hex.EncodeToString(h.Sum(nil)), size), nil +}