From 92c3ce3993438256f5314c16920397cd4a936cc4 Mon Sep 17 00:00:00 2001 From: Keybored <78739882+Keybored02@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:40:24 +0100 Subject: [PATCH] [Feature] Rework Archive duplicates tagging (#718) [Feature] Rework Archive duplicates tagging (#718) --- backend/app/api/routes/archives.py | 100 ++++++++++++++++++++++-- backend/app/schemas/archive.py | 2 + backend/app/services/archive.py | 36 ++++++--- frontend/src/api/client.ts | 2 + frontend/src/i18n/locales/de.ts | 3 + frontend/src/i18n/locales/en.ts | 3 + frontend/src/i18n/locales/fr.ts | 3 + frontend/src/i18n/locales/it.ts | 3 + frontend/src/i18n/locales/ja.ts | 3 + frontend/src/i18n/locales/pt-BR.ts | 3 + frontend/src/i18n/locales/zh-CN.ts | 3 + frontend/src/pages/ArchivesPage.tsx | 117 +++++++++++++++++++++++++--- 12 files changed, 251 insertions(+), 27 deletions(-) diff --git a/backend/app/api/routes/archives.py b/backend/app/api/routes/archives.py index 46a28c4b0..e7d8749e9 100644 --- a/backend/app/api/routes/archives.py +++ b/backend/app/api/routes/archives.py @@ -2,13 +2,14 @@ import io import json import logging import zipfile +from collections import defaultdict from datetime import date, datetime, time, timezone from decimal import ROUND_HALF_UP, Decimal from pathlib import Path from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse, Response -from sqlalchemy import func, select +from sqlalchemy import and_, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from backend.app.core.auth import ( @@ -62,6 +63,8 @@ def archive_to_response( archive: PrintArchive, duplicates: list[dict] | None = None, duplicate_count: int = 0, + duplicate_sequence: int = 0, + original_archive_id: int | None = None, ) -> dict: """Convert archive model to response dict with computed fields.""" data = { @@ -79,6 +82,8 @@ def archive_to_response( "f3d_path": archive.f3d_path, "duplicates": duplicates, "duplicate_count": duplicate_count if duplicates is None else len(duplicates), + "duplicate_sequence": duplicate_sequence, + "original_archive_id": original_archive_id, "print_name": archive.print_name, "print_time_seconds": archive.print_time_seconds, "filament_used_grams": archive.filament_used_grams, @@ -141,16 +146,99 @@ async def list_archives( offset=offset, ) - # Get sets of hashes and names that have duplicates (efficient single queries) - duplicate_hashes, duplicate_names = await service.get_duplicate_hashes_and_names() + # Get sets of duplicate hashes and duplicate (name, hash) pairs (efficient single queries) + duplicate_hashes, duplicate_name_hash_pairs = await service.get_duplicate_hashes_and_names() - # Mark archives that have duplicates (by hash or by print name) + # Batch-load duplicate groups once for the current page keys. + duplicate_hashes_in_page = { + a.content_hash for a in archives if a.content_hash and a.content_hash in duplicate_hashes + } + duplicate_name_hash_keys_in_page = { + (a.print_name.lower(), a.content_hash) + for a in archives + if a.print_name and a.content_hash and (a.print_name.lower(), a.content_hash) in duplicate_name_hash_pairs + } + + duplicate_meta_by_archive_id: dict[int, tuple[int, int, int]] = {} + + if duplicate_hashes_in_page or duplicate_name_hash_keys_in_page: + duplicate_group_conditions = [] + if duplicate_hashes_in_page: + duplicate_group_conditions.append(PrintArchive.content_hash.in_(duplicate_hashes_in_page)) + if duplicate_name_hash_keys_in_page: + name_hash_conditions = [ + and_(func.lower(PrintArchive.print_name) == name, PrintArchive.content_hash == hash_) + for name, hash_ in duplicate_name_hash_keys_in_page + ] + duplicate_group_conditions.extend(name_hash_conditions) + + duplicate_group_rows = await db.execute( + select( + PrintArchive.id, + PrintArchive.created_at, + PrintArchive.content_hash, + func.lower(PrintArchive.print_name).label("print_name_lower"), + ).where(or_(*duplicate_group_conditions)) + ) + + duplicate_groups_by_hash: dict[str, list[tuple[int, datetime]]] = defaultdict(list) + duplicate_groups_by_name_hash: dict[tuple[str, str], list[tuple[int, datetime]]] = defaultdict(list) + + for archive_id, created_at, content_hash, print_name_lower in duplicate_group_rows.all(): + if content_hash and content_hash in duplicate_hashes_in_page: + duplicate_groups_by_hash[content_hash].append((archive_id, created_at)) + if ( + print_name_lower + and content_hash + and (print_name_lower, content_hash) in duplicate_name_hash_keys_in_page + ): + duplicate_groups_by_name_hash[(print_name_lower, content_hash)].append((archive_id, created_at)) + + for group in duplicate_groups_by_hash.values(): + if len(group) < 2: + continue + group.sort(key=lambda x: x[1]) + original_id = group[0][0] + duplicate_count = len(group) - 1 + for sequence, (archive_id, _) in enumerate(group): + duplicate_meta_by_archive_id[archive_id] = (sequence, original_id, duplicate_count) + + # Keep hash-based grouping precedence; name/hash groups only fill missing items. + for group in duplicate_groups_by_name_hash.values(): + if len(group) < 2: + continue + group.sort(key=lambda x: x[1]) + original_id = group[0][0] + duplicate_count = len(group) - 1 + for sequence, (archive_id, _) in enumerate(group): + duplicate_meta_by_archive_id.setdefault(archive_id, (sequence, original_id, duplicate_count)) + + # Build response with duplicate sequence and original archive ID pre-computed result = [] for a in archives: has_hash_dup = a.content_hash in duplicate_hashes if a.content_hash else False - has_name_dup = a.print_name and a.print_name.lower() in duplicate_names + has_name_dup = ( + bool(a.print_name and a.content_hash) + and (a.print_name.lower(), a.content_hash) in duplicate_name_hash_pairs + ) has_duplicate = has_hash_dup or has_name_dup - result.append(archive_to_response(a, duplicate_count=1 if has_duplicate else 0)) + + # Pre-compute duplicate sequence and original archive ID + duplicate_sequence = 0 + original_archive_id: int | None = None + duplicate_count = 1 if has_duplicate else 0 + + if has_duplicate and a.id in duplicate_meta_by_archive_id: + duplicate_sequence, original_archive_id, duplicate_count = duplicate_meta_by_archive_id[a.id] + + result.append( + archive_to_response( + a, + duplicate_count=duplicate_count, + duplicate_sequence=duplicate_sequence, + original_archive_id=original_archive_id, + ) + ) return result diff --git a/backend/app/schemas/archive.py b/backend/app/schemas/archive.py index 6dae2ae67..6264f3b95 100644 --- a/backend/app/schemas/archive.py +++ b/backend/app/schemas/archive.py @@ -48,6 +48,8 @@ class ArchiveResponse(BaseModel): # Duplicate detection duplicates: list[ArchiveDuplicate] | None = None duplicate_count: int = 0 # Quick count for list views + duplicate_sequence: int = 0 # 0 = original, 1+ = nth duplicate + original_archive_id: int | None = None # ID of the first/original archive # Object count (computed from extra_data.printable_objects) object_count: int | None = None diff --git a/backend/app/services/archive.py b/backend/app/services/archive.py index 8c4430fcb..156594ab4 100644 --- a/backend/app/services/archive.py +++ b/backend/app/services/archive.py @@ -727,10 +727,14 @@ class ArchiveService: sha256.update(chunk) return sha256.hexdigest() - async def get_duplicate_hashes_and_names(self) -> tuple[set[str], set[str]]: - """Get all content hashes and print names that appear more than once. + async def get_duplicate_hashes_and_names(self) -> tuple[set[str], set[tuple[str, str]]]: + """Get all content hashes and (print name, hash) pairs that appear more than once. - Returns a tuple of (duplicate_hashes, duplicate_names). + For hashes: returns all hashes with > 1 archive (true duplicates). + For name/hash pairs: returns only pairs that have > 1 archive + (i.e., same file archived multiple times, not different files with same name). + + Returns a tuple of (duplicate_hashes, duplicate_name_hash_pairs). """ from sqlalchemy import func @@ -742,15 +746,17 @@ class ArchiveService: ) duplicate_hashes = {row[0] for row in result.all()} + # Find print names that have multiple archives with the SAME hash + # This avoids marking different files with the same name as duplicates result = await self.db.execute( - select(func.lower(PrintArchive.print_name)) - .where(PrintArchive.print_name.isnot(None)) - .group_by(func.lower(PrintArchive.print_name)) + select(func.lower(PrintArchive.print_name), PrintArchive.content_hash) + .where(PrintArchive.print_name.isnot(None), PrintArchive.content_hash.isnot(None)) + .group_by(func.lower(PrintArchive.print_name), PrintArchive.content_hash) .having(func.count(PrintArchive.id) > 1) ) - duplicate_names = {row[0] for row in result.all()} + duplicate_name_hash_pairs = {(row[0], row[1]) for row in result.all()} - return duplicate_hashes, duplicate_names + return duplicate_hashes, duplicate_name_hash_pairs async def find_duplicates( self, @@ -789,15 +795,23 @@ class ArchiveService: ) # Then, find similar matches by print name or MakerWorld ID + # Prefer strict name+hash matching when hash exists; fallback to name-only for legacy/manual + # archives that may not have a content_hash. if print_name or makerworld_model_id: conditions = [PrintArchive.id != archive_id] name_conditions = [] if print_name: - # Match if print names are similar (ignoring case) - name_conditions.append(PrintArchive.print_name.ilike(print_name)) + if content_hash: + # Match if print names are similar AND have the same hash (same file) + name_conditions.append( + and_(PrintArchive.print_name.ilike(print_name), PrintArchive.content_hash == content_hash) + ) + else: + # Fallback for archives without hash data: match by print name only. + name_conditions.append(PrintArchive.print_name.ilike(print_name)) if makerworld_model_id: - # Match by MakerWorld model ID stored in extra_data + # Match by MakerWorld model ID stored in extra_data (same design from MakerWorld) # Use json_extract for SQLite compatibility (astext is PostgreSQL-only) from sqlalchemy import func diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 2a5fb1159..fc5d930ae 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -349,6 +349,8 @@ export interface Archive { f3d_path: string | null; duplicates: ArchiveDuplicate[] | null; duplicate_count: number; + duplicate_sequence: number; // 0 = original, 1+ = nth duplicate + original_archive_id: number | null; // ID of the first/original archive object_count: number | null; print_name: string | null; print_time_seconds: number | null; diff --git a/frontend/src/i18n/locales/de.ts b/frontend/src/i18n/locales/de.ts index 68f30773f..f186d4c6e 100644 --- a/frontend/src/i18n/locales/de.ts +++ b/frontend/src/i18n/locales/de.ts @@ -524,6 +524,7 @@ export default { sortSize: 'Größe', noArchives: 'Keine Archive gefunden', noArchivesSearch: 'Keine Archive entsprechen Ihrer Suche', + originalPrintNotVisible: 'Ursprünglicher Druck nicht sichtbar - versuchen Sie, die Filter zu löschen', noArchivesYet: 'Noch keine Archive', loadingArchives: 'Lade Archive...', releaseToUpload: 'Loslassen zum Hochladen', @@ -536,6 +537,8 @@ export default { manageTags: 'Tags verwalten', showFailedPrints: 'Fehlgeschlagene Drucke anzeigen', hideFailedPrints: 'Fehlgeschlagene Drucke ausblenden', + hideDuplicates: 'Duplikate ausblenden', + viewOriginalPrint: 'Klicken, um den ursprünglichen Druck anzuzeigen (#{{id}})', printTime: 'Druckzeit', filamentUsed: 'Verbrauchtes Filament', cost: 'Kosten', diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index fb81467b0..d481900a1 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -524,6 +524,7 @@ export default { sortSize: 'Size', noArchives: 'No archives found', noArchivesSearch: 'No archives match your search', + originalPrintNotVisible: 'Original print not visible - try clearing filters', noArchivesYet: 'No archives yet', loadingArchives: 'Loading archives...', releaseToUpload: 'Release to upload', @@ -536,6 +537,8 @@ export default { manageTags: 'Manage Tags', showFailedPrints: 'Show failed prints', hideFailedPrints: 'Hide failed prints', + hideDuplicates: 'Hide Duplicates', + viewOriginalPrint: 'Click to view original print (#{{id}})', printTime: 'Print Time', filamentUsed: 'Filament Used', cost: 'Cost', diff --git a/frontend/src/i18n/locales/fr.ts b/frontend/src/i18n/locales/fr.ts index 0eed7132e..f6048b070 100644 --- a/frontend/src/i18n/locales/fr.ts +++ b/frontend/src/i18n/locales/fr.ts @@ -524,6 +524,7 @@ export default { sortSize: 'Taille', noArchives: 'Aucune archive trouvée', noArchivesSearch: 'Aucune archive ne correspond', + originalPrintNotVisible: 'Impression d\'origine non visible - essayez d\'effacer les filtres', noArchivesYet: 'Pas encore d\'archive', loadingArchives: 'Chargement...', releaseToUpload: 'Relâcher pour téléverser', @@ -536,6 +537,8 @@ export default { manageTags: 'Gérer les tags', showFailedPrints: 'Afficher les échecs', hideFailedPrints: 'Masquer les échecs', + hideDuplicates: 'Masquer les doublons', + viewOriginalPrint: 'Cliquez pour afficher l\'impression originale (#{{id}})', printTime: 'Temps d\'impression', filamentUsed: 'Filament utilisé', cost: 'Coût', diff --git a/frontend/src/i18n/locales/it.ts b/frontend/src/i18n/locales/it.ts index 836cc3f87..3305913b9 100644 --- a/frontend/src/i18n/locales/it.ts +++ b/frontend/src/i18n/locales/it.ts @@ -524,6 +524,7 @@ export default { sortSize: 'Dimensione', noArchives: 'Nessun archivio trovato', noArchivesSearch: 'Nessun archivio corrisponde alla ricerca', + originalPrintNotVisible: 'Stampa originale non visibile - prova a rimuovere i filtri', noArchivesYet: 'Nessun archivio ancora', loadingArchives: 'Caricamento archivi...', releaseToUpload: 'Rilascia per caricare', @@ -536,6 +537,8 @@ export default { manageTags: 'Gestisci tag', showFailedPrints: 'Mostra stampe fallite', hideFailedPrints: 'Nascondi stampe fallite', + hideDuplicates: 'Nascondi duplicati', + viewOriginalPrint: 'Fai clic per visualizzare la stampa originale (#{{id}})', printTime: 'Tempo di stampa', filamentUsed: 'Filamento usato', cost: 'Costo', diff --git a/frontend/src/i18n/locales/ja.ts b/frontend/src/i18n/locales/ja.ts index 4d72904b3..ee0dc2bbb 100644 --- a/frontend/src/i18n/locales/ja.ts +++ b/frontend/src/i18n/locales/ja.ts @@ -523,6 +523,7 @@ export default { sortSize: 'サイズ', noArchives: 'アーカイブが見つかりません', noArchivesSearch: '検索条件に一致するアーカイブがありません', + originalPrintNotVisible: '元の印刷が表示されていません - フィルターをクリアしてみてください', noArchivesYet: 'アーカイブはまだありません', loadingArchives: 'アーカイブを読み込み中...', releaseToUpload: 'ドロップしてアップロード', @@ -535,6 +536,8 @@ export default { manageTags: 'タグを管理', showFailedPrints: '失敗した印刷を表示', hideFailedPrints: '失敗した印刷を非表示', + hideDuplicates: '重複を非表示', + viewOriginalPrint: 'クリックして元の印刷を表示 (#{{id}})', printTime: '印刷時間', filamentUsed: 'フィラメント使用量', cost: 'コスト', diff --git a/frontend/src/i18n/locales/pt-BR.ts b/frontend/src/i18n/locales/pt-BR.ts index 65b4326d9..cbd903f9a 100644 --- a/frontend/src/i18n/locales/pt-BR.ts +++ b/frontend/src/i18n/locales/pt-BR.ts @@ -524,6 +524,7 @@ export default { sortSize: 'Tamanho', noArchives: 'Nenhum arquivo encontrado', noArchivesSearch: 'Nenhum arquivo corresponde à sua pesquisa', + originalPrintNotVisible: 'Impressão original não visível - tente limpar os filtros', noArchivesYet: 'Ainda não há arquivos', loadingArchives: 'Carregando arquivos...', releaseToUpload: 'Solte para enviar', @@ -536,6 +537,8 @@ export default { manageTags: 'Gerenciar etiquetas', showFailedPrints: 'Mostrar impressões falhas', hideFailedPrints: 'Ocultar impressões falhas', + hideDuplicates: 'Ocultar duplicados', + viewOriginalPrint: 'Clique para visualizar a impressão original (#{{id}})', printTime: 'Tempo de impressão', filamentUsed: 'Filamento usado', cost: 'Custo', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 77172e6d2..505cdf8f8 100644 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -524,6 +524,7 @@ export default { sortSize: '大小', noArchives: '未找到归档', noArchivesSearch: '没有匹配搜索的归档', + originalPrintNotVisible: '原始打印不可见 - 请尝试清除筛选条件', noArchivesYet: '暂无归档', loadingArchives: '加载归档中...', releaseToUpload: '释放以上传', @@ -536,6 +537,8 @@ export default { manageTags: '管理标签', showFailedPrints: '显示失败的打印', hideFailedPrints: '隐藏失败的打印', + hideDuplicates: '隐藏重复项', + viewOriginalPrint: '点击查看原始打印 (#{{id}})', printTime: '打印时间', filamentUsed: '耗材用量', cost: '成本', diff --git a/frontend/src/pages/ArchivesPage.tsx b/frontend/src/pages/ArchivesPage.tsx index 59688914d..4c83ede3c 100644 --- a/frontend/src/pages/ArchivesPage.tsx +++ b/frontend/src/pages/ArchivesPage.tsx @@ -41,6 +41,7 @@ import { MoreVertical, FileSpreadsheet, GitCompare, + GitBranch, Loader2, FolderKanban, ChevronLeft, @@ -151,6 +152,7 @@ function ArchiveCard({ preferredSlicer = 'bambu_studio', currency, t, + onNavigateToArchive, }: { archive: Archive; printerName: string; @@ -163,6 +165,7 @@ function ArchiveCard({ preferredSlicer?: SlicerType; currency: string; t: TFunction; + onNavigateToArchive?: (archiveId: number) => void; }) { // Debug: log when card is highlighted if (isHighlighted) { @@ -202,6 +205,10 @@ function ArchiveCard({ staleTime: 5 * 60 * 1000, // Cache for 5 minutes }); + // Use pre-computed duplicate sequence and original archive ID from list response + const duplicateSequence = archive.duplicate_sequence ?? 0; + const originalArchiveId = archive.original_archive_id ?? null; + const plates = platesData?.plates ?? []; const isMultiPlate = platesData?.is_multi_plate ?? false; const displayPlateIndex = currentPlateIndex ?? 0; @@ -755,14 +762,27 @@ function ArchiveCard({ )} {/* Duplicate badge */} - {archive.duplicate_count > 0 && ( -
#{archive.id}
+ {/* Title */}