diff --git a/docs/rom-verification-adr.md b/docs/rom-verification-adr.md new file mode 100644 index 00000000..4ac2fd80 --- /dev/null +++ b/docs/rom-verification-adr.md @@ -0,0 +1,103 @@ +# ADR, No-Intro ROM verification architecture contract + +## Status + +Approved for Todo 1 planning and implementation follow-up. + +## Context + +Gamarr needs a generic No-Intro verification design for ROM libraries that works across No-Intro-backed systems without bending current game file behavior into ROM-specific rules. The architecture scope is generic No-Intro systems, with Game Boy Advance and Nintendo DS used as the first validation examples because they already show the ZIP-heavy and mixed ZIP plus raw shapes that v1 must handle. Switch is deferred because its TitleID-driven metadata model is a separate problem. + +## Decision summary + +Gamarr will add a separate ROM inventory and verification aggregate for No-Intro-backed systems. It will not implicitly reuse `GameFileId`, `HasFile`, or existing wanted semantics. This ADR acts as the migration guard for those legacy concepts. ZIP verification is read-only and does not imply extraction or import overhaul. Catalog truth comes from automatic pinned upstream DAT or catalog sync with visible source, version, and last-sync metadata, plus a manual refresh action. + +## Data model boundary + +The No-Intro feature owns a separate ROM inventory and verification aggregate. + +That aggregate is responsible for: + +- catalog source identity, pinned upstream release metadata, sync timestamps, and sync failure state +- canonical catalog entries and their hash truth +- managed library membership for verification roots and system scope +- per-library-file verification results for raw files and ZIP-contained ROM payloads +- orthogonal duplicate and missing-set reporting +- expected filename resolution for the selected rename profile + +The existing game and game-file model remains separate. + +- Do not implicitly reuse `GameFileId` as the identity of a verified ROM record. +- Do not implicitly reuse `HasFile` as proof that a ROM is verified or present in a managed verification set. +- Do not reuse existing wanted semantics. `Missing` for ROM verification is a separate verification-set concept and not a replacement for existing wanted or monitored flows. + +If later implementation needs any compatibility bridge, it must be an explicit migration path, not hidden semantic reuse. + +## Verification status table + +| Status | Scope | Meaning | Notes | +| --- | --- | --- | --- | +| Verified | per file | The ROM payload hash matches a loaded No-Intro catalog entry and the filename matches the selected rename profile | Name checks are profile-relative | +| Name mismatch | per file | The ROM payload hash matches a loaded No-Intro catalog entry, but the filename or path shape differs from the selected rename profile | `Name mismatch` is relative to the selected rename profile | +| Unknown | per file | The ROM payload hash does not match any loaded No-Intro catalog entry | Not a naming issue | +| Bad dump | per file | The ROM payload hash matches a known bad, overdump, headered, or otherwise non-good catalog entry, or a deterministic bad-dump rule | Takes precedence over Verified in summary views | +| Duplicate | orthogonal flag | More than one managed library file resolves to the same canonical No-Intro entry | Does not replace the per-file verification state | +| Missing | verification set | A catalog entry expected by the managed verification set is absent from the managed library scope | `Missing` is scoped to the managed verification set | + +Summary precedence for the primary per-file state is: Bad dump, Unknown, Name mismatch, Verified. Duplicate and Missing remain orthogonal signals. + +## ZIP/raw matching rules + +The authoritative verification unit is the normalized catalog ROM entry derived from raw ROM bytes. + +- Loose files are hashed from the raw ROM payload. +- ZIP verification is read-only and does not imply extraction, import, or archive-management overhaul. +- ZIP-contained matching hashes the selected ROM payload inside the archive against the same catalog truth used for loose files. +- Raw and ZIP matches converge on the same canonical ROM entry when the payload bytes are the same. +- V1 must classify ambiguous or unsupported multi-member archives deterministically instead of guessing. +- A successful ZIP match does not grant any separate import semantics, extracted-file lifecycle, or ownership change. + +## V1 enablement matrix + +| Area | V1 decision | +| --- | --- | +| Architecture scope | Generic No-Intro systems architecture scope | +| Initial validation examples | GBA and Nintendo DS | +| Catalog source | Automatic pinned upstream DAT or catalog sync | +| Catalog visibility | Show source, version, last-sync, last-attempt, and refresh failure state | +| Refresh behavior | Keep prior good catalog data visible on failed refresh, allow manual refresh | +| Verification inputs | Raw ROM files and ZIP-contained ROM payloads | +| Naming behavior | Keep current Gamarr naming as default, add No-Intro-aware profile evaluation | +| Missing scope | Managed verification set only | +| ZIP behavior | Read-only verification only | +| Switch | Deferred | + +## Automatic upstream catalog policy + +The approved owner decision is automatic pinned upstream DAT or catalog sync, not an unresolved default. + +- The system stores the upstream source identity, pinned version or revision, last successful sync, last attempted sync, and failure state. +- The UI and API must expose visible source, version, and last-sync details. +- Manual refresh is available. +- Failed refresh attempts must not silently replace or clear the prior good catalog snapshot. + +## Validation examples + +Game Boy Advance and Nintendo DS are the first validation examples for v1 because they prove the generic design against real library shapes. + +- GBA validates ZIP-heavy libraries, numbered by-id naming, and non-retail subsets. +- Nintendo DS validates mixed raw `.nds` and ZIP libraries plus additional subset folders. +- These examples validate the architecture. They do not narrow the architecture to only those two systems. + +## Explicit non-goals + +- No implicit reuse of `GameFileId`, `HasFile`, or existing wanted semantics. +- No archive extraction workflow change. +- No ROM import overhaul. +- No forced bulk rename of existing ROM libraries. +- No assumption that current folder names are authoritative catalog truth. +- No Switch implementation in v1. Switch is deferred. + +## Consequences + +This boundary keeps today’s game and file behavior stable while making room for ROM-specific verification truth, duplicate detection, missing-set reporting, and profile-relative naming checks. It also keeps ZIP support narrow and safe, because verification stays read-only in v1. diff --git a/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.test.tsx b/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.test.tsx index 7c08f7b3..c7776ed0 100644 --- a/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.test.tsx +++ b/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.test.tsx @@ -230,6 +230,7 @@ describe('AddNewGameModalContent', () => { expect( screen.getByTestId('form-input-qualityProfileId') ).toBeInTheDocument(); + expect(screen.getByTestId('form-input-platform')).toBeInTheDocument(); expect(screen.getByTestId('form-input-tags')).toBeInTheDocument(); }); diff --git a/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.tsx b/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.tsx index 1dc46555..37a83d08 100644 --- a/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.tsx +++ b/frontend/src/AddGame/AddNewGame/AddNewGameModalContent.tsx @@ -55,8 +55,26 @@ const platformOptions = [ { key: 'linux', value: 'Linux' }, { key: 'mac', value: 'macOS' }, { key: 'playStation', value: 'PlayStation' }, + { key: 'sonyPS3', value: 'Sony PlayStation 3' }, + { key: 'sonyPSP', value: 'Sony PSP' }, + { key: 'sonyPSVita', value: 'Sony PlayStation Vita' }, { key: 'xbox', value: 'Xbox' }, { key: 'nintendo', value: 'Nintendo' }, + { key: 'nintendoSwitch', value: 'Nintendo Switch' }, + { key: 'nintendoWiiU', value: 'Nintendo Wii U' }, + { key: 'nintendoWii', value: 'Nintendo Wii' }, + { key: 'nintendo3DS', value: 'Nintendo 3DS' }, + { key: 'nintendoDSi', value: 'Nintendo DSi' }, + { key: 'nintendoDS', value: 'Nintendo DS' }, + { key: 'nintendoGBA', value: 'Nintendo Game Boy Advance' }, + { key: 'nintendoGBC', value: 'Nintendo Game Boy Color' }, + { key: 'nintendoGB', value: 'Nintendo Game Boy' }, + { key: 'nintendoNES', value: 'Nintendo Entertainment System' }, + { key: 'nintendoSNES', value: 'Super Nintendo Entertainment System' }, + { key: 'nintendoN64', value: 'Nintendo 64' }, + { key: 'nintendoFDS', value: 'Family Computer Disk System' }, + { key: 'nintendoVirtualBoy', value: 'Virtual Boy' }, + { key: 'nintendoPokemonMini', value: 'Pokemon Mini' }, ]; function AddNewGameModalContent(props: AddNewGameModalContentProps) { diff --git a/frontend/src/DiscoverGame/AddNewDiscoverGameModalContentConnector.tsx b/frontend/src/DiscoverGame/AddNewDiscoverGameModalContentConnector.tsx index a83c71f0..7b964ffc 100644 --- a/frontend/src/DiscoverGame/AddNewDiscoverGameModalContentConnector.tsx +++ b/frontend/src/DiscoverGame/AddNewDiscoverGameModalContentConnector.tsx @@ -17,6 +17,7 @@ interface DiscoverGameDefaults { qualityProfileId: number; minimumAvailability: string; searchForGame: boolean; + platform: string; tags: number[]; } @@ -87,6 +88,7 @@ function AddNewDiscoverGameModalContentConnector({ qualityProfileId, minimumAvailability, searchForGame, + platform, tags, } = settings; @@ -107,6 +109,7 @@ function AddNewDiscoverGameModalContentConnector({ qualityProfileId: qualityProfileId.value, minimumAvailability: minimumAvailability.value, searchForGame: searchForGame.value, + platform: platform.value, tags: tags.value, }) ); @@ -121,6 +124,7 @@ function AddNewDiscoverGameModalContentConnector({ qualityProfileId, minimumAvailability, searchForGame, + platform, tags, onModalClose, ]); @@ -140,6 +144,7 @@ function AddNewDiscoverGameModalContentConnector({ qualityProfileId={qualityProfileId} minimumAvailability={minimumAvailability} searchForGame={searchForGame} + platform={platform} tags={tags} folder={folder} onModalClose={onModalClose} diff --git a/frontend/src/Game/Details/Components/GameComponentsTable.tsx b/frontend/src/Game/Details/Components/GameComponentsTable.tsx index 54b29010..1202330d 100644 --- a/frontend/src/Game/Details/Components/GameComponentsTable.tsx +++ b/frontend/src/Game/Details/Components/GameComponentsTable.tsx @@ -5,6 +5,7 @@ import { GAME_COMPONENT_SEARCH } from 'Commands/commandNames'; import EnhancedSelectInput from 'Components/Form/Select/EnhancedSelectInput'; import Icon from 'Components/Icon'; import Label from 'Components/Label'; +import IconButton from 'Components/Link/IconButton'; import SpinnerIconButton from 'Components/Link/SpinnerIconButton'; import LoadingIndicator from 'Components/Loading/LoadingIndicator'; import MonitorToggleButton from 'Components/MonitorToggleButton'; @@ -18,11 +19,12 @@ import createCommandExecutingSelector from 'Store/Selectors/createCommandExecuti import createAjaxRequest from 'Utilities/createAjaxRequest'; import formatBytes from 'Utilities/Number/formatBytes'; import translate from 'Utilities/String/translate'; +import GameInteractiveSearchModal from '../../Search/GameInteractiveSearchModal'; interface GameComponent { id: number; gameId: number; - componentType: 'base' | 'update' | 'dlc'; + componentType: string; key: string; title: string; monitored: boolean; @@ -30,6 +32,22 @@ interface GameComponent { qualityProfileId: number; hasFile: boolean; sizeOnDisk: number; + releaseGroup?: string; + version?: string; + noIntroCatalogMatches: NoIntroCatalogMatch[]; +} + +interface NoIntroCatalogMatch { + id: number; + catalogSourceId: number; + sourceName?: string; + systemKey: string; + canonicalName: string; + canonicalFileName: string; + catalogVersion?: string; + lastSyncError?: string; + hashType?: string; + hashValue?: string; } const columns = [ @@ -37,6 +55,7 @@ const columns = [ { name: 'title', label: () => translate('Title'), isVisible: true }, { name: 'size', label: () => translate('Size'), isVisible: true }, { name: 'status', label: () => translate('Status'), isVisible: true }, + { name: 'noIntro', label: () => 'Catalog', isVisible: true }, { name: 'qualityProfileId', label: () => translate('QualityProfile'), @@ -47,12 +66,71 @@ const columns = [ ]; const typeKinds: Record< - GameComponent['componentType'], - 'info' | 'success' | 'primary' + string, + 'info' | 'success' | 'primary' | 'warning' | 'danger' > = { base: 'info', update: 'success', dlc: 'primary', + noIntroRetailRom: 'info', + noIntroMultiboot: 'success', + noIntroVideo: 'primary', + noIntroBios: 'warning', + noIntroRomhackOrUnverified: 'danger', +}; + +const typeLabels: Record = { + base: 'BASE', + update: 'UPDATE', + dlc: 'DLC', + noIntroRetailRom: 'REGIONAL VARIANT', + noIntroMultiboot: 'RELEASE VARIANT', + noIntroVideo: 'VIDEO', + noIntroBios: 'BIOS', + noIntroRomhackOrUnverified: 'UNVERIFIED', +}; + +const fallbackTypeLabels: Record = { + nointroretailrom: 'REGIONAL VARIANT', + nointromultiboot: 'RELEASE VARIANT', + nointrovideo: 'VIDEO', + nointrobios: 'BIOS', + nointroromhackorunverified: 'UNVERIFIED', +}; + +function getTypeLabel(componentType: string) { + return ( + typeLabels[componentType] ?? + fallbackTypeLabels[componentType.toLowerCase()] ?? + componentType + ); +} + +function getComponentMeta(component: GameComponent) { + const parts = [component.version, component.releaseGroup].filter( + (value): value is string => value != null && value.trim().length > 0 + ); + + return parts.join(' · '); +} + +const noIntroSystemNames: Record = { + 'nintendo---game-boy': 'Nintendo Game Boy', + 'nintendo---game-boy-color': 'Nintendo Game Boy Color', + 'nintendo---game-boy-advance': 'Nintendo Game Boy Advance', + 'nintendo---game-boy-advance-multiboot': 'Nintendo GBA Multiboot', + 'nintendo---game-boy-advance--multiboot': 'Nintendo GBA Multiboot', + 'nintendo---game-boy-advance-e-reader': 'Nintendo GBA e-Reader', + 'nintendo---game-boy-advance--e-reader': 'Nintendo GBA e-Reader', + 'nintendo---game-boy-advance-play-yan': 'Nintendo GBA Play-Yan', + 'nintendo---game-boy-advance--play-yan': 'Nintendo GBA Play-Yan', + 'nintendo---game-boy-advance-video': 'Nintendo GBA Video', + 'nintendo---game-boy-advance--video': 'Nintendo GBA Video', + 'nintendo---nintendo-ds': 'Nintendo DS', + 'nintendo---nintendo-ds-download-play': 'Nintendo DS Download Play', + 'nintendo---nintendo-ds--download-play': 'Nintendo DS Download Play', + 'nintendo---nintendo-ds-dsvision-sd-cards': 'Nintendo DS DSvision', + 'nintendo---nintendo-ds--dsvision-sd-cards': 'Nintendo DS DSvision', }; interface GameComponentsTableProps { @@ -89,6 +167,47 @@ function getStatusIcon(component: GameComponent) { ); } +function getNoIntroStatus(component: GameComponent) { + const matches = component.noIntroCatalogMatches ?? []; + + if (matches.length === 0) { + return '-'; + } + + const match = matches[0]; + + if (match == null) { + return '-'; + } + + const suffix = matches.length > 1 ? ` +${matches.length - 1}` : ''; + const version = match.catalogVersion ? ` (${match.catalogVersion})` : ''; + const systemName = noIntroSystemNames[match.systemKey] ?? match.systemKey; + const title = [ + `${match.sourceName ?? 'Catalog'}${version}`, + ...matches.flatMap((catalogMatch) => { + const lines = [catalogMatch.canonicalFileName]; + + if (catalogMatch.hashType && catalogMatch.hashValue) { + lines.push( + `${catalogMatch.hashType.toUpperCase()} ${catalogMatch.hashValue}` + ); + } + + return lines; + }), + ].join('\n'); + + return ( + + ); +} + interface GameComponentRowProps { component: GameComponent; isSaving: boolean; @@ -105,6 +224,8 @@ function GameComponentRow({ onProfileChange, }: GameComponentRowProps) { const dispatch = useDispatch(); + const [isInteractiveSearchModalOpen, setIsInteractiveSearchModalOpen] = + useState(false); const isSearching = useSelector( useMemo( @@ -133,6 +254,14 @@ function GameComponentRow({ ); }, [dispatch, component.gameId, component.id]); + const handleInteractiveSearchPress = useCallback(() => { + setIsInteractiveSearchModalOpen(true); + }, []); + + const handleInteractiveSearchModalClose = useCallback(() => { + setIsInteractiveSearchModalOpen(false); + }, []); + const handleProfileChange = useCallback( ({ value }: { value: number }) => { onProfileChange(component.id, value); @@ -143,12 +272,19 @@ function GameComponentRow({ return ( - - {component.title} + +
{component.title}
+ {getComponentMeta(component) ? ( +
+ {getComponentMeta(component)} +
+ ) : null} + {component.hasFile ? formatBytes(component.sizeOnDisk) : '-'} @@ -156,6 +292,8 @@ function GameComponentRow({ {getStatusIcon(component)} + {getNoIntroStatus(component)} + {component.componentType === 'dlc' ? ( + + + + ); @@ -204,6 +356,18 @@ function GameComponentsTable({ gameId }: GameComponentsTableProps) { ); }, [qualityProfiles]); + const visibleComponents = useMemo(() => { + const hasNoIntroVariants = components.some((component) => + component.componentType.startsWith('noIntro') + ); + + if (!hasNoIntroVariants) { + return components; + } + + return components.filter((component) => component.componentType !== 'base'); + }, [components]); + useEffect(() => { let aborted = false; setIsFetching(true); @@ -292,14 +456,14 @@ function GameComponentsTable({ gameId }: GameComponentsTableProps) { return ; } - if (!components.length) { + if (!visibleComponents.length) { return null; } return ( - {components.map((component) => ( + {visibleComponents.map((component) => ( diff --git a/frontend/src/Game/Search/GameInteractiveSearchModalContent.tsx b/frontend/src/Game/Search/GameInteractiveSearchModalContent.tsx index eb6b24dd..53ed1732 100644 --- a/frontend/src/Game/Search/GameInteractiveSearchModalContent.tsx +++ b/frontend/src/Game/Search/GameInteractiveSearchModalContent.tsx @@ -19,11 +19,15 @@ import translate from 'Utilities/String/translate'; export interface GameInteractiveSearchModalContentProps { gameId: number; + componentId?: number; + componentTitle?: string; onModalClose(): void; } function GameInteractiveSearchModalContent({ gameId, + componentId, + componentTitle, onModalClose, }: GameInteractiveSearchModalContentProps) { const dispatch = useDispatch(); @@ -41,19 +45,22 @@ function GameInteractiveSearchModalContent({ }, [dispatch]); const gameTitle = `${title}${year > 0 ? ` (${year})` : ''}`; + const modalTitle = componentTitle + ? `${gameTitle} - ${componentTitle}` + : gameTitle; return ( - {gameTitle + {modalTitle ? translate('InteractiveSearchModalHeaderTitle', { - title: gameTitle, + title: modalTitle, }) : translate('InteractiveSearchModalHeader')} - + diff --git a/frontend/src/GameFile/Editor/GameFileEditorRow.css b/frontend/src/GameFile/Editor/GameFileEditorRow.css index 9e5b5032..f8c97406 100644 --- a/frontend/src/GameFile/Editor/GameFileEditorRow.css +++ b/frontend/src/GameFile/Editor/GameFileEditorRow.css @@ -4,6 +4,23 @@ word-break: break-all; } +.component, +.componentContent, +.componentTitle { + composes: cell from '~Components/Table/Cells/TableRowCell.css'; +} + +.componentContent { + display: flex; + align-items: flex-start; + flex-direction: column; + gap: 4px; +} + +.componentTitle { + word-break: break-word; +} + .quality, .formats, .language { diff --git a/frontend/src/GameFile/Editor/GameFileEditorRow.css.d.ts b/frontend/src/GameFile/Editor/GameFileEditorRow.css.d.ts index b1f761e1..5ebda4e1 100644 --- a/frontend/src/GameFile/Editor/GameFileEditorRow.css.d.ts +++ b/frontend/src/GameFile/Editor/GameFileEditorRow.css.d.ts @@ -3,6 +3,9 @@ interface CssExports { 'actions': string; 'age': string; + 'component': string; + 'componentContent': string; + 'componentTitle': string; 'customFormatScore': string; 'dateAdded': string; 'download': string; diff --git a/frontend/src/GameFile/Editor/GameFileEditorRow.tsx b/frontend/src/GameFile/Editor/GameFileEditorRow.tsx index 0b7e61f8..0d082600 100644 --- a/frontend/src/GameFile/Editor/GameFileEditorRow.tsx +++ b/frontend/src/GameFile/Editor/GameFileEditorRow.tsx @@ -1,4 +1,5 @@ import { useCallback, useState } from 'react'; +import Label from 'Components/Label'; import IconButton from 'Components/Link/IconButton'; import ConfirmModal from 'Components/Modal/ConfirmModal'; import RelativeDateCell from 'Components/Table/Cells/RelativeDateCell'; @@ -14,9 +15,22 @@ import translate from 'Utilities/String/translate'; import FileDetailsModal from '../FileDetailsModal'; import styles from './GameFileEditorRow.css'; +type ComponentLabelKind = + | 'warning' + | 'danger' + | 'default' + | 'disabled' + | 'info' + | 'inverse' + | 'primary' + | 'success' + | 'queue'; + interface GameFileEditorRowProps { id: number; path?: string; + componentType: string; + componentTitle?: string; size: number; relativePath: string; sceneName?: string; @@ -32,6 +46,8 @@ function GameFileEditorRow(props: GameFileEditorRowProps) { const { id, path, + componentType, + componentTitle, relativePath, size, sceneName, @@ -82,6 +98,38 @@ function GameFileEditorRow(props: GameFileEditorRowProps) { path?.split(/[/\\]/).filter(Boolean).pop() || translate('GameFolder'); + const componentKinds: Record = { + base: kinds.INFO, + update: kinds.SUCCESS, + dlc: kinds.PRIMARY, + noIntroRetailRom: kinds.INFO, + noIntroMultiboot: kinds.SUCCESS, + noIntroVideo: kinds.PRIMARY, + noIntroBios: kinds.WARNING, + noIntroRomhackOrUnverified: kinds.DANGER, + file: kinds.DEFAULT, + }; + + const componentLabels: Record = { + base: 'BASE', + update: 'UPDATE', + dlc: 'DLC', + noIntroRetailRom: 'REGIONAL VARIANT', + noIntroMultiboot: 'RELEASE VARIANT', + noIntroVideo: 'VIDEO', + noIntroBios: 'BIOS', + noIntroRomhackOrUnverified: 'UNVERIFIED', + file: 'FILE', + }; + + const fallbackComponentLabels: Record = { + nointroretailrom: 'REGIONAL VARIANT', + nointromultiboot: 'RELEASE VARIANT', + nointrovideo: 'VIDEO', + nointrobios: 'BIOS', + nointroromhackorunverified: 'UNVERIFIED', + }; + return ( {columns.map((column) => { @@ -123,6 +171,26 @@ function GameFileEditorRow(props: GameFileEditorRowProps) { ); } + if (name === 'componentTitle') { + if (!componentTitle) { + return -; + } + + return ( + +
+ + + {componentTitle} +
+ + ); + } + if (name === 'languages') { return ( diff --git a/frontend/src/GameFile/GameFile.ts b/frontend/src/GameFile/GameFile.ts index 20d39633..be1796a1 100644 --- a/frontend/src/GameFile/GameFile.ts +++ b/frontend/src/GameFile/GameFile.ts @@ -7,6 +7,9 @@ export interface GameFile extends ModelBase { gameId: number; relativePath: string; path: string; + componentType: string; + componentKey?: string; + componentTitle?: string; size: number; dateAdded: string; sceneName: string; diff --git a/frontend/src/InteractiveSearch/InteractiveSearchPayload.ts b/frontend/src/InteractiveSearch/InteractiveSearchPayload.ts index 7217185c..6c96bf19 100644 --- a/frontend/src/InteractiveSearch/InteractiveSearchPayload.ts +++ b/frontend/src/InteractiveSearch/InteractiveSearchPayload.ts @@ -1,5 +1,6 @@ interface GameSearchPayload { gameId: number; + componentId?: number; } type InteractiveSearchPayload = GameSearchPayload; diff --git a/frontend/src/Settings/MediaManagement/Naming/Naming.tsx b/frontend/src/Settings/MediaManagement/Naming/Naming.tsx index c76ba4e3..57150424 100644 --- a/frontend/src/Settings/MediaManagement/Naming/Naming.tsx +++ b/frontend/src/Settings/MediaManagement/Naming/Naming.tsx @@ -126,6 +126,18 @@ function Naming() { }, ]; + const renameProfileOptions = [ + { key: 'gamarr', value: translate('RenameProfileGamarr') }, + { + key: 'noIntroPreserveById', + value: translate('RenameProfileNoIntroNumbered'), + }, + { + key: 'noIntroCanonical', + value: translate('RenameProfileNoIntroCanonical'), + }, + ]; + const standardGameFormatHelpTexts = []; const standardGameFormatErrors = []; const gameFolderFormatHelpTexts = []; @@ -175,6 +187,19 @@ function Naming() { /> + + {translate('RenameProfile')} + + + + {translate('ReplaceIllegalCharacters')} diff --git a/frontend/src/Settings/Metadata/Options/MetadataOptions.tsx b/frontend/src/Settings/Metadata/Options/MetadataOptions.tsx index d6e46488..ccf1ac90 100644 --- a/frontend/src/Settings/Metadata/Options/MetadataOptions.tsx +++ b/frontend/src/Settings/Metadata/Options/MetadataOptions.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import Alert from 'Components/Alert'; import FieldSet from 'Components/FieldSet'; @@ -20,6 +20,7 @@ import { OnChildStateChange, SetChildSave, } from 'typings/Settings/SettingsState'; +import createAjaxRequest from 'Utilities/createAjaxRequest'; import translate from 'Utilities/String/translate'; const SECTION = 'metadataOptions'; @@ -45,11 +46,37 @@ interface MetadataOptionsProps { onChildStateChange: OnChildStateChange; } +interface NoIntroCatalogSourceStatus { + id: number; + name: string; + catalogVersion?: string; + lastSuccessfulSync?: string; + lastAttemptedSync?: string; + lastSyncError?: string; + entryCount: number; +} + +interface NoIntroCatalogStatus { + sources: NoIntroCatalogSourceStatus[]; +} + +function fetchNoIntroCatalogStatus() { + return createAjaxRequest({ + url: '/romcatalog/status', + dataType: 'json', + }).request; +} + function MetadataOptions({ setChildSave, onChildStateChange, }: MetadataOptionsProps) { const dispatch = useDispatch(); + const [noIntroStatus, setNoIntroStatus] = + useState(null); + const [noIntroStatusError, setNoIntroStatusError] = useState( + null + ); const { isFetching, isPopulated, @@ -72,6 +99,19 @@ function MetadataOptions({ setChildSave(() => dispatch(saveMetadataOptions())); }, [dispatch, setChildSave]); + useEffect(() => { + fetchNoIntroCatalogStatus() + .done(setNoIntroStatus) + .fail((error: unknown) => { + if (error instanceof Error) { + setNoIntroStatusError(error.message); + return; + } + + setNoIntroStatusError('Unable to load No-Intro catalog status'); + }); + }, []); + useEffect(() => { onChildStateChange({ isSaving, @@ -147,6 +187,38 @@ function MetadataOptions({ {...settings.rawgApiKey} /> + + {noIntroStatusError ? ( + + No-Intro DB + +
{noIntroStatusError}
+ + ) : null} + + {noIntroStatus ? ( + + No-Intro DB + +
+ {noIntroStatus.sources.map((source) => { + const version = source.catalogVersion ?? 'unknown version'; + const synced = source.lastSuccessfulSync + ? new Date(source.lastSuccessfulSync).toLocaleString() + : 'never synced'; + const status = `${source.name}: ${version}, ${source.entryCount} entries, synced ${synced}`; + + return ( +
+ {source.lastSyncError + ? `${status} (${source.lastSyncError})` + : status} +
+ ); + })} +
+ + ) : null} ) : null} diff --git a/frontend/src/Settings/Profiles/Quality/EditQualityProfileModalContent.tsx b/frontend/src/Settings/Profiles/Quality/EditQualityProfileModalContent.tsx index 37091a98..43e80304 100644 --- a/frontend/src/Settings/Profiles/Quality/EditQualityProfileModalContent.tsx +++ b/frontend/src/Settings/Profiles/Quality/EditQualityProfileModalContent.tsx @@ -29,8 +29,26 @@ const platformOptions = [ { key: 8, value: 'Linux', order: 2 }, { key: 9, value: 'Mac', order: 3 }, { key: 2, value: 'PlayStation', order: 4 }, + { key: 25, value: 'Sony PlayStation 3', order: 4.1 }, + { key: 26, value: 'Sony PSP', order: 4.2 }, + { key: 27, value: 'Sony PlayStation Vita', order: 4.3 }, { key: 3, value: 'Xbox', order: 5 }, { key: 4, value: 'Nintendo', order: 6 }, + { key: 10, value: 'Nintendo Switch', order: 7 }, + { key: 11, value: 'Nintendo Wii U', order: 8 }, + { key: 12, value: 'Nintendo Wii', order: 9 }, + { key: 13, value: 'Nintendo 3DS', order: 10 }, + { key: 24, value: 'Nintendo DSi', order: 10.5 }, + { key: 14, value: 'Nintendo DS', order: 11 }, + { key: 15, value: 'Nintendo Game Boy Advance', order: 12 }, + { key: 17, value: 'Nintendo Game Boy Color', order: 13 }, + { key: 16, value: 'Nintendo Game Boy', order: 14 }, + { key: 18, value: 'Nintendo Entertainment System', order: 15 }, + { key: 19, value: 'Super Nintendo Entertainment System', order: 16 }, + { key: 20, value: 'Nintendo 64', order: 17 }, + { key: 21, value: 'Family Computer Disk System', order: 18 }, + { key: 22, value: 'Virtual Boy', order: 19 }, + { key: 23, value: 'Pokemon Mini', order: 20 }, ]; interface PendingValue { diff --git a/frontend/src/Store/Actions/discoverGameActions.ts b/frontend/src/Store/Actions/discoverGameActions.ts index d630be8d..d789fb8d 100644 --- a/frontend/src/Store/Actions/discoverGameActions.ts +++ b/frontend/src/Store/Actions/discoverGameActions.ts @@ -41,6 +41,7 @@ interface DiscoverGameDefaults { qualityProfileId: number; minimumAvailability: string; searchForGame: boolean; + platform: string; tags: number[]; } @@ -178,6 +179,7 @@ export const defaultState: DiscoverGameState = { qualityProfileId: 0, minimumAvailability: 'released', searchForGame: true, + platform: 'unknown', tags: [], }, diff --git a/frontend/src/Store/Actions/gameFileActions.ts b/frontend/src/Store/Actions/gameFileActions.ts index 9eff99f3..4da7c23b 100644 --- a/frontend/src/Store/Actions/gameFileActions.ts +++ b/frontend/src/Store/Actions/gameFileActions.ts @@ -82,6 +82,12 @@ export const defaultState = { isVisible: true, isSortable: true, }, + { + name: 'componentTitle', + label: () => 'Component', + isVisible: true, + isSortable: true, + }, { name: 'size', label: () => translate('Size'), diff --git a/frontend/src/typings/RomCatalog/NoIntroRomCatalog.test.ts b/frontend/src/typings/RomCatalog/NoIntroRomCatalog.test.ts new file mode 100644 index 00000000..81faa65c --- /dev/null +++ b/frontend/src/typings/RomCatalog/NoIntroRomCatalog.test.ts @@ -0,0 +1,46 @@ +import { + type NoIntroCatalogPlan, + noIntroRomComponentTypes, + noIntroVerificationStatuses, +} from './NoIntroRomCatalog'; + +describe('NoIntroRomCatalog', () => { + it('keeps Download Play as a mapped component and standalone products separate', () => { + const plan: NoIntroCatalogPlan = { + games: [ + { + systemKey: 'nintendo-ds', + gameTitle: 'Mario Kart DS', + regionLanguageComponents: [ + { + slotLabel: 'USA', + canonicalName: 'Mario Kart DS (USA)', + componentType: 'retailRom', + }, + ], + downloadPlayComponents: [ + { + slotLabel: 'Download Play', + canonicalName: 'Mario Kart DS (Download Play)', + componentType: 'multiboot', + }, + ], + }, + ], + standaloneGames: [ + { + title: + 'Game Boy Advance Video - Cartoon Network Collection - Volume 1 (USA)', + componentType: 'video', + }, + ], + }; + + expect(noIntroVerificationStatuses).toContain('nameMismatch'); + expect(noIntroRomComponentTypes).toContain('multiboot'); + expect(plan.games[0].downloadPlayComponents[0].slotLabel).toBe( + 'Download Play' + ); + expect(plan.standaloneGames[0].componentType).toBe('video'); + }); +}); diff --git a/frontend/src/typings/RomCatalog/NoIntroRomCatalog.ts b/frontend/src/typings/RomCatalog/NoIntroRomCatalog.ts new file mode 100644 index 00000000..d5c35b63 --- /dev/null +++ b/frontend/src/typings/RomCatalog/NoIntroRomCatalog.ts @@ -0,0 +1,54 @@ +export const noIntroVerificationStatuses = [ + 'verified', + 'nameMismatch', + 'unknown', + 'badDump', + 'missing', + 'duplicate', +] as const; + +export type NoIntroVerificationStatus = + (typeof noIntroVerificationStatuses)[number]; + +export const noIntroRomComponentTypes = [ + 'retailRom', + 'eReaderCards', + 'multiboot', + 'video', + 'bios', + 'romhackOrUnverified', +] as const; + +export type NoIntroRomComponentType = (typeof noIntroRomComponentTypes)[number]; + +export interface NoIntroCatalogComponentSlot { + readonly slotLabel: string; + readonly canonicalName: string; + readonly componentType: NoIntroRomComponentType; +} + +export interface NoIntroCatalogGamePlan { + readonly systemKey: string; + readonly gameTitle: string; + readonly regionLanguageComponents: readonly NoIntroCatalogComponentSlot[]; + readonly downloadPlayComponents: readonly NoIntroCatalogComponentSlot[]; +} + +export interface NoIntroCatalogStandalonePlan { + readonly title: string; + readonly componentType: NoIntroRomComponentType; +} + +export interface NoIntroCatalogPlan { + readonly games: readonly NoIntroCatalogGamePlan[]; + readonly standaloneGames: readonly NoIntroCatalogStandalonePlan[]; +} + +export interface NoIntroVerificationResult { + readonly id: number; + readonly verificationStatus: NoIntroVerificationStatus; + readonly actualFileName: string; + readonly expectedFileName?: string; + readonly isDuplicate: boolean; + readonly isMissing: boolean; +} diff --git a/frontend/src/typings/Settings/NamingConfig.ts b/frontend/src/typings/Settings/NamingConfig.ts index 3e1de019..c5975bd5 100644 --- a/frontend/src/typings/Settings/NamingConfig.ts +++ b/frontend/src/typings/Settings/NamingConfig.ts @@ -1,8 +1,11 @@ type ColonReplacementFormat = 'delete' | 'dash' | 'spaceDash' | 'spaceDashSpace' | 'smart'; +type RenameProfile = 'gamarr' | 'noIntroPreserveById' | 'noIntroCanonical'; + export default interface NamingConfig { renameGames: boolean; + renameProfile: RenameProfile; replaceIllegalCharacters: boolean; colonReplacementFormat: ColonReplacementFormat; standardGameFormat: string; diff --git a/src/Gamarr.Api.V3/Config/NamingConfigResource.cs b/src/Gamarr.Api.V3/Config/NamingConfigResource.cs index 1eb2fabf..dac45012 100644 --- a/src/Gamarr.Api.V3/Config/NamingConfigResource.cs +++ b/src/Gamarr.Api.V3/Config/NamingConfigResource.cs @@ -6,6 +6,7 @@ namespace Gamarr.Api.V3.Config public class NamingConfigResource : RestResource { public bool RenameGames { get; set; } + public RenameProfile RenameProfile { get; set; } public bool ReplaceIllegalCharacters { get; set; } public ColonReplacementFormat ColonReplacementFormat { get; set; } public string StandardGameFormat { get; set; } diff --git a/src/Gamarr.Api.V3/Config/NamingExampleResource.cs b/src/Gamarr.Api.V3/Config/NamingExampleResource.cs index c03d10a7..d1368642 100644 --- a/src/Gamarr.Api.V3/Config/NamingExampleResource.cs +++ b/src/Gamarr.Api.V3/Config/NamingExampleResource.cs @@ -17,6 +17,7 @@ public static NamingConfigResource ToResource(this NamingConfig model) Id = model.Id, RenameGames = model.RenameGames, + RenameProfile = model.RenameProfile, ReplaceIllegalCharacters = model.ReplaceIllegalCharacters, ColonReplacementFormat = model.ColonReplacementFormat, StandardGameFormat = model.StandardGameFormat, @@ -31,6 +32,7 @@ public static NamingConfig ToModel(this NamingConfigResource resource) Id = resource.Id, RenameGames = resource.RenameGames, + RenameProfile = resource.RenameProfile, ReplaceIllegalCharacters = resource.ReplaceIllegalCharacters, ColonReplacementFormat = resource.ColonReplacementFormat, StandardGameFormat = resource.StandardGameFormat, diff --git a/src/Gamarr.Api.V3/GameComponents/GameComponentController.cs b/src/Gamarr.Api.V3/GameComponents/GameComponentController.cs index 8dcfe35b..42369b08 100644 --- a/src/Gamarr.Api.V3/GameComponents/GameComponentController.cs +++ b/src/Gamarr.Api.V3/GameComponents/GameComponentController.cs @@ -1,9 +1,13 @@ using System.Collections.Generic; using System.Linq; using Microsoft.AspNetCore.Mvc; +using NzbDrone.Common.Disk; +using NLog; +using NzbDrone.Core.Games; using NzbDrone.Core.Games.Components; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Profiles.Qualities; +using NzbDrone.Core.RomCatalog; using Gamarr.Http; using Gamarr.Http.REST; @@ -14,15 +18,33 @@ public class GameComponentController : Controller { private readonly IGameComponentService _componentService; private readonly IMediaFileService _mediaFileService; + private readonly IGameService _gameService; private readonly IQualityProfileService _qualityProfileService; + private readonly INoIntroCatalogEntryRepository _noIntroCatalogEntryRepository; + private readonly INoIntroCatalogSourceRepository _noIntroCatalogSourceRepository; + private readonly INoIntroCatalogHashRepository _noIntroCatalogHashRepository; + private readonly IDiskProvider _diskProvider; + private readonly Logger _logger; public GameComponentController(IGameComponentService componentService, - IMediaFileService mediaFileService, - IQualityProfileService qualityProfileService) + IMediaFileService mediaFileService, + IGameService gameService, + IQualityProfileService qualityProfileService, + INoIntroCatalogEntryRepository noIntroCatalogEntryRepository, + INoIntroCatalogSourceRepository noIntroCatalogSourceRepository, + INoIntroCatalogHashRepository noIntroCatalogHashRepository, + IDiskProvider diskProvider, + Logger logger) { _componentService = componentService; _mediaFileService = mediaFileService; + _gameService = gameService; _qualityProfileService = qualityProfileService; + _noIntroCatalogEntryRepository = noIntroCatalogEntryRepository; + _noIntroCatalogSourceRepository = noIntroCatalogSourceRepository; + _noIntroCatalogHashRepository = noIntroCatalogHashRepository; + _diskProvider = diskProvider; + _logger = logger; } [HttpGet] @@ -35,11 +57,22 @@ public List GetComponents([FromQuery] int gameId) } var files = _mediaFileService.GetFilesByGame(gameId); + var noIntroEntries = _noIntroCatalogEntryRepository.All().ToList(); + var noIntroSources = _noIntroCatalogSourceRepository.All().ToList(); + var noIntroHashes = _noIntroCatalogHashRepository.GetByEntryIds(noIntroEntries.Select(x => x.Id).ToList()); + var game = _gameService.GetGame(gameId); + var context = new GameComponentNoIntroCatalogContext + { + GameFiles = files, + Entries = noIntroEntries, + Sources = noIntroSources, + HashMatches = GetFileHashMatches(files, game, noIntroHashes) + }; return _componentService.GetByGame(gameId) .OrderBy(c => c.ComponentType) .ThenBy(c => c.Title) - .Select(c => c.ToResource(files)) + .Select(c => c.ToResource(context)) .ToList(); } @@ -56,7 +89,73 @@ public GameComponentResource SetComponent(int id, [FromBody] GameComponentResour var component = _componentService.SetComponentOptions(id, resource.Monitored, resource.QualityProfileId); var files = _mediaFileService.GetFilesByGame(component.GameId); - return component.ToResource(files); + return component.ToResource(new GameComponentNoIntroCatalogContext { GameFiles = files }); + } + + private List GetFileHashMatches(List files, Game game, List catalogHashes) + { + var hashByKey = catalogHashes + .GroupBy(x => $"{x.HashType}:{x.HashValue}".ToLowerInvariant()) + .ToDictionary(x => x.Key, x => x.First()); + var matches = new List(); + + foreach (var file in files.Where(x => x.ComponentId > 0)) + { + var path = file.GetPath(game); + + if (!_diskProvider.FileExists(path)) + { + continue; + } + + NoIntroHashTriplet hashes; + + try + { + using var stream = _diskProvider.OpenReadStream(path); + hashes = NoIntroRomHasher.Compute(stream); + } + catch (global::System.Exception ex) + { + _logger.Warn(ex, "Failed hashing game file {0} for component inspection", path); + continue; + } + + var matchedHash = FindMatch(hashes, hashByKey); + + if (matchedHash == null) + { + continue; + } + + matches.Add(new NoIntroCatalogFileHashMatch + { + GameFileId = file.Id, + CatalogEntryId = matchedHash.CatalogEntryId, + HashType = matchedHash.HashType, + HashValue = matchedHash.HashValue + }); + } + + return matches; + } + + private static NoIntroCatalogHash FindMatch(NoIntroHashTriplet hashes, Dictionary catalogHashes) + { + return TryGetHash("sha1", hashes.Sha1, catalogHashes) ?? + TryGetHash("md5", hashes.Md5, catalogHashes) ?? + TryGetHash("crc32", hashes.Crc32, catalogHashes); + } + + private static NoIntroCatalogHash TryGetHash(string hashType, string hashValue, Dictionary catalogHashes) + { + if (string.IsNullOrWhiteSpace(hashValue)) + { + return null; + } + + catalogHashes.TryGetValue($"{hashType}:{hashValue}".ToLowerInvariant(), out var matchedHash); + return matchedHash; } } } diff --git a/src/Gamarr.Api.V3/GameComponents/GameComponentResource.cs b/src/Gamarr.Api.V3/GameComponents/GameComponentResource.cs index f74eeeaa..487c5d31 100644 --- a/src/Gamarr.Api.V3/GameComponents/GameComponentResource.cs +++ b/src/Gamarr.Api.V3/GameComponents/GameComponentResource.cs @@ -2,6 +2,7 @@ using System.Linq; using NzbDrone.Core.Games.Components; using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.RomCatalog; using Gamarr.Http.REST; namespace Gamarr.Api.V3.GameComponents @@ -22,13 +23,46 @@ public class GameComponentResource : RestResource // component without a file is "missing". public bool HasFile { get; set; } public long SizeOnDisk { get; set; } + public string ReleaseGroup { get; set; } + public string Version { get; set; } + public List NoIntroCatalogMatches { get; set; } = new List(); + } + + public class GameComponentNoIntroCatalogResource : RestResource + { + public int CatalogSourceId { get; set; } + public string SourceName { get; set; } + public string SystemKey { get; set; } + public string CanonicalName { get; set; } + public string CanonicalFileName { get; set; } + public string CatalogVersion { get; set; } + public string LastSyncError { get; set; } + public string HashType { get; set; } + public string HashValue { get; set; } + } + + public class GameComponentNoIntroCatalogContext + { + public List GameFiles { get; set; } = new List(); + public List Entries { get; set; } = new List(); + public List Sources { get; set; } = new List(); + public List HashMatches { get; set; } = new List(); + } + + public class NoIntroCatalogFileHashMatch + { + public int GameFileId { get; set; } + public int CatalogEntryId { get; set; } + public string HashType { get; set; } + public string HashValue { get; set; } } public static class GameComponentResourceMapper { - public static GameComponentResource ToResource(this GameComponent model, List gameFiles) + public static GameComponentResource ToResource(this GameComponent model, GameComponentNoIntroCatalogContext context = null) { - var files = gameFiles?.Where(f => f.ComponentId == model.Id).ToList() ?? new List(); + context ??= new GameComponentNoIntroCatalogContext(); + var files = context.GameFiles.Where(f => f.ComponentId == model.Id).ToList(); return new GameComponentResource { @@ -41,8 +75,101 @@ public static GameComponentResource ToResource(this GameComponent model, List f.Size) + SizeOnDisk = files.Sum(f => f.Size), + ReleaseGroup = JoinDistinct(files.Select(f => f.ReleaseGroup)), + Version = JoinDistinct(files.Select(f => f.GameVersion?.ToString())), + NoIntroCatalogMatches = GetNoIntroCatalogMatches(model, files, context) }; } + + private static string JoinDistinct(IEnumerable values) + { + var distinct = values + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct() + .ToList(); + + return distinct.Count == 0 ? null : string.Join(", ", distinct); + } + + private static List GetNoIntroCatalogMatches(GameComponent component, List files, GameComponentNoIntroCatalogContext context) + { + if (context.Entries.Count == 0 || string.IsNullOrWhiteSpace(component.Title)) + { + return new List(); + } + + var sourceById = context.Sources.ToDictionary(x => x.Id); + var exactMatches = GetExactHashMatches(files, context); + var entries = exactMatches.Count > 0 ? exactMatches.Select(x => x.Entry) : context.Entries.Where(entry => IsComponentCatalogMatch(component, entry)); + + return entries + .OrderBy(entry => entry.SystemKey) + .ThenBy(entry => entry.CanonicalName) + .Take(20) + .Select(entry => + { + sourceById.TryGetValue(entry.CatalogSourceId, out var source); + var hashMatch = exactMatches + .Where(x => x.Entry.Id == entry.Id) + .Select(x => x.HashMatch) + .FirstOrDefault(); + + return new GameComponentNoIntroCatalogResource + { + Id = entry.Id, + CatalogSourceId = entry.CatalogSourceId, + SourceName = source?.Name, + SystemKey = entry.SystemKey, + CanonicalName = entry.CanonicalName, + CanonicalFileName = entry.CanonicalFileName, + CatalogVersion = source?.CatalogVersion, + LastSyncError = source?.LastSyncError, + HashType = hashMatch?.HashType, + HashValue = hashMatch?.HashValue + }; + }) + .ToList(); + } + + private static List<(NoIntroCatalogEntry Entry, NoIntroCatalogFileHashMatch HashMatch)> GetExactHashMatches(List files, GameComponentNoIntroCatalogContext context) + { + var fileIds = files.Select(x => x.Id).ToHashSet(); + var entryById = context.Entries.ToDictionary(x => x.Id); + + return context.HashMatches + .Where(match => fileIds.Contains(match.GameFileId) && entryById.ContainsKey(match.CatalogEntryId)) + .Select(match => (entryById[match.CatalogEntryId], match)) + .ToList(); + } + + private static bool IsComponentCatalogMatch(GameComponent component, NoIntroCatalogEntry entry) + { + if (entry == null || string.IsNullOrWhiteSpace(entry.CanonicalName)) + { + return false; + } + + if (IsNoIntroComponent(component)) + { + return component.Key?.EndsWith($":{NzbDrone.Core.Parser.Parser.ToUrlSlug(entry.CanonicalName, true)}", global::System.StringComparison.Ordinal) == true; + } + + if (!string.IsNullOrWhiteSpace(entry.ParentCanonicalName) && entry.ParentCanonicalName == component.Title) + { + return true; + } + + return entry.CanonicalName == component.Title || entry.CanonicalName.StartsWith($"{component.Title} (", global::System.StringComparison.Ordinal); + } + + private static bool IsNoIntroComponent(GameComponent component) + { + return component.ComponentType is GameComponentType.NoIntroRetailRom or + GameComponentType.NoIntroMultiboot or + GameComponentType.NoIntroVideo or + GameComponentType.NoIntroBios or + GameComponentType.NoIntroRomhackOrUnverified; + } } } diff --git a/src/Gamarr.Api.V3/GameFiles/GameFileController.cs b/src/Gamarr.Api.V3/GameFiles/GameFileController.cs index 3e5d0a20..524ac1c5 100644 --- a/src/Gamarr.Api.V3/GameFiles/GameFileController.cs +++ b/src/Gamarr.Api.V3/GameFiles/GameFileController.cs @@ -11,6 +11,7 @@ using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Games; +using NzbDrone.Core.Games.Components; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Qualities; @@ -30,20 +31,23 @@ public class GameFileController : RestControllerWithSignalR x.Id); + componentsById.TryGetValue(gameFile.ComponentId, out var component); - var resource = gameFile.ToResource(game, _upgradableSpecification, _formatCalculator); + var resource = gameFile.ToResource(game, _upgradableSpecification, _formatCalculator, component); return resource; } @@ -77,8 +83,17 @@ public List GetGameFiles([FromQuery(Name = "gameId")] List e.GameId) - .SelectMany(f => f.ToList() - .ConvertAll(e => e.ToResource(_gameService.GetGame(f.Key), _upgradableSpecification, _formatCalculator))) + .SelectMany(group => + { + var game = _gameService.GetGame(group.Key); + var componentsById = _gameComponentService.GetByGame(group.Key).ToDictionary(x => x.Id); + + return group.Select(file => + { + componentsById.TryGetValue(file.ComponentId, out var component); + return file.ToResource(game, _upgradableSpecification, _formatCalculator, component); + }); + }) .ToList(); } @@ -206,7 +221,13 @@ public object SetPropertiesBulk([FromBody] List resources) var game = _gameService.GetGame(gameFiles.First().GameId); - return Accepted(gameFiles.ConvertAll(f => f.ToResource(game, _upgradableSpecification, _formatCalculator))); + var componentsById = _gameComponentService.GetByGame(game.Id).ToDictionary(x => x.Id); + + return Accepted(gameFiles.ConvertAll(f => + { + componentsById.TryGetValue(f.ComponentId, out var component); + return f.ToResource(game, _upgradableSpecification, _formatCalculator, component); + })); } [NonAction] diff --git a/src/Gamarr.Api.V3/GameFiles/GameFileResource.cs b/src/Gamarr.Api.V3/GameFiles/GameFileResource.cs index 301f628e..ac55a58c 100644 --- a/src/Gamarr.Api.V3/GameFiles/GameFileResource.cs +++ b/src/Gamarr.Api.V3/GameFiles/GameFileResource.cs @@ -3,6 +3,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Core.CustomFormats; using NzbDrone.Core.DecisionEngine.Specifications; +using NzbDrone.Core.Games.Components; using NzbDrone.Core.Languages; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Qualities; @@ -23,10 +24,9 @@ public class GameFileResource : RestResource public string Edition { get; set; } public string Version { get; set; } - // Derived component classification (#149): "base" (the game folder - // itself), "update" (Updates/ unit), "dlc" (DLC/ unit), - // or "file" (legacy file-based record). public string ComponentType { get; set; } + public string ComponentKey { get; set; } + public string ComponentTitle { get; set; } public List Languages { get; set; } public QualityModel Quality { get; set; } public List CustomFormats { get; set; } @@ -40,7 +40,7 @@ public class GameFileResource : RestResource public static class GameFileResourceMapper { - private static GameFileResource ToResource(this GameFile model) + private static GameFileResource ToResource(this GameFile model, GameComponent component = null) { if (model == null) { @@ -64,14 +64,32 @@ private static GameFileResource ToResource(this GameFile model) ReleaseGroup = model.ReleaseGroup, Edition = model.Edition, Version = model.GameVersion?.ToString(), - ComponentType = GetComponentType(model.RelativePath), + ComponentType = GetComponentType(model.RelativePath, component), + ComponentKey = component?.Key, + ComponentTitle = component?.Title, MediaInfo = model.MediaInfo.ToResource(model.SceneName), OriginalFilePath = model.OriginalFilePath }; } - private static string GetComponentType(string relativePath) + private static string GetComponentType(string relativePath, GameComponent component = null) { + if (component != null) + { + return component.ComponentType switch + { + GameComponentType.Base => "base", + GameComponentType.Update => "update", + GameComponentType.Dlc => "dlc", + GameComponentType.NoIntroRetailRom => "noIntroRetailRom", + GameComponentType.NoIntroMultiboot => "noIntroMultiboot", + GameComponentType.NoIntroVideo => "noIntroVideo", + GameComponentType.NoIntroBios => "noIntroBios", + GameComponentType.NoIntroRomhackOrUnverified => "noIntroRomhackOrUnverified", + _ => "file" + }; + } + if (relativePath.IsNullOrWhiteSpace()) { return "base"; @@ -90,7 +108,7 @@ private static string GetComponentType(string relativePath) return "file"; } - public static GameFileResource ToResource(this GameFile model, NzbDrone.Core.Games.Game game, IUpgradableSpecification upgradableSpecification, ICustomFormatCalculationService formatCalculationService) + public static GameFileResource ToResource(this GameFile model, NzbDrone.Core.Games.Game game, IUpgradableSpecification upgradableSpecification, ICustomFormatCalculationService formatCalculationService, GameComponent component = null) { if (model == null) { @@ -112,7 +130,9 @@ public static GameFileResource ToResource(this GameFile model, NzbDrone.Core.Gam Edition = model.Edition, ReleaseGroup = model.ReleaseGroup, Version = model.GameVersion?.ToString(), - ComponentType = GetComponentType(model.RelativePath), + ComponentType = GetComponentType(model.RelativePath, component), + ComponentKey = component?.Key, + ComponentTitle = component?.Title, MediaInfo = model.MediaInfo.ToResource(model.SceneName), QualityCutoffNotMet = upgradableSpecification?.QualityCutoffNotMet(game.QualityProfile, model.Quality) ?? false, OriginalFilePath = model.OriginalFilePath, diff --git a/src/Gamarr.Api.V3/RomCatalog/NoIntroCatalogController.cs b/src/Gamarr.Api.V3/RomCatalog/NoIntroCatalogController.cs new file mode 100644 index 00000000..ca18eb2f --- /dev/null +++ b/src/Gamarr.Api.V3/RomCatalog/NoIntroCatalogController.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Mvc; +using Gamarr.Http; +using NzbDrone.Core.RomCatalog; + +namespace Gamarr.Api.V3.RomCatalog +{ + [V3ApiController("romcatalog")] + public class NoIntroCatalogController : Controller + { + private readonly INoIntroCatalogSourceRepository _sourceRepository; + private readonly INoIntroCatalogEntryRepository _entryRepository; + private readonly INoIntroVerificationResultRepository _resultRepository; + private readonly INoIntroComponentClassifier _componentClassifier; + + public NoIntroCatalogController( + INoIntroCatalogSourceRepository sourceRepository, + INoIntroCatalogEntryRepository entryRepository, + INoIntroVerificationResultRepository resultRepository, + INoIntroComponentClassifier componentClassifier) + { + _sourceRepository = sourceRepository; + _entryRepository = entryRepository; + _resultRepository = resultRepository; + _componentClassifier = componentClassifier; + } + + [HttpGet("source")] + [Produces("application/json")] + public List GetSources() + { + return _sourceRepository.All().Select(x => x.ToResource()).ToList(); + } + + [HttpGet("status")] + [Produces("application/json")] + public NoIntroCatalogStatusResource GetStatus() + { + var entryCounts = _entryRepository.All() + .GroupBy(x => x.CatalogSourceId) + .ToDictionary(x => x.Key, x => x.Count()); + + return new NoIntroCatalogStatusResource + { + Sources = _sourceRepository.All() + .OrderBy(x => x.Name) + .Select(source => new NoIntroCatalogSourceStatusResource + { + Id = source.Id, + Name = source.Name, + CatalogVersion = source.CatalogVersion, + LastSuccessfulSync = source.LastSuccessfulSync, + LastAttemptedSync = source.LastAttemptedSync, + LastSyncError = source.LastSyncError, + EntryCount = entryCounts.GetValueOrDefault(source.Id) + }) + .ToList() + }; + } + + [HttpGet("entry")] + [Produces("application/json")] + public List GetEntries([FromQuery] int catalogSourceId) + { + var entries = catalogSourceId > 0 ? _entryRepository.GetBySourceId(catalogSourceId) : _entryRepository.All(); + return entries.Select(x => x.ToResource()).ToList(); + } + + [HttpGet("verification")] + [Produces("application/json")] + public List GetVerificationResults() + { + return _resultRepository.All().Select(x => x.ToResource()).ToList(); + } + + [HttpGet("componentplan")] + [Produces("application/json")] + public NoIntroCatalogPlanResource GetComponentPlan([FromQuery] int catalogSourceId) + { + var entries = catalogSourceId > 0 ? _entryRepository.GetBySourceId(catalogSourceId) : _entryRepository.All(); + return _componentClassifier.BuildCatalogPlan(entries).ToResource(); + } + } +} diff --git a/src/Gamarr.Api.V3/RomCatalog/NoIntroCatalogResource.cs b/src/Gamarr.Api.V3/RomCatalog/NoIntroCatalogResource.cs new file mode 100644 index 00000000..02ada54f --- /dev/null +++ b/src/Gamarr.Api.V3/RomCatalog/NoIntroCatalogResource.cs @@ -0,0 +1,207 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Gamarr.Http.REST; +using NzbDrone.Core.Games; +using NzbDrone.Core.RomCatalog; + +namespace Gamarr.Api.V3.RomCatalog +{ + public class NoIntroCatalogSourceResource : RestResource + { + public string Name { get; set; } + public string SourceUrl { get; set; } + public string PinnedRevision { get; set; } + public string CatalogVersion { get; set; } + public DateTime? LastSuccessfulSync { get; set; } + public DateTime? LastAttemptedSync { get; set; } + public string LastSyncError { get; set; } + } + + public class NoIntroCatalogEntryResource : RestResource + { + public int CatalogSourceId { get; set; } + public string SystemKey { get; set; } + public string CanonicalName { get; set; } + public string ParentCanonicalName { get; set; } + public string CanonicalFileName { get; set; } + public string ReleaseNumber { get; set; } + public string NumberedCanonicalFileName { get; set; } + public PlatformFamily PlatformFamily { get; set; } + } + + public class NoIntroCatalogStatusResource + { + public List Sources { get; set; } = new List(); + } + + public class NoIntroCatalogSourceStatusResource : RestResource + { + public string Name { get; set; } + public string CatalogVersion { get; set; } + public DateTime? LastSuccessfulSync { get; set; } + public DateTime? LastAttemptedSync { get; set; } + public string LastSyncError { get; set; } + public int EntryCount { get; set; } + } + + public class NoIntroCatalogPlanResource + { + public List Games { get; set; } = new List(); + public List StandaloneGames { get; set; } = new List(); + } + + public class NoIntroCatalogGamePlanResource + { + public string SystemKey { get; set; } + public string GameTitle { get; set; } + public List RegionLanguageComponents { get; set; } = new List(); + public List DownloadPlayComponents { get; set; } = new List(); + } + + public class NoIntroCatalogStandalonePlanResource + { + public string Title { get; set; } + public NoIntroRomComponentType ComponentType { get; set; } + } + + public class NoIntroCatalogComponentSlotResource + { + public string SlotLabel { get; set; } + public string CanonicalName { get; set; } + public NoIntroRomComponentType ComponentType { get; set; } + } + + public class NoIntroVerificationResultResource : RestResource + { + public int SnapshotId { get; set; } + public int VerificationSetId { get; set; } + public int? CatalogEntryId { get; set; } + public string RelativePath { get; set; } + public string ArchivePath { get; set; } + public string MemberPath { get; set; } + public string ActualFileName { get; set; } + public string ExpectedFileName { get; set; } + public string HashType { get; set; } + public string HashValue { get; set; } + public NoIntroVerificationStatus VerificationStatus { get; set; } + public bool IsDuplicate { get; set; } + public bool IsMissing { get; set; } + public DateTime VerifiedAt { get; set; } + } + + public static class NoIntroCatalogResourceMapper + { + public static NoIntroCatalogSourceResource ToResource(this NoIntroCatalogSource model) + { + if (model == null) + { + return null; + } + + return new NoIntroCatalogSourceResource + { + Id = model.Id, + Name = model.Name, + SourceUrl = model.SourceUrl, + PinnedRevision = model.PinnedRevision, + CatalogVersion = model.CatalogVersion, + LastSuccessfulSync = model.LastSuccessfulSync, + LastAttemptedSync = model.LastAttemptedSync, + LastSyncError = model.LastSyncError + }; + } + + public static NoIntroCatalogEntryResource ToResource(this NoIntroCatalogEntry model) + { + if (model == null) + { + return null; + } + + return new NoIntroCatalogEntryResource + { + Id = model.Id, + CatalogSourceId = model.CatalogSourceId, + SystemKey = model.SystemKey, + CanonicalName = model.CanonicalName, + ParentCanonicalName = model.ParentCanonicalName, + CanonicalFileName = model.CanonicalFileName, + ReleaseNumber = model.ReleaseNumber, + NumberedCanonicalFileName = model.NumberedCanonicalFileName, + PlatformFamily = model.PlatformFamily + }; + } + + public static NoIntroVerificationResultResource ToResource(this NoIntroVerificationResult model) + { + if (model == null) + { + return null; + } + + return new NoIntroVerificationResultResource + { + Id = model.Id, + SnapshotId = model.SnapshotId, + VerificationSetId = model.VerificationSetId, + CatalogEntryId = model.CatalogEntryId, + RelativePath = model.RelativePath, + ArchivePath = model.ArchivePath, + MemberPath = model.MemberPath, + ActualFileName = model.ActualFileName, + ExpectedFileName = model.ExpectedFileName, + HashType = model.HashType, + HashValue = model.HashValue, + VerificationStatus = model.VerificationStatus, + IsDuplicate = model.IsDuplicate, + IsMissing = model.IsMissing, + VerifiedAt = model.VerifiedAt + }; + } + + public static NoIntroCatalogPlanResource ToResource(this NoIntroCatalogPlan model) + { + if (model == null) + { + return null; + } + + return new NoIntroCatalogPlanResource + { + Games = model.Games.Select(ToResource).ToList(), + StandaloneGames = model.StandaloneGames.Select(ToResource).ToList() + }; + } + + private static NoIntroCatalogGamePlanResource ToResource(this NoIntroCatalogGamePlan model) + { + return new NoIntroCatalogGamePlanResource + { + SystemKey = model.SystemKey, + GameTitle = model.GameTitle, + RegionLanguageComponents = model.RegionLanguageComponents.Select(ToResource).ToList(), + DownloadPlayComponents = model.DownloadPlayComponents.Select(ToResource).ToList() + }; + } + + private static NoIntroCatalogStandalonePlanResource ToResource(this NoIntroCatalogStandalonePlan model) + { + return new NoIntroCatalogStandalonePlanResource + { + Title = model.Title, + ComponentType = model.ComponentType + }; + } + + private static NoIntroCatalogComponentSlotResource ToResource(this NoIntroCatalogComponentSlot model) + { + return new NoIntroCatalogComponentSlotResource + { + SlotLabel = model.SlotLabel, + CanonicalName = model.CanonicalName, + ComponentType = model.ComponentType + }; + } + } +} diff --git a/src/NzbDrone.Api.Test/GameComponents/GameComponentResourceMapperFixture.cs b/src/NzbDrone.Api.Test/GameComponents/GameComponentResourceMapperFixture.cs new file mode 100644 index 00000000..8034d013 --- /dev/null +++ b/src/NzbDrone.Api.Test/GameComponents/GameComponentResourceMapperFixture.cs @@ -0,0 +1,83 @@ +using System.Collections.Generic; +using FluentAssertions; +using NUnit.Framework; +using Gamarr.Api.V3.GameComponents; +using NzbDrone.Core.Games.Components; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.RomCatalog; + +namespace NzbDrone.Api.Test.GameComponents +{ + [TestFixture] + public class GameComponentResourceMapperFixture + { + [Test] + public void should_prefer_hash_matched_nointro_entry_over_title_matches() + { + var component = new GameComponent + { + Id = 50, + GameId = 51, + Title = "Mario Kart DS" + }; + + var context = new GameComponentNoIntroCatalogContext + { + GameFiles = new List + { + new GameFile + { + Id = 41, + ComponentId = 50, + Size = 33554432 + } + }, + Entries = new List + { + Entry(1, "Mario Kart DS (Europe) (En,Fr,De,Es,It)"), + Entry(2, "Mario Kart DS (Japan)"), + Entry(3, "Mario Kart DS (USA, Australia) (En,Fr,De,Es,It)") + }, + Sources = new List + { + new NoIntroCatalogSource + { + Id = 7, + Name = "No-Intro Nintendo DS", + CatalogVersion = "2026.05.02" + } + }, + HashMatches = new List + { + new NoIntroCatalogFileHashMatch + { + GameFileId = 41, + CatalogEntryId = 1, + HashType = "sha1", + HashValue = "CE97D9B43F0D3CA0D48B781983E8A16F6393378F" + } + } + }; + + var resource = component.ToResource(context); + + resource.NoIntroCatalogMatches.Should().ContainSingle() + .Subject.Should().Match(match => + match.CanonicalFileName == "Mario Kart DS (Europe) (En,Fr,De,Es,It).nds" && + match.HashType == "sha1" && + match.HashValue == "CE97D9B43F0D3CA0D48B781983E8A16F6393378F"); + } + + private static NoIntroCatalogEntry Entry(int id, string canonicalName) + { + return new NoIntroCatalogEntry + { + Id = id, + CatalogSourceId = 7, + SystemKey = "nintendo---nintendo-ds", + CanonicalName = canonicalName, + CanonicalFileName = $"{canonicalName}.nds" + }; + } + } +} diff --git a/src/NzbDrone.Api.Test/GameFiles/GameFileResourceMapperFixture.cs b/src/NzbDrone.Api.Test/GameFiles/GameFileResourceMapperFixture.cs new file mode 100644 index 00000000..439129cf --- /dev/null +++ b/src/NzbDrone.Api.Test/GameFiles/GameFileResourceMapperFixture.cs @@ -0,0 +1,59 @@ +using FluentAssertions; +using NUnit.Framework; +using Gamarr.Api.V3.GameFiles; +using NzbDrone.Core.Games; +using NzbDrone.Core.Games.Components; +using NzbDrone.Core.MediaFiles; + +namespace NzbDrone.Api.Test.GameFiles +{ + [TestFixture] + public class GameFileResourceMapperFixture + { + [Test] + public void should_use_linked_component_metadata_for_folder_backed_nointro_files() + { + var game = new Game { Path = "/games/Mario Kart DS" }; + var gameFile = new GameFile + { + Id = 43, + GameId = 51, + RelativePath = string.Empty, + ComponentId = 53 + }; + var component = new GameComponent + { + Id = 53, + GameId = 51, + ComponentType = GameComponentType.NoIntroRetailRom, + Key = "nointro:retail:mario-kart-ds-europe-en-fr-de-es-it", + Title = "Europe (En,Fr,De,Es,It)" + }; + + var resource = gameFile.ToResource(game, null, null, component); + + resource.ComponentType.Should().Be("noIntroRetailRom"); + resource.ComponentKey.Should().Be(component.Key); + resource.ComponentTitle.Should().Be(component.Title); + } + + [Test] + public void should_fall_back_to_derived_base_type_when_component_is_unknown() + { + var game = new Game { Path = "/games/Mario Kart DS" }; + var gameFile = new GameFile + { + Id = 43, + GameId = 51, + RelativePath = string.Empty, + ComponentId = 0 + }; + + var resource = gameFile.ToResource(game, null, null); + + resource.ComponentType.Should().Be("base"); + resource.ComponentKey.Should().BeNull(); + resource.ComponentTitle.Should().BeNull(); + } + } +} diff --git a/src/NzbDrone.Api.Test/RomCatalog/NoIntroCatalogControllerFixture.cs b/src/NzbDrone.Api.Test/RomCatalog/NoIntroCatalogControllerFixture.cs new file mode 100644 index 00000000..2e46680a --- /dev/null +++ b/src/NzbDrone.Api.Test/RomCatalog/NoIntroCatalogControllerFixture.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using FluentAssertions; +using NUnit.Framework; +using Gamarr.Api.V3.RomCatalog; +using NzbDrone.Core.Games; +using NzbDrone.Core.RomCatalog; +using NzbDrone.Test.Common; + +namespace NzbDrone.Api.Test.RomCatalog +{ + [TestFixture] + public class NoIntroCatalogControllerFixture : TestBase + { + [Test] + public void NoIntroApi_should_surface_catalog_source_metadata_and_verification_rows() + { + Mocker.GetMock() + .Setup(x => x.All()) + .Returns(new List + { + new NoIntroCatalogSource + { + Id = 7, + Name = "No-Intro", + SourceUrl = "https://example.invalid/gba.dat", + CatalogVersion = "2026-07-18", + LastSuccessfulSync = new DateTime(2026, 7, 18, 0, 0, 0, DateTimeKind.Utc) + } + }); + + Mocker.GetMock() + .Setup(x => x.All()) + .Returns(new List + { + new NoIntroVerificationResult + { + Id = 11, + SnapshotId = 2, + VerificationSetId = 3, + ActualFileName = "0001 - F-Zero.zip", + ExpectedFileName = "F-Zero.zip", + VerificationStatus = NoIntroVerificationStatus.NameMismatch, + IsDuplicate = true, + VerifiedAt = new DateTime(2026, 7, 18, 1, 0, 0, DateTimeKind.Utc) + } + }); + + Subject.GetSources().Should().ContainSingle(x => x.Id == 7 && x.CatalogVersion == "2026-07-18" && x.LastSuccessfulSync.HasValue); + Subject.GetVerificationResults().Should().ContainSingle(x => x.Id == 11 && x.VerificationStatus == NoIntroVerificationStatus.NameMismatch && x.IsDuplicate); + } + + [Test] + public void NoIntroApi_should_surface_catalog_status_counts() + { + Mocker.GetMock() + .Setup(x => x.All()) + .Returns(new List + { + new NoIntroCatalogSource + { + Id = 7, + Name = "No-Intro Nintendo DS", + CatalogVersion = "2026.05.02", + LastSuccessfulSync = new DateTime(2026, 7, 19, 0, 0, 0, DateTimeKind.Utc) + } + }); + + Mocker.GetMock() + .Setup(x => x.All()) + .Returns(new List + { + Entry(7, "nintendo---nintendo-ds", "Mario Kart DS (Europe) (En,Fr,De,Es,It)"), + Entry(7, "nintendo---nintendo-ds", "Mario Kart DS (Japan)") + }); + + var status = Subject.GetStatus(); + + status.Sources.Should().ContainSingle(x => x.Name == "No-Intro Nintendo DS" && x.CatalogVersion == "2026.05.02" && x.EntryCount == 2); + } + + [Test] + public void NoIntroApi_should_surface_region_download_play_and_standalone_component_plan() + { + var entries = new List + { + Entry(1, "nintendo-ds", "Mario Kart DS (USA)"), + ChildEntry(1, "nintendo-ds", "Mario Kart DS (Download Play)", "Mario Kart DS"), + Entry(1, "nintendo-ds", "Mario Party DS (Download Play)"), + Entry(1, "nintendo-gba", "Pokemon Emerald Version (Germany)") + }; + + Mocker.GetMock() + .Setup(x => x.GetBySourceId(1)) + .Returns(entries); + + Mocker.GetMock() + .Setup(x => x.BuildCatalogPlan(entries)) + .Returns(new NoIntroComponentClassifier().BuildCatalogPlan(entries)); + + var plan = Subject.GetComponentPlan(1); + + plan.Games.Should().ContainSingle(x => x.GameTitle == "Mario Kart DS") + .Subject.DownloadPlayComponents.Should().ContainSingle(x => x.SlotLabel == "Download Play"); + plan.Games.Should().ContainSingle(x => x.GameTitle == "Pokemon Emerald Version") + .Subject.RegionLanguageComponents.Should().ContainSingle(x => x.SlotLabel == "Germany"); + plan.StandaloneGames.Should().ContainSingle(x => x.Title == "Mario Party DS (Download Play)" && x.ComponentType == NoIntroRomComponentType.Multiboot); + } + + private static NoIntroCatalogEntry Entry(int sourceId, string systemKey, string canonicalName) + { + return new NoIntroCatalogEntry + { + CatalogSourceId = sourceId, + SystemKey = systemKey, + CanonicalName = canonicalName, + CanonicalFileName = $"{canonicalName}.zip", + PlatformFamily = PlatformFamily.Nintendo + }; + } + + private static NoIntroCatalogEntry ChildEntry(int sourceId, string systemKey, string canonicalName, string parentCanonicalName) + { + var entry = Entry(sourceId, systemKey, canonicalName); + entry.ParentCanonicalName = parentCanonicalName; + return entry; + } + } +} diff --git a/src/NzbDrone.Core.Test/Datastore/Converters/PlatformFamilyListConverterFixture.cs b/src/NzbDrone.Core.Test/Datastore/Converters/PlatformFamilyListConverterFixture.cs index 69c9eaff..d12c16c0 100644 --- a/src/NzbDrone.Core.Test/Datastore/Converters/PlatformFamilyListConverterFixture.cs +++ b/src/NzbDrone.Core.Test/Datastore/Converters/PlatformFamilyListConverterFixture.cs @@ -66,11 +66,11 @@ public void should_deserialize_empty_array() [Test] public void should_deserialize_all_platform_families() { - var json = "[\"pc\",\"playStation\",\"xbox\",\"nintendo\",\"linux\",\"mac\",\"mobile\"]"; + var json = "[\"pc\",\"playStation\",\"xbox\",\"nintendo\",\"linux\",\"mac\",\"mobile\",\"nintendoDS\",\"nintendoGBA\",\"nintendoGB\",\"nintendoGBC\"]"; var result = Subject.Parse(json); - result.Should().HaveCount(7); + result.Should().HaveCount(11); result.Should().Contain(PlatformFamily.PC); result.Should().Contain(PlatformFamily.PlayStation); result.Should().Contain(PlatformFamily.Xbox); @@ -78,6 +78,10 @@ public void should_deserialize_all_platform_families() result.Should().Contain(PlatformFamily.Linux); result.Should().Contain(PlatformFamily.Mac); result.Should().Contain(PlatformFamily.Mobile); + result.Should().Contain(PlatformFamily.NintendoDS); + result.Should().Contain(PlatformFamily.NintendoGBA); + result.Should().Contain(PlatformFamily.NintendoGB); + result.Should().Contain(PlatformFamily.NintendoGBC); } [Test] diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/PlatformSpecificationFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/PlatformSpecificationFixture.cs index 17278669..1622e617 100644 --- a/src/NzbDrone.Core.Test/DecisionEngineTests/PlatformSpecificationFixture.cs +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/PlatformSpecificationFixture.cs @@ -86,5 +86,53 @@ public void should_reject_when_non_matching_platform() result.Accepted.Should().BeFalse(); result.Reason.Should().Be(DownloadRejectionReason.WantedPlatform); } + + [Test] + public void should_accept_broad_nintendo_release_for_specific_nintendo_entry() + { + _remoteGame.Game.Platform = PlatformFamily.NintendoDS; + _remoteGame.ParsedGameInfo.Platform = PlatformFamily.Nintendo; + + Subject.IsSatisfiedBy(_remoteGame, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_accept_specific_nintendo_release_for_broad_nintendo_preference() + { + _remoteGame.Game.Platform = PlatformFamily.Unknown; + _remoteGame.ParsedGameInfo.Platform = PlatformFamily.NintendoGBA; + _remoteGame.Game.QualityProfile.PreferredPlatforms = new List + { + PlatformFamily.Nintendo + }; + + Subject.IsSatisfiedBy(_remoteGame, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_accept_game_boy_color_release_for_broad_nintendo_preference() + { + _remoteGame.Game.Platform = PlatformFamily.Unknown; + _remoteGame.ParsedGameInfo.Platform = PlatformFamily.NintendoGBC; + _remoteGame.Game.QualityProfile.PreferredPlatforms = new List + { + PlatformFamily.Nintendo + }; + + Subject.IsSatisfiedBy(_remoteGame, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_accept_specific_playstation_release_for_broad_playstation_preference() + { + _remoteGame.Game.Platform = PlatformFamily.Unknown; + _remoteGame.ParsedGameInfo.Platform = PlatformFamily.SonyPSP; + _remoteGame.Game.QualityProfile.PreferredPlatforms = new List + { + PlatformFamily.PlayStation + }; + + Subject.IsSatisfiedBy(_remoteGame, null).Accepted.Should().BeTrue(); + } } } diff --git a/src/NzbDrone.Core.Test/GameTests/GameComponentServiceFixture.cs b/src/NzbDrone.Core.Test/GameTests/GameComponentServiceFixture.cs index 806ec094..a3337098 100644 --- a/src/NzbDrone.Core.Test/GameTests/GameComponentServiceFixture.cs +++ b/src/NzbDrone.Core.Test/GameTests/GameComponentServiceFixture.cs @@ -4,9 +4,11 @@ using Moq; using NUnit.Framework; using NzbDrone.Core.Games; +using NzbDrone.Core.Games.AlternativeTitles; using NzbDrone.Core.Games.Components; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.Events; +using NzbDrone.Core.RomCatalog; using NzbDrone.Core.Test.Framework; namespace NzbDrone.Core.Test.GameTests @@ -40,6 +42,10 @@ public void Setup() Mocker.GetMock() .Setup(m => m.GetFilesByGame(_game.Id)) .Returns(new List()); + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(It.IsAny())) + .Returns(new List()); } private List CapturedInserts() @@ -55,6 +61,24 @@ private List CapturedInserts() return captured ?? new List(); } + private static NoIntroCatalogEntry Entry(string canonicalName, PlatformFamily platform, string numberedCanonicalFileName = null, string extension = "nds") + { + return new NoIntroCatalogEntry + { + SystemKey = platform switch + { + PlatformFamily.SonyPSP => "sony---playstation-portable", + PlatformFamily.Nintendo3DS => "nintendo---nintendo-3ds", + PlatformFamily.NintendoDS => "nintendo---nintendo-ds", + _ => "nintendo---game-boy-advance" + }, + CanonicalName = canonicalName, + CanonicalFileName = $"{canonicalName}.{extension}", + NumberedCanonicalFileName = numberedCanonicalFileName, + PlatformFamily = platform + }; + } + [Test] public void should_create_base_and_metadata_dlc_components() { @@ -86,6 +110,227 @@ public void should_create_update_and_imported_dlc_components_from_files() inserted.Should().Contain(c => c.ComponentType == GameComponentType.Dlc && c.Key == "import:Some.Release.DLC-GRP" && c.Monitored); } + [Test] + public void should_create_nointro_catalog_components_for_matching_game_title_and_platform() + { + _game.Platform = PlatformFamily.NintendoDS; + _game.GameMetadata.Value.Title = "Mario Kart DS"; + _game.GameMetadata.Value.DlcReferences = new List(); + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(PlatformFamily.NintendoDS)) + .Returns(new List + { + Entry("Mario Kart DS (Europe) (En,Fr,De,Es,It)", PlatformFamily.NintendoDS), + Entry("Mario Kart DS (USA, Australia) (En,Fr,De,Es,It)", PlatformFamily.NintendoDS), + Entry("Mario Kart DS (Japan)", PlatformFamily.NintendoDS), + Entry("Mario Kart DS (USA)", PlatformFamily.Nintendo3DS, extension: "3ds"), + Entry("Mario Kart DS (USA)", PlatformFamily.SonyPSP, extension: "iso"), + Entry("Pokemon Dash (USA)", PlatformFamily.NintendoDS), + Entry("Mario Kart DS (USA)", PlatformFamily.NintendoGBA) + }); + + var inserted = CapturedInserts(); + + inserted.Should().Contain(c => c.ComponentType == GameComponentType.Base && c.Key == "base"); + inserted.Where(c => c.ComponentType == GameComponentType.NoIntroRetailRom).Select(c => c.Title).Should().BeEquivalentTo( + "Europe (En,Fr,De,Es,It)", + "USA, Australia (En,Fr,De,Es,It)", + "Japan"); + inserted.Where(c => c.ComponentType == GameComponentType.NoIntroRetailRom).Should().OnlyContain(c => !c.Monitored && c.Key.StartsWith("nointro:retail:")); + } + + [Test] + public void should_create_download_play_components_for_matching_game_title_and_platform() + { + _game.Platform = PlatformFamily.NintendoDS; + _game.GameMetadata.Value.Title = "Mario Kart DS"; + _game.GameMetadata.Value.DlcReferences = new List(); + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(PlatformFamily.NintendoDS)) + .Returns(new List + { + Entry("Mario Kart DS (Europe) (En,Fr,De,Es,It)", PlatformFamily.NintendoDS), + new NoIntroCatalogEntry + { + SystemKey = "nintendo---nintendo-ds--download-play", + CanonicalName = "Mario Kart DS (Europe) (Demo) (Download Station Vol. 1)", + CanonicalFileName = "Mario Kart DS (Europe) (Demo) (Download Station Vol. 1).nds", + PlatformFamily = PlatformFamily.NintendoDS + } + }); + + var inserted = CapturedInserts(); + + inserted.Should().Contain(c => c.ComponentType == GameComponentType.NoIntroRetailRom && c.Title == "Europe (En,Fr,De,Es,It)"); + inserted.Should().Contain(c => c.ComponentType == GameComponentType.NoIntroMultiboot && c.Title == "Europe (Demo) (Download Station Vol. 1)"); + } + + [Test] + public void should_create_nointro_components_from_alternative_titles() + { + _game.Platform = PlatformFamily.NintendoDS; + _game.GameMetadata.Value.Title = "Pokémon Weiß"; + _game.GameMetadata.Value.OriginalTitle = "Pokémon White Version"; + _game.GameMetadata.Value.DlcReferences = new List(); + _game.GameMetadata.Value.AlternativeTitles = new List + { + new AlternativeTitle("Pokemon White"), + new AlternativeTitle("Pocket Monsters White") + }; + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(PlatformFamily.NintendoDS)) + .Returns(new List + { + Entry("Pokemon - White Version (USA, Europe) (NDSi Enhanced)", PlatformFamily.NintendoDS), + Entry("Pokemon - Black Version (USA, Europe) (NDSi Enhanced)", PlatformFamily.NintendoDS) + }); + + var inserted = CapturedInserts(); + + inserted.Should().Contain(c => c.ComponentType == GameComponentType.NoIntroRetailRom && c.Title == "USA, Europe (NDSi Enhanced)"); + inserted.Where(c => c.ComponentType == GameComponentType.NoIntroRetailRom).Should().ContainSingle(); + } + + [Test] + public void should_create_3ds_nointro_components_for_matching_game_title_and_platform() + { + _game.Platform = PlatformFamily.Nintendo3DS; + _game.GameMetadata.Value.Title = "Mario Kart 7"; + _game.GameMetadata.Value.DlcReferences = new List(); + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(PlatformFamily.Nintendo3DS)) + .Returns(new List + { + Entry("Mario Kart 7 (Europe) (En,Fr,De,Es,It,Nl,Pt,Ru)", PlatformFamily.Nintendo3DS, extension: "3ds"), + Entry("Mario Kart 7 (USA) (En,Fr,Es)", PlatformFamily.Nintendo3DS, extension: "3ds"), + Entry("Mario Kart DS (USA)", PlatformFamily.NintendoDS) + }); + + var inserted = CapturedInserts(); + + inserted.Where(c => c.ComponentType == GameComponentType.NoIntroRetailRom).Select(c => c.Title).Should().BeEquivalentTo( + "Europe (En,Fr,De,Es,It,Nl,Pt,Ru)", + "USA (En,Fr,Es)"); + } + + [Test] + public void should_link_nointro_file_to_matching_catalog_component() + { + _game.Platform = PlatformFamily.NintendoDS; + _game.GameMetadata.Value.Title = "Mario Kart DS"; + _game.GameMetadata.Value.DlcReferences = new List(); + + var baseSlot = new GameComponent { Id = 10, GameId = _game.Id, ComponentType = GameComponentType.Base, Key = "base" }; + var europeSlot = new GameComponent { Id = 11, GameId = _game.Id, ComponentType = GameComponentType.NoIntroRetailRom, Key = "nointro:retail:mario-kart-ds-europe-en-fr-de-es-it" }; + var file = new GameFile { Id = 1, GameId = _game.Id, RelativePath = "0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds" }; + + Mocker.GetMock() + .Setup(r => r.GetByGame(_game.Id)) + .Returns(new List { baseSlot, europeSlot }); + + Mocker.GetMock() + .Setup(m => m.GetFilesByGame(_game.Id)) + .Returns(new List { file }); + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(PlatformFamily.NintendoDS)) + .Returns(new List + { + Entry("Mario Kart DS (Europe) (En,Fr,De,Es,It)", PlatformFamily.NintendoDS, "0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds") + }); + + Subject.EnsureComponents(_game); + + file.ComponentId.Should().Be(11); + Mocker.GetMock() + .Verify(m => m.Update(It.Is>(l => l.Count == 1 && l[0].ComponentId == 11)), Times.Once()); + } + + [Test] + public void should_link_renamed_nointro_file_using_original_file_path() + { + _game.Platform = PlatformFamily.NintendoGB; + _game.GameMetadata.Value.Title = "Mega Man IV"; + _game.GameMetadata.Value.DlcReferences = new List(); + + var baseSlot = new GameComponent { Id = 10, GameId = _game.Id, ComponentType = GameComponentType.Base, Key = "base" }; + var usaSlot = new GameComponent { Id = 11, GameId = _game.Id, ComponentType = GameComponentType.NoIntroRetailRom, Key = "nointro:retail:mega-man-iv-usa" }; + var file = new GameFile + { + Id = 1, + GameId = _game.Id, + RelativePath = "Mega Man IV (1993) Retail - Gamarr.zip", + OriginalFilePath = "Nintendo - Game Boy/Mega Man IV (USA).zip" + }; + + Mocker.GetMock() + .Setup(r => r.GetByGame(_game.Id)) + .Returns(new List { baseSlot, usaSlot }); + + Mocker.GetMock() + .Setup(m => m.GetFilesByGame(_game.Id)) + .Returns(new List { file }); + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(PlatformFamily.NintendoGB)) + .Returns(new List + { + Entry("Mega Man IV (USA)", PlatformFamily.NintendoGB, extension: "zip") + }); + + Subject.EnsureComponents(_game); + + file.ComponentId.Should().Be(11); + Mocker.GetMock() + .Verify(m => m.Update(It.Is>(l => l.Count == 1 && l[0].ComponentId == 11)), Times.Once()); + } + + [Test] + public void should_link_folder_backed_nointro_file_to_matching_catalog_component() + { + _game.Platform = PlatformFamily.NintendoDS; + _game.Path = "/games/Mario Kart DS"; + _game.GameMetadata.Value.Title = "Mario Kart DS"; + _game.GameMetadata.Value.DlcReferences = new List(); + + var baseSlot = new GameComponent { Id = 10, GameId = _game.Id, ComponentType = GameComponentType.Base, Key = "base" }; + var europeSlot = new GameComponent { Id = 11, GameId = _game.Id, ComponentType = GameComponentType.NoIntroRetailRom, Key = "nointro:retail:mario-kart-ds-europe-en-fr-de-es-it", Title = "Europe (En,Fr,De,Es,It)" }; + var file = new GameFile { Id = 1, GameId = _game.Id, RelativePath = string.Empty }; + + Mocker.GetMock() + .Setup(r => r.GetByGame(_game.Id)) + .Returns(new List { baseSlot, europeSlot }); + + Mocker.GetMock() + .Setup(m => m.GetFilesByGame(_game.Id)) + .Returns(new List { file }); + + Mocker.GetMock() + .Setup(d => d.FolderExists(_game.Path)) + .Returns(true); + + Mocker.GetMock() + .Setup(d => d.GetFiles(_game.Path, true)) + .Returns(new List { "/games/Mario Kart DS/0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds" }); + + Mocker.GetMock() + .Setup(r => r.GetByPlatformFamily(PlatformFamily.NintendoDS)) + .Returns(new List + { + Entry("Mario Kart DS (Europe) (En,Fr,De,Es,It)", PlatformFamily.NintendoDS, "0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds") + }); + + Subject.EnsureComponents(_game); + + file.ComponentId.Should().Be(11); + Mocker.GetMock() + .Verify(m => m.Update(It.Is>(l => l.Count == 1 && l[0].ComponentId == 11)), Times.Once()); + } + [Test] public void should_be_idempotent_when_components_already_exist() { diff --git a/src/NzbDrone.Core.Test/MediaFiles/MediaFileExtensionsFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/MediaFileExtensionsFixture.cs index 8887571b..1d449cf5 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/MediaFileExtensionsFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/MediaFileExtensionsFixture.cs @@ -19,6 +19,14 @@ public class MediaFileExtensionsFixture : CoreTest [TestCase(@"setup.exe")] [TestCase(@"game.iso")] [TestCase(@"data1.bin")] + [TestCase(@"0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds")] + [TestCase(@"Pokemon Emerald Version (Germany).gba")] + [TestCase(@"Super Mario Bros.nes")] + [TestCase(@"Super Mario World.sfc")] + [TestCase(@"Mario Kart 64.z64")] + [TestCase(@"The Legend of Zelda Breath of the Wild.xci")] + [TestCase(@"Metroid Prime.rvz")] + [TestCase(@"No-Intro archive.zip")] public void should_recognize_game_file(string fileName) { MediaFileExtensions.IsGameFileExtension(Path.GetExtension(fileName)).Should().BeTrue(); diff --git a/src/NzbDrone.Core.Test/Organizer/RenameProfileNamingBehaviorFixture.cs b/src/NzbDrone.Core.Test/Organizer/RenameProfileNamingBehaviorFixture.cs new file mode 100644 index 00000000..d3609f55 --- /dev/null +++ b/src/NzbDrone.Core.Test/Organizer/RenameProfileNamingBehaviorFixture.cs @@ -0,0 +1,112 @@ +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.CustomFormats; +using NzbDrone.Core.Games; +using NzbDrone.Core.Games.Components; +using NzbDrone.Core.Games.Translations; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Organizer; +using NzbDrone.Core.Qualities; +using NzbDrone.Core.RomCatalog; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Organizer +{ + [TestFixture] + public class RenameProfileNamingBehaviorFixture : CoreTest + { + private NamingConfig _namingConfig; + + [SetUp] + public void Setup() + { + _namingConfig = NamingConfig.Default; + _namingConfig.RenameGames = true; + _namingConfig.RenameProfile = RenameProfile.Gamarr; + + Mocker.GetMock() + .Setup(x => x.GetConfig()) + .Returns(_namingConfig); + + Mocker.GetMock() + .Setup(x => x.Get(It.IsAny())) + .Returns(quality => Quality.DefaultQualityDefinitions.Single(x => x.Quality == quality)); + + Mocker.GetMock() + .Setup(x => x.All()) + .Returns(new System.Collections.Generic.List()); + + Mocker.GetMock() + .Setup(x => x.GetAllTranslationsForGameMetadata(It.IsAny())) + .Returns(new System.Collections.Generic.List()); + } + + [Test] + public void RenameProfile_should_preserve_existing_default_file_name_builder_output_for_normal_gamarr_profile() + { + var game = new Game + { + Title = "South Park", + Year = 1998 + }; + + var gameFile = new GameFile + { + Quality = new QualityModel(Quality.Uplay) + }; + + Subject.BuildFileName(game, gameFile) + .Should().Be("South Park (1998) Uplay"); + } + + [Test] + public void RenameProfile_should_preserve_original_nointro_variant_filename_for_gamarr_profile() + { + var game = new Game + { + Id = 5, + Title = "Mega Man IV", + Year = 1993 + }; + + var gameFile = new GameFile + { + GameId = 5, + Quality = new QualityModel(Quality.Retail), + OriginalFilePath = "Nintendo - Game Boy/Mega Man IV (USA).zip", + RelativePath = "Mega Man IV (1993) Retail - Gamarr.zip" + }; + + Mocker.GetMock() + .Setup(x => x.GetByGame(5)) + .Returns(new System.Collections.Generic.List + { + new GameComponent + { + Id = 11, + GameId = 5, + ComponentType = GameComponentType.NoIntroRetailRom, + Key = "nointro:retail:mega-man-iv-usa", + Title = "USA" + } + }); + + Mocker.GetMock() + .Setup(x => x.All()) + .Returns(new[] + { + new NoIntroCatalogEntry + { + SystemKey = "nintendo---game-boy", + CanonicalName = "Mega Man IV (USA)", + CanonicalFileName = "Mega Man IV (USA).zip" + } + }); + + Subject.BuildFileName(game, gameFile) + .Should().Be("Mega Man IV (USA)"); + } + } +} diff --git a/src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/FileNameBuilderFixture.cs b/src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/FileNameBuilderFixture.cs index ac55af45..278313b2 100644 --- a/src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/FileNameBuilderFixture.cs +++ b/src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/FileNameBuilderFixture.cs @@ -12,9 +12,11 @@ using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.MediaInfo; using NzbDrone.Core.Games; +using NzbDrone.Core.Games.Components; using NzbDrone.Core.Games.Translations; using NzbDrone.Core.Organizer; using NzbDrone.Core.Qualities; +using NzbDrone.Core.RomCatalog; using NzbDrone.Core.Test.Framework; using NzbDrone.Test.Common; @@ -173,6 +175,41 @@ public void should_replace_translated_game_title_with_base_title_if_no_translati .Should().Be("South Park"); } + [Test] + public void should_use_nointro_catalog_filename_when_nointro_profile_is_selected() + { + _namingConfig.RenameProfile = RenameProfile.NoIntroPreserveById; + _gameFile.ComponentId = 50; + _gameFile.RelativePath = "Mario Kart DS (2005) Unknown - Gamarr.nds"; + _gameFile.OriginalFilePath = "Nintendo - Nintendo DS/Mario Kart DS (Europe) (En,Fr,De,Es,It).nds"; + + Mocker.GetMock() + .Setup(x => x.Get(50)) + .Returns(new GameComponent + { + Id = 50, + Key = "nointro:retail:mario-kart-ds-europe-en-fr-de-es-it", + Title = "Mario Kart DS (Europe) (En,Fr,De,Es,It)" + }); + + Mocker.GetMock() + .Setup(x => x.All()) + .Returns(new[] + { + new NoIntroCatalogEntry + { + SystemKey = "nintendo---nintendo-ds", + CanonicalName = "Mario Kart DS (Europe) (En,Fr,De,Es,It)", + CanonicalFileName = "Mario Kart DS (Europe) (En,Fr,De,Es,It).nds", + ReleaseNumber = "0201", + NumberedCanonicalFileName = "0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds" + } + }); + + Subject.BuildFileName(_game, _gameFile) + .Should().Be("0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It)"); + } + [Test] public void should_replace_translated_game_title_with_fallback_if_no_translation_exists() { diff --git a/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs index 0f2e0648..95045ebc 100644 --- a/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs @@ -77,6 +77,18 @@ public void should_parse_language_french_english(string postTitle) result.Languages.Should().Contain(Language.English); } + [Test] + public void should_parse_nointro_language_code_list() + { + var result = LanguageParser.ParseLanguages("0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds"); + + result.Should().Contain(Language.English); + result.Should().Contain(Language.French); + result.Should().Contain(Language.German); + result.Should().Contain(Language.Spanish); + result.Should().Contain(Language.Italian); + } + [TestCase("Game.Title.1994.Spanish.1080p.XviD-LOL")] [TestCase("Game Title (2020)[BDRemux AVC 1080p][E-AC3 DD Plus 5.1 Castellano-Inglés Subs]")] [TestCase("Game Title (2020) [UHDRemux2160p HDR][DTS-HD MA 5.1 AC3 5.1 Castellano - True-HD 7.1 Atmos Inglés Subs]")] diff --git a/src/NzbDrone.Core.Test/ParserTests/PlatformParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/PlatformParserFixture.cs index 54577247..1522c246 100644 --- a/src/NzbDrone.Core.Test/ParserTests/PlatformParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/PlatformParserFixture.cs @@ -9,13 +9,13 @@ namespace NzbDrone.Core.Test.ParserTests [TestFixture] public class PlatformParserFixture : CoreTest { - [TestCase("Portal 2 (2011) [Ps3][EUR FREE][MULTi5]", PlatformFamily.PlayStation, "PS3")] - [TestCase("Game Title 2023 PS3 EUR ISO", PlatformFamily.PlayStation, "PS3")] - [TestCase("Game.Title.2023.PlayStation3.EUR.ISO", PlatformFamily.PlayStation, "PS3")] + [TestCase("Portal 2 (2011) [Ps3][EUR FREE][MULTi5]", PlatformFamily.SonyPS3, "PS3")] + [TestCase("Game Title 2023 PS3 EUR ISO", PlatformFamily.SonyPS3, "PS3")] + [TestCase("Game.Title.2023.PlayStation3.EUR.ISO", PlatformFamily.SonyPS3, "PS3")] [TestCase("Game Title (2023) [PS4] [USA]", PlatformFamily.PlayStation, "PS4")] [TestCase("Game.Title.2023.PS5.EUR.PKG", PlatformFamily.PlayStation, "PS5")] - [TestCase("Game Title 2023 PSVita USA VPK", PlatformFamily.PlayStation, "PS Vita")] - [TestCase("Game.Title.2023.PSP.EUR.ISO", PlatformFamily.PlayStation, "PSP")] + [TestCase("Game Title 2023 PSVita USA VPK", PlatformFamily.SonyPSVita, "PS Vita")] + [TestCase("Game.Title.2023.PSP.EUR.ISO", PlatformFamily.SonyPSP, "PSP")] public void should_parse_playstation_platform(string postTitle, PlatformFamily expectedFamily, string expectedString) { var result = PlatformParser.ParsePlatform(postTitle); @@ -40,13 +40,19 @@ public void should_parse_xbox_platform(string postTitle, PlatformFamily expected resultString.Should().Be(expectedString); } - [TestCase("Game Title 2023 Switch NSP", PlatformFamily.Nintendo, "Switch")] - [TestCase("Game.Title.2023.NSW.USA.XCI", PlatformFamily.Nintendo, "Switch")] - [TestCase("Game Title (2023) [Nintendo Switch]", PlatformFamily.Nintendo, "Switch")] - [TestCase("Game.Title.2023.WiiU.USA.WUX", PlatformFamily.Nintendo, "Wii U")] - [TestCase("Game Title 2023 Wii ISO PAL", PlatformFamily.Nintendo, "Wii")] - [TestCase("Game.Title.2023.3DS.USA.CIA", PlatformFamily.Nintendo, "3DS")] - [TestCase("Game Title 2023 NDS USA ROM", PlatformFamily.Nintendo, "NDS")] + [TestCase("Game Title 2023 Switch NSP", PlatformFamily.NintendoSwitch, "Switch")] + [TestCase("Game.Title.2023.NSW.USA.XCI", PlatformFamily.NintendoSwitch, "Switch")] + [TestCase("Game Title (2023) [Nintendo Switch]", PlatformFamily.NintendoSwitch, "Switch")] + [TestCase("Game.Title.2023.WiiU.USA.WUX", PlatformFamily.NintendoWiiU, "Wii U")] + [TestCase("Game Title 2023 Wii ISO PAL", PlatformFamily.NintendoWii, "Wii")] + [TestCase("Game.Title.2023.3DS.USA.CIA", PlatformFamily.Nintendo3DS, "3DS")] + [TestCase("Game Title 2023 NDS USA ROM", PlatformFamily.NintendoDS, "NDS")] + [TestCase("Game Title 2023 GBA USA ROM", PlatformFamily.NintendoGBA, "GBA")] + [TestCase("Game Title 2023 Game Boy Advance USA ROM", PlatformFamily.NintendoGBA, "GBA")] + [TestCase("Game Title 2023 GBC USA ROM", PlatformFamily.NintendoGBC, "GBC")] + [TestCase("Game Title 2023 Game Boy Color USA ROM", PlatformFamily.NintendoGBC, "GBC")] + [TestCase("Game Title 2023 GB USA ROM", PlatformFamily.NintendoGB, "GB")] + [TestCase("Game Title 2023 Game Boy USA ROM", PlatformFamily.NintendoGB, "GB")] public void should_parse_nintendo_platform(string postTitle, PlatformFamily expectedFamily, string expectedString) { var result = PlatformParser.ParsePlatform(postTitle); @@ -99,7 +105,7 @@ public void should_parse_platform_from_full_parser(string postTitle) var result = Parser.Parser.ParseGameTitle(postTitle, true); result.Should().NotBeNull(); - result.Platform.Should().Be(PlatformFamily.PlayStation); + result.Platform.Should().Be(PlatformFamily.SonyPS3); result.PlatformString.Should().Be("PS3"); } } diff --git a/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs b/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs index 55da1460..60e73637 100644 --- a/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs +++ b/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs @@ -23,6 +23,8 @@ public void Setup() @"Game2.exe", @"Game3.rar", @"Game4.zip", + @"Mario Kart DS.nds", + @"Pokemon Emerald.gba", @"readme.txt", @"game" }; @@ -80,7 +82,7 @@ public void should_return_video_files_only() var path = @"C:\Test\"; GivenFiles(GetFiles(path)); - Subject.GetVideoFiles(path).Should().HaveCount(4); + Subject.GetVideoFiles(path).Should().HaveCount(6); } [TestCase("Extras")] diff --git a/src/NzbDrone.Core.Test/RomCatalog/NoIntroCatalogPersistenceFixture.cs b/src/NzbDrone.Core.Test/RomCatalog/NoIntroCatalogPersistenceFixture.cs new file mode 100644 index 00000000..aa8f741a --- /dev/null +++ b/src/NzbDrone.Core.Test/RomCatalog/NoIntroCatalogPersistenceFixture.cs @@ -0,0 +1,57 @@ +using System; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.RomCatalog; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.RomCatalog +{ + [TestFixture] + public class NoIntroCatalogPersistenceFixture : DbTest + { + [Test] + public void should_persist_generic_nointro_catalog_models_without_gamefile_semantics() + { + var sourceRepository = Mocker.Resolve(); + var entryRepository = Mocker.Resolve(); + var resultRepository = Mocker.Resolve(); + + var source = sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/nointro.dat", + PinnedRevision = "rev-1", + CatalogVersion = "2026-07-01" + }); + + var entry = entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "F-Zero for Game Boy Advance (Japan)", + CanonicalFileName = "0001 - F-Zero for Game Boy Advance (Japan).zip" + }); + + var result = resultRepository.Insert(new NoIntroVerificationResult + { + SnapshotId = 1, + VerificationSetId = 1, + CatalogEntryId = entry.Id, + RelativePath = "GBA (by-id)/0001 - F-Zero for Game Boy Advance (Japan).zip", + ActualFileName = "0001 - F-Zero for Game Boy Advance (Japan).zip", + ExpectedFileName = "0001 - F-Zero for Game Boy Advance (Japan).zip", + HashType = "sha1", + HashValue = "abc123", + VerificationStatus = NoIntroVerificationStatus.Verified, + IsDuplicate = false, + IsMissing = false, + VerifiedAt = DateTime.UtcNow + }); + + source.Id.Should().BeGreaterThan(0); + entry.Id.Should().BeGreaterThan(0); + result.Id.Should().BeGreaterThan(0); + result.CatalogEntryId.Should().Be(entry.Id); + } + } +} diff --git a/src/NzbDrone.Core.Test/RomCatalog/NoIntroCatalogSyncServiceFixture.cs b/src/NzbDrone.Core.Test/RomCatalog/NoIntroCatalogSyncServiceFixture.cs new file mode 100644 index 00000000..3854f016 --- /dev/null +++ b/src/NzbDrone.Core.Test/RomCatalog/NoIntroCatalogSyncServiceFixture.cs @@ -0,0 +1,269 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Games; +using NzbDrone.Core.RomCatalog; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Test.Common; + +namespace NzbDrone.Core.Test.RomCatalog +{ + [TestFixture] + public class NoIntroCatalogSyncServiceFixture : DbTest + { + private NoIntroCatalogSourceRepository _sourceRepository; + private NoIntroCatalogEntryRepository _entryRepository; + private NoIntroCatalogHashRepository _hashRepository; + private NoIntroCatalogSyncService _subject; + + [SetUp] + public void Setup() + { + _sourceRepository = Mocker.Resolve(); + _entryRepository = Mocker.Resolve(); + _hashRepository = Mocker.Resolve(); + _subject = Mocker.Resolve(); + } + + [Test] + public void sync_should_ingest_snapshot_and_update_metadata() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/gba.dat", + PinnedRevision = "pin-1" + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(source.SourceUrl)) + .Returns("
Nintendo - Game Boy Advance2026.07
"); + + _subject.Sync(source.Id); + + var storedSource = _sourceRepository.Get(source.Id); + storedSource.CatalogVersion.Should().Be("2026.07"); + storedSource.LastSuccessfulSync.Should().NotBeNull(); + storedSource.LastSyncError.Should().BeNull(); + + var entry = _entryRepository.All().Should().ContainSingle().Subject; + entry.CatalogSourceId.Should().Be(source.Id); + entry.SystemKey.Should().Be("nintendo---game-boy-advance"); + + _hashRepository.All().Should().Contain(x => x.CatalogEntryId == entry.Id && x.HashType == "crc32" && x.HashValue == "ABCDEF12"); + } + + [Test] + public void sync_should_preserve_quoted_clrmamepro_rom_filenames() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/gba.dat" + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(source.SourceUrl)) + .Returns("clrmamepro (\n" + + "\tname \"Nintendo - Game Boy Advance\"\n" + + "\tversion \"2026.07\"\n" + + ")\n" + + "game (\n" + + "\tname \"007 - Everything or Nothing (USA, Europe) (En,Fr,De)\"\n" + + "\trom ( name \"007 - Everything or Nothing (USA, Europe) (En,Fr,De).gba\" size 16777216 crc 1234ABCD md5 0123456789ABCDEF0123456789ABCDEF sha1 0123456789ABCDEF0123456789ABCDEF01234567 )\n" + + ")\n"); + + _subject.Sync(source.Id); + + var entry = _entryRepository.All().Should().ContainSingle().Subject; + entry.CanonicalName.Should().Be("007 - Everything or Nothing (USA, Europe) (En,Fr,De)"); + entry.CanonicalFileName.Should().Be("007 - Everything or Nothing (USA, Europe) (En,Fr,De).gba"); + _hashRepository.All().Should().Contain(x => x.CatalogEntryId == entry.Id && x.HashType == "crc32" && x.HashValue == "1234ABCD"); + } + + [Test] + public void sync_should_enrich_entries_with_datomatic_numbered_filename_by_hash() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro Nintendo DS", + SourceUrl = "https://example.invalid/nds.dat" + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(source.SourceUrl)) + .Returns("
Nintendo - Nintendo DS2026.05.02
"); + + Mocker.GetMock() + .Setup(x => x.FetchDatOMaticNumbered(28)) + .Returns("
Nintendo - Nintendo DS
"); + + _subject.Sync(source.Id); + + var entry = _entryRepository.All().Should().ContainSingle().Subject; + entry.CanonicalName.Should().Be("Mario Kart DS (Europe) (En,Fr,De,Es,It)"); + entry.CanonicalFileName.Should().Be("Mario Kart DS (Europe) (En,Fr,De,Es,It).nds"); + entry.ReleaseNumber.Should().Be("0201"); + entry.NumberedCanonicalFileName.Should().Be("0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds"); + } + + [Test] + public void sync_should_fallback_to_advanscene_release_number_when_datomatic_fails() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro Nintendo DS", + SourceUrl = "https://example.invalid/nds.dat" + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(source.SourceUrl)) + .Returns("
Nintendo - Nintendo DS2026.05.02
"); + + Mocker.GetMock() + .Setup(x => x.FetchDatOMaticNumbered(28)) + .Throws(new InvalidOperationException("DAT-o-MATIC did not return a numbered DAT download token")); + + Mocker.GetMock() + .Setup(x => x.FetchAdvanscene("https://advanscene.com/offline/datas/ADVANsCEne_NDS_S.zip")) + .Returns("20194E8127E"); + + _subject.Sync(source.Id); + + var entry = _entryRepository.All().Should().ContainSingle().Subject; + entry.ReleaseNumber.Should().Be("0201"); + entry.NumberedCanonicalFileName.Should().Be("0201 - Mario Kart DS (Europe) (En,Fr,De,Es,It).nds"); + } + + [Test] + public void sync_should_seed_missing_default_game_boy_sources() + { + _sourceRepository.Purge(); + + var existingSource = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro Nintendo DS", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20DS.dat" + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(It.IsAny())) + .Returns("
Nintendo - Nintendo DS
"); + + _subject.Sync(); + + var sources = _sourceRepository.All().ToList(); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo Game Boy"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo Game Boy Color"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo Game Boy Advance"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo DS Download Play"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo 3DS"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo 3DS Digital"); + sources.Should().Contain(x => x.Name == "No-Intro New Nintendo 3DS"); + sources.Should().Contain(x => x.Name == "No-Intro New Nintendo 3DS Digital"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo Entertainment System"); + sources.Should().Contain(x => x.Name == "No-Intro Super Nintendo Entertainment System"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo 64"); + sources.Should().Contain(x => x.Name == "No-Intro Nintendo DSi"); + sources.Should().Contain(x => x.Name == "No-Intro Wii Digital"); + sources.Should().Contain(x => x.Name == "No-Intro Wii U Digital"); + sources.Should().Contain(x => x.Name == "No-Intro Sony PlayStation Portable"); + sources.Should().Contain(x => x.Name == "No-Intro Sony PlayStation Vita"); + sources.Should().Contain(x => x.Name == "No-Intro Sony PlayStation 3 PSN"); + + // The DAT-o-MATIC-only niche systems are intentionally NOT seeded while + // the datomatic:// scrape path is non-functional (would surface + // permanently-failing sources). Restore once the scrape is hardened. + sources.Should().NotContain(x => x.SourceUrl.StartsWith("datomatic://")); + sources.Should().ContainSingle(x => x.SourceUrl == existingSource.SourceUrl); + } + + [Test] + public void sync_should_map_3ds_sources_to_nintendo_3ds_platform() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro Nintendo 3DS", + SourceUrl = "https://example.invalid/3ds.dat" + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(source.SourceUrl)) + .Returns("
Nintendo - Nintendo 3DS2026.07
"); + + _subject.Sync(source.Id); + + var entry = _entryRepository.All().Should().ContainSingle().Subject; + entry.SystemKey.Should().Be("nintendo---nintendo-3ds"); + entry.PlatformFamily.Should().Be(PlatformFamily.Nintendo3DS); + entry.CanonicalFileName.Should().Be("Mario Kart 7 (USA) (En,Fr,Es).3ds"); + } + + [Test] + public void sync_should_map_sony_sources_to_specific_playstation_platforms() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro Sony PlayStation Portable", + SourceUrl = "https://example.invalid/psp.dat" + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(source.SourceUrl)) + .Returns("
Sony - PlayStation Portable2026.07
"); + + _subject.Sync(source.Id); + + var entry = _entryRepository.All().Should().ContainSingle().Subject; + entry.SystemKey.Should().Be("sony---playstation-portable"); + entry.PlatformFamily.Should().Be(PlatformFamily.SonyPSP); + } + + [Test] + public void sync_failure_should_preserve_existing_catalog_and_record_failure() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/gba.dat", + PinnedRevision = "pin-1", + CatalogVersion = "old" + }); + + var existingEntry = _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "Existing Entry", + CanonicalFileName = "Existing Entry.gba" + }); + + _hashRepository.Insert(new NoIntroCatalogHash + { + CatalogEntryId = existingEntry.Id, + HashType = "sha1", + HashValue = "oldhash", + IsPrimary = true + }); + + Mocker.GetMock() + .Setup(x => x.Fetch(source.SourceUrl)) + .Throws(new InvalidOperationException("boom")); + + Action act = () => _subject.Sync(source.Id); + + act.Should().Throw(); + ExceptionVerification.ExpectedWarns(1); + + _entryRepository.All().Should().ContainSingle(x => x.Id == existingEntry.Id); + _hashRepository.All().Should().ContainSingle(x => x.HashValue == "oldhash"); + + var storedSource = _sourceRepository.Get(source.Id); + storedSource.LastAttemptedSync.Should().NotBeNull(); + storedSource.LastSyncError.Should().Be("boom"); + storedSource.CatalogVersion.Should().Be("old"); + } + } +} diff --git a/src/NzbDrone.Core.Test/RomCatalog/NoIntroComponentClassifierFixture.cs b/src/NzbDrone.Core.Test/RomCatalog/NoIntroComponentClassifierFixture.cs new file mode 100644 index 00000000..629ada81 --- /dev/null +++ b/src/NzbDrone.Core.Test/RomCatalog/NoIntroComponentClassifierFixture.cs @@ -0,0 +1,223 @@ +using System.Linq; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.RomCatalog; + +namespace NzbDrone.Core.Test.RomCatalog +{ + [TestFixture] + public class NoIntroComponentClassifierFixture + { + private NoIntroComponentClassifier _subject; + + [SetUp] + public void Setup() + { + _subject = new NoIntroComponentClassifier(); + } + + [TestCase("GBA (by-id)/0001 - F-Zero for Game Boy Advance (Japan).zip", "0001 - F-Zero for Game Boy Advance (Japan).zip", NoIntroRomComponentType.RetailRom, false)] + [TestCase("Nintendo 3DS/3ds/Mario Kart 7 (USA) (En,Fr,Es).3ds", "Mario Kart 7 (USA) (En,Fr,Es).3ds", NoIntroRomComponentType.RetailRom, false)] + [TestCase("Sony PlayStation Portable/iso/Grand Theft Auto - Liberty City Stories (USA).iso", "Grand Theft Auto - Liberty City Stories (USA).iso", NoIntroRomComponentType.RetailRom, false)] + [TestCase("GBA (e-Reader)/Animal Crossing-e - Series 1 - A-001 - K.K. Slider (USA).zip", "Animal Crossing-e - Series 1 - A-001 - K.K. Slider (USA).zip", NoIntroRomComponentType.EReaderCards, false)] + [TestCase("GBA (Multiboot)/Animal Crossing - Balloon Fight (USA, Europe).gba", "Animal Crossing - Balloon Fight (USA, Europe).gba", NoIntroRomComponentType.Multiboot, false)] + [TestCase("GBA (Play-Yan)/Nintendo - Game Boy Advance (Play-Yan).zip", "Nintendo - Game Boy Advance (Play-Yan).zip", NoIntroRomComponentType.Video, false)] + [TestCase("GBA (Video)/Game Boy Advance Video - Cartoon Network Collection - Volume 1 (USA, Europe).gba", "Game Boy Advance Video - Cartoon Network Collection - Volume 1 (USA, Europe).gba", NoIntroRomComponentType.Video, false)] + [TestCase("Download Play/Mario Kart DS Demo.nds", "Mario Kart DS Demo.nds", NoIntroRomComponentType.Multiboot, false)] + [TestCase("DSvision SD cards/Media Title.nds", "Media Title.nds", NoIntroRomComponentType.Video, false)] + [TestCase("GBA (by-id)/xB02 - [BIOS] Game Boy Advance (World).zip", "xB02 - [BIOS] Game Boy Advance (World).zip", NoIntroRomComponentType.Bios, false)] + [TestCase("Unknown Folder/Prototype Build.nds", "Prototype Build.nds", NoIntroRomComponentType.RomhackOrUnverified, true)] + public void should_classify_known_component_shapes(string relativePath, string fileName, NoIntroRomComponentType expectedType, bool expectedFallback) + { + var result = _subject.Classify(relativePath, fileName); + + result.ComponentType.Should().Be(expectedType); + result.IsFallback.Should().Be(expectedFallback); + } + + [Test] + public void should_build_RegionLanguageComponents_from_catalog_confirmed_entries_only() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-gba", "Pokemon Emerald Version (USA)"), + Entry("nintendo-gba", "Pokemon Emerald Version (Germany)"), + Entry("nintendo-gba", "Pokemon Emerald Version (France)"), + Entry("nintendo-gba", "Pokemon Emerald Version (Spain)"), + Entry("nintendo-gba", "Pokemon Emerald Version (Italy)") + }); + + var game = plan.Games.Should().ContainSingle(x => x.SystemKey == "nintendo-gba" && x.GameTitle == "Pokemon Emerald Version").Subject; + + game.RegionLanguageComponents.Should().HaveCount(5); + game.RegionLanguageComponents.Select(x => x.SlotLabel).Should().BeEquivalentTo("USA", "Germany", "France", "Spain", "Italy"); + game.DownloadPlayComponents.Should().BeEmpty(); + plan.StandaloneGames.Should().BeEmpty(); + } + + [Test] + public void should_group_multi_parenthetical_region_language_releases_under_one_game() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-ds", "Mario Kart DS (Europe) (En,Fr,De,Es,It)"), + Entry("nintendo-ds", "Mario Kart DS (USA, Australia) (En,Fr,De,Es,It)"), + Entry("nintendo-ds", "Mario Kart DS (Japan)"), + Entry("nintendo-ds", "Mario Kart DS (Korea)") + }); + + var game = plan.Games.Should().ContainSingle(x => x.SystemKey == "nintendo-ds" && x.GameTitle == "Mario Kart DS").Subject; + + game.RegionLanguageComponents.Select(x => x.SlotLabel).Should().BeEquivalentTo( + "Europe (En,Fr,De,Es,It)", + "USA, Australia (En,Fr,De,Es,It)", + "Japan", + "Korea"); + } + + [Test] + public void should_keep_revision_tags_on_the_region_component_label() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-gba", "Example Game (USA)"), + Entry("nintendo-gba", "Example Game (USA) (Rev 1)"), + Entry("nintendo-gba", "Example Game (USA) (Rev 2)") + }); + + var game = plan.Games.Should().ContainSingle(x => x.SystemKey == "nintendo-gba" && x.GameTitle == "Example Game").Subject; + + game.RegionLanguageComponents.Select(x => x.SlotLabel).Should().BeEquivalentTo( + "USA", + "USA (Rev 1)", + "USA (Rev 2)"); + } + + [Test] + public void should_build_DownloadPlayComponents_only_when_parent_mapping_is_explicit() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-ds", "Mario Kart DS (USA)"), + Entry("nintendo-ds", "Mario Kart DS (Download Play)", parentCanonicalName: "Mario Kart DS") + }); + + var game = plan.Games.Should().ContainSingle(x => x.SystemKey == "nintendo-ds" && x.GameTitle == "Mario Kart DS").Subject; + + game.RegionLanguageComponents.Should().ContainSingle(x => x.SlotLabel == "USA"); + game.DownloadPlayComponents.Should().ContainSingle(x => x.SlotLabel == "Download Play"); + plan.StandaloneGames.Should().BeEmpty(); + } + + [Test] + public void should_build_download_play_source_entries_as_multiboot_game_components() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo---nintendo-ds--download-play", "Mario Kart DS (Europe) (Demo) (Download Station Vol. 1)"), + Entry("nintendo---nintendo-ds--download-play", "Mario Kart DS (USA) (Demo) (Nintendo Channel)") + }); + + var game = plan.Games.Should().ContainSingle(x => x.SystemKey == "nintendo---nintendo-ds--download-play" && x.GameTitle == "Mario Kart DS").Subject; + + game.RegionLanguageComponents.Select(x => x.SlotLabel).Should().BeEquivalentTo( + "Europe (Demo) (Download Station Vol. 1)", + "USA (Demo) (Nintendo Channel)"); + game.RegionLanguageComponents.Should().OnlyContain(x => x.ComponentType == NoIntroRomComponentType.Multiboot); + plan.StandaloneGames.Should().BeEmpty(); + } + + [Test] + public void should_build_kiosk_releases_as_game_variants() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-ds", "New Super Mario Bros. Demo (Kiosk)"), + Entry("nintendo-ds", "Pokemon Distribution 2011 (USA) (Wi-Fi Kiosk) (Save Data)") + }); + + plan.Games.Should().ContainSingle(x => x.GameTitle == "New Super Mario Bros.") + .Subject.RegionLanguageComponents.Should().ContainSingle(x => x.SlotLabel == "Kiosk"); + plan.Games.Should().ContainSingle(x => x.GameTitle == "Pokemon Distribution 2011") + .Subject.RegionLanguageComponents.Should().ContainSingle(x => x.SlotLabel == "USA (Wi-Fi Kiosk) (Save Data)"); + plan.StandaloneGames.Should().BeEmpty(); + } + + [Test] + public void should_build_NoIntroStandaloneGames_for_clean_standalone_products() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-gba", "Game Boy Advance Video - Cartoon Network Collection - Volume 1 (USA)"), + Entry("nintendo-gba", "Nintendo - Game Boy Advance (Play-Yan)"), + Entry("nintendo-ds", "DSvision SD cards - Aquarium Tour (Japan)") + }); + + plan.Games.Should().BeEmpty(); + plan.StandaloneGames.Select(x => x.Title).Should().BeEquivalentTo( + "Game Boy Advance Video - Cartoon Network Collection - Volume 1 (USA)", + "Nintendo - Game Boy Advance (Play-Yan)", + "DSvision SD cards - Aquarium Tour (Japan)"); + } + + [Test] + public void should_build_NoPhantomRegionComponents_when_catalog_release_is_missing() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-gba", "Advance Wars (USA)"), + Entry("nintendo-gba", "Advance Wars (Germany)") + }); + + var game = plan.Games.Should().ContainSingle(x => x.GameTitle == "Advance Wars").Subject; + var labels = game.RegionLanguageComponents.Select(x => x.SlotLabel).ToList(); + + labels.Should().BeEquivalentTo("USA", "Germany"); + labels.Should().NotContain(new[] { "Europe", "Japan", "France", "Italy", "Spain" }); + } + + [Test] + public void should_build_NoUnmappedDownloadPlayComponents_as_children() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-ds", "Mario Party DS (Download Play)") + }); + + plan.Games.Should().BeEmpty(); + plan.StandaloneGames.Should().ContainSingle(x => x.Title == "Mario Party DS (Download Play)" && x.ComponentType == NoIntroRomComponentType.Multiboot); + } + + [Test] + public void should_build_StandaloneProductsNotBaseGameVariants() + { + var plan = _subject.BuildCatalogPlan(new[] + { + Entry("nintendo-ds", "Pokemon Dash (USA)"), + Entry("nintendo-gba", "Nintendo - Game Boy Advance (BIOS) (World)"), + Entry("nintendo-ds", "New Super Mario Bros. Demo (Kiosk)"), + Entry("nintendo-ds", "DSvision SD cards - Aquarium Tour (Japan)") + }); + + var game = plan.Games.Should().ContainSingle(x => x.GameTitle == "Pokemon Dash").Subject; + + game.RegionLanguageComponents.Should().ContainSingle(x => x.SlotLabel == "USA"); + game.DownloadPlayComponents.Should().BeEmpty(); + + plan.StandaloneGames.Select(x => x.Title).Should().BeEquivalentTo( + "Nintendo - Game Boy Advance (BIOS) (World)", + "DSvision SD cards - Aquarium Tour (Japan)"); + } + + private static NoIntroCatalogEntry Entry(string systemKey, string canonicalName, string parentCanonicalName = null) + { + return new NoIntroCatalogEntry + { + SystemKey = systemKey, + CanonicalName = canonicalName, + CanonicalFileName = $"{canonicalName}.zip", + ParentCanonicalName = parentCanonicalName + }; + } + } +} diff --git a/src/NzbDrone.Core.Test/RomCatalog/NoIntroEndToEndFixture.cs b/src/NzbDrone.Core.Test/RomCatalog/NoIntroEndToEndFixture.cs new file mode 100644 index 00000000..7e14056f --- /dev/null +++ b/src/NzbDrone.Core.Test/RomCatalog/NoIntroEndToEndFixture.cs @@ -0,0 +1,144 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Games; +using NzbDrone.Core.Organizer; +using NzbDrone.Core.RomCatalog; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.RomCatalog +{ + [TestFixture] + public class NoIntroEndToEndFixture : DbTest + { + private NoIntroCatalogSourceRepository _sourceRepository; + private NoIntroCatalogEntryRepository _entryRepository; + private NoIntroCatalogHashRepository _hashRepository; + private NoIntroVerificationSetRepository _verificationSetRepository; + private NoIntroVerificationResultRepository _resultRepository; + private NoIntroVerificationService _verificationService; + private NoIntroComponentClassifier _componentClassifier; + private string _tempRoot; + + [SetUp] + public void Setup() + { + _sourceRepository = Mocker.Resolve(); + _entryRepository = Mocker.Resolve(); + _hashRepository = Mocker.Resolve(); + _verificationSetRepository = Mocker.Resolve(); + _resultRepository = Mocker.Resolve(); + _componentClassifier = new NoIntroComponentClassifier(); + + Mocker.GetMock() + .Setup(x => x.GetConfig()) + .Returns(new NamingConfig { RenameProfile = RenameProfile.NoIntroPreserveById }); + + _verificationService = Mocker.Resolve(); + _tempRoot = Path.Combine(TempFolder, "nointro-e2e"); + Directory.CreateDirectory(_tempRoot); + } + + [Test] + public void EndToEndNoIntro_should_validate_gba_nds_components_and_standalone_products() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/nointro.dat", + CatalogVersion = "fixture" + }); + + var fzeroBytes = new byte[] { 1, 2, 3, 4 }; + var marioKartBytes = new byte[] { 5, 6, 7, 8 }; + var fzero = InsertEntry(source.Id, "nintendo-gba", "F-Zero for Game Boy Advance (Japan)", "F-Zero for Game Boy Advance (Japan).zip"); + var marioKart = InsertEntry(source.Id, "nintendo-ds", "Mario Kart DS (USA)", "Mario Kart DS (USA).nds"); + + InsertHash(fzero.Id, fzeroBytes); + InsertHash(marioKart.Id, marioKartBytes); + InsertEntry(source.Id, "nintendo-ds", "Mario Kart DS (Download Play)", "Mario Kart DS (Download Play).nds", "Mario Kart DS"); + InsertEntry(source.Id, "nintendo-gba", "Pokemon Emerald Version (Germany)", "Pokemon Emerald Version (Germany).zip"); + InsertEntry(source.Id, "nintendo-gba", "Game Boy Advance Video - Cartoon Network Collection - Volume 1 (USA)", "Game Boy Advance Video - Cartoon Network Collection - Volume 1 (USA).zip"); + + var verificationSet = _verificationSetRepository.Insert(new NoIntroVerificationSet + { + CatalogSourceId = source.Id, + SystemKey = "mixed-fixture", + RootPath = _tempRoot, + Enabled = true + }); + + var fzeroZip = WriteZip("Nintendo/Game Boy Advance/GBA (by-id)/0001 - F-Zero for Game Boy Advance (Japan).zip", "F-Zero for Game Boy Advance (Japan).gba", fzeroBytes); + var marioKartRaw = WriteBytes("Nintendo/Nintendo DS/nds/Mario Kart DS (USA).nds", marioKartBytes); + + _verificationService.Verify(verificationSet.Id, new[] { fzeroZip, marioKartRaw }); + var results = _resultRepository.All().ToList(); + var plan = _componentClassifier.BuildCatalogPlan(_entryRepository.GetBySourceId(source.Id)); + + results.Should().Contain(x => x.CatalogEntryId == fzero.Id && x.VerificationStatus == NoIntroVerificationStatus.Verified); + results.Should().Contain(x => x.CatalogEntryId == marioKart.Id && x.VerificationStatus == NoIntroVerificationStatus.Verified); + plan.Games.Should().ContainSingle(x => x.GameTitle == "Mario Kart DS") + .Subject.DownloadPlayComponents.Should().ContainSingle(x => x.SlotLabel == "Download Play"); + plan.Games.Should().ContainSingle(x => x.GameTitle == "Pokemon Emerald Version") + .Subject.RegionLanguageComponents.Should().ContainSingle(x => x.SlotLabel == "Germany"); + plan.StandaloneGames.Should().ContainSingle(x => x.Title.StartsWith("Game Boy Advance Video", StringComparison.Ordinal)); + } + + private NoIntroCatalogEntry InsertEntry(int sourceId, string systemKey, string canonicalName, string fileName) + { + return _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = sourceId, + SystemKey = systemKey, + CanonicalName = canonicalName, + CanonicalFileName = fileName, + PlatformFamily = PlatformFamily.Nintendo + }); + } + + private NoIntroCatalogEntry InsertEntry(int sourceId, string systemKey, string canonicalName, string fileName, string parentCanonicalName) + { + var entry = InsertEntry(sourceId, systemKey, canonicalName, fileName); + entry.ParentCanonicalName = parentCanonicalName; + return _entryRepository.Update(entry); + } + + private void InsertHash(int entryId, byte[] content) + { + using var sha1 = System.Security.Cryptography.SHA1.Create(); + + _hashRepository.Insert(new NoIntroCatalogHash + { + CatalogEntryId = entryId, + HashType = "sha1", + HashValue = BitConverter.ToString(sha1.ComputeHash(content)).Replace("-", string.Empty), + IsPrimary = true, + IsBadDump = false + }); + } + + private string WriteBytes(string relativePath, byte[] content) + { + var path = Path.Combine(_tempRoot, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + File.WriteAllBytes(path, content); + return path; + } + + private string WriteZip(string relativePath, string memberName, byte[] content) + { + var path = Path.Combine(_tempRoot, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + using var archive = ZipFile.Open(path, ZipArchiveMode.Create); + var entry = archive.CreateEntry(memberName); + using var stream = entry.Open(); + stream.Write(content, 0, content.Length); + + return path; + } + } +} diff --git a/src/NzbDrone.Core.Test/RomCatalog/NoIntroVerificationRenameProfileFixture.cs b/src/NzbDrone.Core.Test/RomCatalog/NoIntroVerificationRenameProfileFixture.cs new file mode 100644 index 00000000..396f55a7 --- /dev/null +++ b/src/NzbDrone.Core.Test/RomCatalog/NoIntroVerificationRenameProfileFixture.cs @@ -0,0 +1,159 @@ +using System.IO; +using System.IO.Compression; +using System.Linq; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Games; +using NzbDrone.Core.Organizer; +using NzbDrone.Core.RomCatalog; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.RomCatalog +{ + [TestFixture] + public class NoIntroVerificationRenameProfileFixture : DbTest + { + private NoIntroCatalogSourceRepository _sourceRepository; + private NoIntroCatalogEntryRepository _entryRepository; + private NoIntroCatalogHashRepository _hashRepository; + private NoIntroVerificationSetRepository _verificationSetRepository; + private NoIntroVerificationResultRepository _resultRepository; + private NoIntroVerificationService _subject; + private string _tempRoot; + + [SetUp] + public void Setup() + { + _sourceRepository = Mocker.Resolve(); + _entryRepository = Mocker.Resolve(); + _hashRepository = Mocker.Resolve(); + _verificationSetRepository = Mocker.Resolve(); + _resultRepository = Mocker.Resolve(); + _subject = Mocker.Resolve(); + _tempRoot = Path.Combine(TempFolder, "rename-profile-roms"); + Directory.CreateDirectory(_tempRoot); + } + + [Test] + public void RenameProfile_should_keep_by_id_file_verified_when_preserve_profile_is_selected() + { + SetRenameProfile(RenameProfile.NoIntroPreserveById); + var verificationSet = CreateVerificationSet(); + var zipPath = CreateVerifiedZip("0001 - F-Zero for Game Boy Advance (Japan).zip"); + + _subject.Verify(verificationSet.Id, new[] { zipPath }); + + var result = _resultRepository.All().Single(x => !x.IsMissing); + result.VerificationStatus.Should().Be(NoIntroVerificationStatus.Verified); + result.ExpectedFileName.Should().Be("0001 - F-Zero for Game Boy Advance (Japan).zip"); + } + + [Test] + public void RenameProfile_should_mark_same_by_id_file_as_mismatch_when_canonical_profile_is_selected() + { + SetRenameProfile(RenameProfile.NoIntroCanonical); + var verificationSet = CreateVerificationSet(); + var zipPath = CreateVerifiedZip("0001 - F-Zero for Game Boy Advance (Japan).zip"); + + _subject.Verify(verificationSet.Id, new[] { zipPath }); + + var result = _resultRepository.All().Single(x => !x.IsMissing); + result.VerificationStatus.Should().Be(NoIntroVerificationStatus.NameMismatch); + result.ExpectedFileName.Should().Be("F-Zero for Game Boy Advance (Japan).zip"); + } + + [Test] + public void RenameProfile_should_keep_existing_default_behavior_under_normal_gamarr_profile() + { + SetRenameProfile(RenameProfile.Gamarr); + var verificationSet = CreateVerificationSet(); + var zipPath = CreateVerifiedZip("0001 - F-Zero for Game Boy Advance (Japan).zip"); + + _subject.Verify(verificationSet.Id, new[] { zipPath }); + + var result = _resultRepository.All().Single(x => !x.IsMissing); + result.VerificationStatus.Should().Be(NoIntroVerificationStatus.NameMismatch); + result.ExpectedFileName.Should().Be("F-Zero for Game Boy Advance (Japan).zip"); + } + + [Test] + public void RenameProfileFailure_should_not_mark_by_id_file_as_name_mismatch_when_preserve_profile_is_selected() + { + SetRenameProfile(RenameProfile.NoIntroPreserveById); + var verificationSet = CreateVerificationSet(); + var zipPath = CreateVerifiedZip("0001 - F-Zero for Game Boy Advance (Japan).zip"); + + _subject.Verify(verificationSet.Id, new[] { zipPath }); + + var result = _resultRepository.All().Single(x => !x.IsMissing); + result.VerificationStatus.Should().NotBe(NoIntroVerificationStatus.NameMismatch); + result.ExpectedFileName.Should().Be("0001 - F-Zero for Game Boy Advance (Japan).zip"); + } + + private void SetRenameProfile(RenameProfile renameProfile) + { + Mocker.GetMock() + .Setup(x => x.GetConfig()) + .Returns(new NamingConfig + { + RenameProfile = renameProfile + }); + } + + private NoIntroVerificationSet CreateVerificationSet() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/catalog.dat" + }); + + var entry = _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "F-Zero for Game Boy Advance (Japan)", + CanonicalFileName = "F-Zero for Game Boy Advance (Japan).zip", + PlatformFamily = PlatformFamily.Nintendo + }); + + InsertSha1Hash(entry.Id, new byte[] { 1, 2, 3, 4 }); + + return _verificationSetRepository.Insert(new NoIntroVerificationSet + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + RootPath = _tempRoot, + Enabled = true + }); + } + + private void InsertSha1Hash(int entryId, byte[] content) + { + using var sha1 = System.Security.Cryptography.SHA1.Create(); + var hash = System.BitConverter.ToString(sha1.ComputeHash(content)).Replace("-", string.Empty); + + _hashRepository.Insert(new NoIntroCatalogHash + { + CatalogEntryId = entryId, + HashType = "sha1", + HashValue = hash, + IsPrimary = true, + IsBadDump = false + }); + } + + private string CreateVerifiedZip(string archiveName) + { + var path = Path.Combine(_tempRoot, archiveName); + var content = new byte[] { 1, 2, 3, 4 }; + + using var archive = ZipFile.Open(path, ZipArchiveMode.Create); + var entry = archive.CreateEntry("F-Zero.gba"); + using var stream = entry.Open(); + stream.Write(content, 0, content.Length); + + return path; + } + } +} diff --git a/src/NzbDrone.Core.Test/RomCatalog/NoIntroVerificationServiceFixture.cs b/src/NzbDrone.Core.Test/RomCatalog/NoIntroVerificationServiceFixture.cs new file mode 100644 index 00000000..10b8829b --- /dev/null +++ b/src/NzbDrone.Core.Test/RomCatalog/NoIntroVerificationServiceFixture.cs @@ -0,0 +1,216 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Games; +using NzbDrone.Core.Organizer; +using NzbDrone.Core.RomCatalog; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.RomCatalog +{ + [TestFixture] + public class NoIntroVerificationServiceFixture : DbTest + { + private NoIntroCatalogSourceRepository _sourceRepository; + private NoIntroCatalogEntryRepository _entryRepository; + private NoIntroCatalogHashRepository _hashRepository; + private NoIntroVerificationSetRepository _verificationSetRepository; + private NoIntroVerificationResultRepository _resultRepository; + private NoIntroVerificationService _subject; + private string _tempRoot; + + [SetUp] + public void Setup() + { + _sourceRepository = Mocker.Resolve(); + _entryRepository = Mocker.Resolve(); + _hashRepository = Mocker.Resolve(); + _verificationSetRepository = Mocker.Resolve(); + _resultRepository = Mocker.Resolve(); + Mocker.GetMock() + .Setup(x => x.GetConfig()) + .Returns(NamingConfig.Default); + + _subject = Mocker.Resolve(); + _tempRoot = Path.Combine(TempFolder, "nointro-roms"); + Directory.CreateDirectory(_tempRoot); + } + + [Test] + public void should_verify_raw_and_zip_roms_and_mark_duplicates_missing_and_bad_dumps() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/catalog.dat" + }); + + var goodBytes = new byte[] { 1, 2, 3, 4 }; + var badBytes = new byte[] { 9, 9, 9, 9 }; + + var verifiedEntry = _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "Verified Game", + CanonicalFileName = "Verified Game.gba", + PlatformFamily = PlatformFamily.Nintendo + }); + + InsertSha1Hash(verifiedEntry.Id, goodBytes, false); + + var badDumpEntry = _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "Bad Dump Game", + CanonicalFileName = "Bad Dump Game.gba", + PlatformFamily = PlatformFamily.Nintendo + }); + + InsertSha1Hash(badDumpEntry.Id, badBytes, true); + + var missingEntry = _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "Missing Game", + CanonicalFileName = "Missing Game.gba", + PlatformFamily = PlatformFamily.Nintendo + }); + + InsertSha1Hash(missingEntry.Id, new byte[] { 7, 7, 7, 7 }, false); + + var verificationSet = _verificationSetRepository.Insert(new NoIntroVerificationSet + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + RootPath = _tempRoot, + Enabled = true + }); + + var rawPath = WriteBytes("Verified Game.gba", goodBytes); + var mismatchZip = WriteZip("0001 - Verified Game.zip", "Verified Game.gba", goodBytes); + var badDumpPath = WriteBytes("Bad Dump Game.gba", badBytes); + var unknownPath = WriteBytes("Unknown Game.gba", new byte[] { 5, 6, 7, 8 }); + var ambiguousZip = WriteMultiZip("Ambiguous.zip", ("one.gba", goodBytes), ("two.gba", badBytes)); + + _subject.Verify(verificationSet.Id, new[] { rawPath, mismatchZip, badDumpPath, unknownPath, ambiguousZip }); + + var results = _resultRepository.All().ToList(); + + results.Should().Contain(x => x.ActualFileName == "Verified Game.gba" && x.VerificationStatus == NoIntroVerificationStatus.Verified && x.IsDuplicate); + results.Should().Contain(x => x.ActualFileName == "0001 - Verified Game.zip" && x.VerificationStatus == NoIntroVerificationStatus.NameMismatch && x.IsDuplicate); + results.Should().Contain(x => x.ActualFileName == "Bad Dump Game.gba" && x.VerificationStatus == NoIntroVerificationStatus.BadDump); + results.Should().Contain(x => x.ActualFileName == "Unknown Game.gba" && x.VerificationStatus == NoIntroVerificationStatus.Unknown && !x.IsMissing); + results.Should().Contain(x => x.ActualFileName == "Ambiguous.zip" && x.VerificationStatus == NoIntroVerificationStatus.Unknown && x.MemberPath == null); + results.Should().Contain(x => x.CatalogEntryId == missingEntry.Id && x.IsMissing); + } + + [Test] + public void should_continue_when_one_file_cannot_be_read() + { + var source = _sourceRepository.Insert(new NoIntroCatalogSource + { + Name = "No-Intro", + SourceUrl = "https://example.invalid/gba.dat" + }); + + var goodBytes = new byte[] { 1, 2, 3, 4 }; + + var verifiedEntry = _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "Verified Game", + CanonicalFileName = "Verified Game.gba", + PlatformFamily = PlatformFamily.Nintendo + }); + + InsertSha1Hash(verifiedEntry.Id, goodBytes, false); + + var missingEntry = _entryRepository.Insert(new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + CanonicalName = "Missing Game", + CanonicalFileName = "Missing Game.gba", + PlatformFamily = PlatformFamily.Nintendo + }); + + InsertSha1Hash(missingEntry.Id, new byte[] { 7, 7, 7, 7 }, false); + + var verificationSet = _verificationSetRepository.Insert(new NoIntroVerificationSet + { + CatalogSourceId = source.Id, + SystemKey = "nintendo-gba", + RootPath = _tempRoot, + Enabled = true + }); + + var rawPath = WriteBytes("Verified Game.gba", goodBytes); + var unreadablePath = Path.Combine(_tempRoot, "Unreadable Game.gba"); + + _subject.Verify(verificationSet.Id, new[] { rawPath, unreadablePath }); + + var results = _resultRepository.All().ToList(); + + results.Should().Contain(x => x.ActualFileName == "Verified Game.gba" && x.VerificationStatus == NoIntroVerificationStatus.Verified); + results.Should().Contain(x => x.ActualFileName == "Unreadable Game.gba" && x.VerificationStatus == NoIntroVerificationStatus.Unknown && !x.IsMissing); + results.Should().Contain(x => x.CatalogEntryId == missingEntry.Id && x.IsMissing); + } + + private void InsertSha1Hash(int entryId, byte[] content, bool isBadDump) + { + using var sha1 = System.Security.Cryptography.SHA1.Create(); + var hash = BitConverter.ToString(sha1.ComputeHash(content)).Replace("-", string.Empty); + + _hashRepository.Insert(new NoIntroCatalogHash + { + CatalogEntryId = entryId, + HashType = "sha1", + HashValue = hash, + IsPrimary = true, + IsBadDump = isBadDump + }); + } + + private string WriteBytes(string fileName, byte[] content) + { + var path = Path.Combine(_tempRoot, fileName); + File.WriteAllBytes(path, content); + return path; + } + + private string WriteZip(string archiveName, string memberName, byte[] content) + { + var path = Path.Combine(_tempRoot, archiveName); + + using var archive = ZipFile.Open(path, ZipArchiveMode.Create); + var entry = archive.CreateEntry(memberName); + using var stream = entry.Open(); + stream.Write(content, 0, content.Length); + + return path; + } + + private string WriteMultiZip(string archiveName, params (string Name, byte[] Content)[] members) + { + var path = Path.Combine(_tempRoot, archiveName); + + using var archive = ZipFile.Open(path, ZipArchiveMode.Create); + + foreach (var member in members) + { + var entry = archive.CreateEntry(member.Name); + using var stream = entry.Open(); + stream.Write(member.Content, 0, member.Content.Length); + } + + return path; + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/008_add_nointro_catalog.cs b/src/NzbDrone.Core/Datastore/Migration/008_add_nointro_catalog.cs new file mode 100644 index 00000000..61afda93 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/008_add_nointro_catalog.cs @@ -0,0 +1,70 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(8)] + public class add_nointro_catalog : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Create.TableForModel("NoIntroCatalogSources") + .WithColumn("Name").AsString().NotNullable() + .WithColumn("SourceUrl").AsString().NotNullable() + .WithColumn("PinnedRevision").AsString().Nullable() + .WithColumn("CatalogVersion").AsString().Nullable() + .WithColumn("LastSuccessfulSync").AsDateTime().Nullable() + .WithColumn("LastAttemptedSync").AsDateTime().Nullable() + .WithColumn("LastSyncError").AsString().Nullable(); + + Create.TableForModel("NoIntroCatalogEntries") + .WithColumn("CatalogSourceId").AsInt32().NotNullable().Indexed() + .WithColumn("SystemKey").AsString().NotNullable().Indexed() + .WithColumn("CanonicalName").AsString().NotNullable() + .WithColumn("CanonicalFileName").AsString().NotNullable() + .WithColumn("PlatformFamily").AsInt32().NotNullable(); + + Create.TableForModel("NoIntroCatalogHashes") + .WithColumn("CatalogEntryId").AsInt32().NotNullable().Indexed() + .WithColumn("HashType").AsString().NotNullable() + .WithColumn("HashValue").AsString().NotNullable().Indexed() + .WithColumn("IsPrimary").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("IsBadDump").AsBoolean().NotNullable().WithDefaultValue(false); + + Create.TableForModel("NoIntroSystemMappings") + .WithColumn("SystemKey").AsString().NotNullable().Indexed() + .WithColumn("DisplayName").AsString().NotNullable() + .WithColumn("PlatformFamily").AsInt32().NotNullable() + .WithColumn("RootRelativePathPattern").AsString().Nullable(); + + Create.TableForModel("NoIntroVerificationSets") + .WithColumn("CatalogSourceId").AsInt32().NotNullable().Indexed() + .WithColumn("SystemKey").AsString().NotNullable().Indexed() + .WithColumn("RootPath").AsString().NotNullable() + .WithColumn("Enabled").AsBoolean().NotNullable().WithDefaultValue(true); + + Create.TableForModel("NoIntroVerificationSnapshots") + .WithColumn("VerificationSetId").AsInt32().NotNullable().Indexed() + .WithColumn("CatalogSourceId").AsInt32().NotNullable().Indexed() + .WithColumn("CatalogRevision").AsString().Nullable() + .WithColumn("StartedAt").AsDateTime().NotNullable() + .WithColumn("CompletedAt").AsDateTime().Nullable(); + + Create.TableForModel("NoIntroVerificationResults") + .WithColumn("SnapshotId").AsInt32().NotNullable().Indexed() + .WithColumn("VerificationSetId").AsInt32().NotNullable().Indexed() + .WithColumn("CatalogEntryId").AsInt32().Nullable().Indexed() + .WithColumn("RelativePath").AsString().NotNullable() + .WithColumn("ArchivePath").AsString().Nullable() + .WithColumn("MemberPath").AsString().Nullable() + .WithColumn("ActualFileName").AsString().NotNullable() + .WithColumn("ExpectedFileName").AsString().Nullable() + .WithColumn("HashType").AsString().Nullable() + .WithColumn("HashValue").AsString().Nullable().Indexed() + .WithColumn("VerificationStatus").AsInt32().NotNullable() + .WithColumn("IsDuplicate").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("IsMissing").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("VerifiedAt").AsDateTime().NotNullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/009_add_rename_profile_to_naming_config.cs b/src/NzbDrone.Core/Datastore/Migration/009_add_rename_profile_to_naming_config.cs new file mode 100644 index 00000000..ba807acf --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/009_add_rename_profile_to_naming_config.cs @@ -0,0 +1,19 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; +using NzbDrone.Core.Organizer; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(9)] + public class add_rename_profile_to_naming_config : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("NamingConfig") + .AddColumn("RenameProfile") + .AsInt32() + .NotNullable() + .WithDefaultValue((int)RenameProfile.Gamarr); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/010_add_parent_canonical_name_to_nointro_catalog_entry.cs b/src/NzbDrone.Core/Datastore/Migration/010_add_parent_canonical_name_to_nointro_catalog_entry.cs new file mode 100644 index 00000000..8253e0a6 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/010_add_parent_canonical_name_to_nointro_catalog_entry.cs @@ -0,0 +1,17 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(10)] + public class add_parent_canonical_name_to_nointro_catalog_entry : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("NoIntroCatalogEntries") + .AddColumn("ParentCanonicalName") + .AsString() + .Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/011_seed_nointro_catalog_sources.cs b/src/NzbDrone.Core/Datastore/Migration/011_seed_nointro_catalog_sources.cs new file mode 100644 index 00000000..3ddf1a0b --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/011_seed_nointro_catalog_sources.cs @@ -0,0 +1,36 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(11)] + public class seed_nointro_catalog_sources : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro Nintendo Game Boy", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Game%20Boy.dat" + }); + + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro Nintendo Game Boy Color", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Game%20Boy%20Color.dat" + }); + + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro Nintendo Game Boy Advance", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Game%20Boy%20Advance.dat" + }); + + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro Nintendo DS", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20DS.dat" + }); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/012_add_numbered_nointro_filenames.cs b/src/NzbDrone.Core/Datastore/Migration/012_add_numbered_nointro_filenames.cs new file mode 100644 index 00000000..c3e6a8a5 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/012_add_numbered_nointro_filenames.cs @@ -0,0 +1,16 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(12)] + public class add_numbered_nointro_filenames : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("NoIntroCatalogEntries") + .AddColumn("ReleaseNumber").AsString().Nullable() + .AddColumn("NumberedCanonicalFileName").AsString().Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/013_seed_additional_nointro_catalog_sources.cs b/src/NzbDrone.Core/Datastore/Migration/013_seed_additional_nointro_catalog_sources.cs new file mode 100644 index 00000000..3698035b --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/013_seed_additional_nointro_catalog_sources.cs @@ -0,0 +1,24 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(13)] + public class seed_additional_nointro_catalog_sources : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro Nintendo DS Download Play", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20DS%20%28Download%20Play%29.dat" + }); + + // Note: the DAT-o-MATIC-only niche systems (GBA Multiboot/e-Reader/ + // Play-Yan/Video, DSvision) are intentionally NOT seeded — the + // datomatic:// scrape path is currently non-functional, so seeding + // them would surface permanently-failing sources. The support code + // remains; re-seed once FetchDatOMaticNumbered is hardened. + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/014_seed_3ds_nointro_catalog_sources.cs b/src/NzbDrone.Core/Datastore/Migration/014_seed_3ds_nointro_catalog_sources.cs new file mode 100644 index 00000000..165abda2 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/014_seed_3ds_nointro_catalog_sources.cs @@ -0,0 +1,36 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(14)] + public class seed_3ds_nointro_catalog_sources : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro Nintendo 3DS", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%203DS.dat" + }); + + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro Nintendo 3DS Digital", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%203DS%20%28Digital%29.dat" + }); + + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro New Nintendo 3DS", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20New%20Nintendo%203DS.dat" + }); + + Insert.IntoTable("NoIntroCatalogSources").Row(new + { + Name = "No-Intro New Nintendo 3DS Digital", + SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20New%20Nintendo%203DS%20%28Digital%29.dat" + }); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/015_seed_nintendo_sony_nointro_catalog_sources.cs b/src/NzbDrone.Core/Datastore/Migration/015_seed_nintendo_sony_nointro_catalog_sources.cs new file mode 100644 index 00000000..65dd0602 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/015_seed_nintendo_sony_nointro_catalog_sources.cs @@ -0,0 +1,31 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(15)] + public class seed_nintendo_sony_nointro_catalog_sources : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Nintendo Entertainment System", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20Entertainment%20System.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Super Nintendo Entertainment System", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Super%20Nintendo%20Entertainment%20System.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Nintendo 64", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%2064.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Nintendo 64DD", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%2064DD.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Family Computer Disk System", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Family%20Computer%20Disk%20System.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Virtual Boy", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Virtual%20Boy.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Pokemon Mini", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Pokemon%20Mini.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Satellaview", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Satellaview.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Sufami Turbo", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Sufami%20Turbo.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Nintendo DSi", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20DSi.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Wii Digital", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Wii%20%28Digital%29.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Wii U Digital", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Wii%20U%20%28Digital%29.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Sony PlayStation Portable", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Portable.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Sony PlayStation Portable PSN", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Portable%20%28PSN%29.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Sony PlayStation Portable PSX2PSP", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Portable%20%28PSX2PSP%29.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Sony PlayStation Vita", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Vita.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Sony PlayStation Vita PSN", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Vita%20%28PSN%29.dat" }); + Insert.IntoTable("NoIntroCatalogSources").Row(new { Name = "No-Intro Sony PlayStation 3 PSN", SourceUrl = "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%203%20%28PSN%29.dat" }); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 306f3c9b..aec6ed26 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -40,6 +40,7 @@ using NzbDrone.Core.Qualities; using NzbDrone.Core.RemotePathMappings; using NzbDrone.Core.RootFolders; +using NzbDrone.Core.RomCatalog; using NzbDrone.Core.Tags; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Update.History; @@ -153,6 +154,13 @@ public static void Map() .Ignore(e => e.RemoteGame); Mapper.Entity("RemotePathMappings").RegisterModel(); + Mapper.Entity("NoIntroCatalogSources").RegisterModel(); + Mapper.Entity("NoIntroCatalogEntries").RegisterModel(); + Mapper.Entity("NoIntroCatalogHashes").RegisterModel(); + Mapper.Entity("NoIntroSystemMappings").RegisterModel(); + Mapper.Entity("NoIntroVerificationSets").RegisterModel(); + Mapper.Entity("NoIntroVerificationSnapshots").RegisterModel(); + Mapper.Entity("NoIntroVerificationResults").RegisterModel(); Mapper.Entity("Tags").RegisterModel(); Mapper.Entity("ReleaseProfiles").RegisterModel(); diff --git a/src/NzbDrone.Core/DecisionEngine/Specifications/PlatformSpecification.cs b/src/NzbDrone.Core/DecisionEngine/Specifications/PlatformSpecification.cs index 9caac931..0c895f02 100644 --- a/src/NzbDrone.Core/DecisionEngine/Specifications/PlatformSpecification.cs +++ b/src/NzbDrone.Core/DecisionEngine/Specifications/PlatformSpecification.cs @@ -16,6 +16,14 @@ public class PlatformSpecification : IDownloadDecisionEngineSpecification PlatformFamily.PlayStation, PlatformFamily.Xbox, PlatformFamily.Nintendo, + PlatformFamily.NintendoSwitch, + PlatformFamily.NintendoWiiU, + PlatformFamily.NintendoWii, + PlatformFamily.Nintendo3DS, + PlatformFamily.NintendoDS, + PlatformFamily.NintendoGBA, + PlatformFamily.NintendoGB, + PlatformFamily.NintendoGBC, PlatformFamily.Sega, PlatformFamily.Atari, PlatformFamily.Mobile @@ -43,7 +51,7 @@ public virtual DownloadSpecDecision IsSatisfiedBy(RemoteGame subject, SearchCrit if (gamePlatform != PlatformFamily.Unknown) { - if (releasePlatform == gamePlatform) + if (GamePlatform.PlatformMatches(gamePlatform, releasePlatform)) { return DownloadSpecDecision.Accept(); } @@ -73,7 +81,7 @@ public virtual DownloadSpecDecision IsSatisfiedBy(RemoteGame subject, SearchCrit return DownloadSpecDecision.Accept(); } - if (preferredPlatforms.Contains(releasePlatform)) + if (preferredPlatforms.Any(platform => GamePlatform.PlatformMatches(platform, releasePlatform))) { _logger.Debug("Release platform {0} matches preferred platforms, accepting.", releasePlatform); return DownloadSpecDecision.Accept(); diff --git a/src/NzbDrone.Core/Games/Components/GameComponent.cs b/src/NzbDrone.Core/Games/Components/GameComponent.cs index bbb3b5bd..6d09571c 100644 --- a/src/NzbDrone.Core/Games/Components/GameComponent.cs +++ b/src/NzbDrone.Core/Games/Components/GameComponent.cs @@ -7,7 +7,12 @@ public enum GameComponentType { Base = 1, Update = 2, - Dlc = 3 + Dlc = 3, + NoIntroRetailRom = 4, + NoIntroMultiboot = 5, + NoIntroVideo = 6, + NoIntroBios = 7, + NoIntroRomhackOrUnverified = 8 } /// diff --git a/src/NzbDrone.Core/Games/Components/GameComponentService.cs b/src/NzbDrone.Core/Games/Components/GameComponentService.cs index 3a13ce77..b84397b3 100644 --- a/src/NzbDrone.Core/Games/Components/GameComponentService.cs +++ b/src/NzbDrone.Core/Games/Components/GameComponentService.cs @@ -1,12 +1,15 @@ using System; using System.Collections.Generic; using System.Linq; +using System.IO; using NLog; +using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Core.Games.Events; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.RomCatalog; namespace NzbDrone.Core.Games.Components { @@ -28,16 +31,24 @@ public class GameComponentService : IGameComponentService, private readonly IGameComponentRepository _componentRepository; private readonly IMediaFileService _mediaFileService; private readonly IGameService _gameService; + private readonly INoIntroCatalogEntryRepository _noIntroCatalogEntryRepository; + private readonly IDiskProvider _diskProvider; + private readonly NoIntroGameComponentPlanner _noIntroComponentPlanner; private readonly Logger _logger; public GameComponentService(IGameComponentRepository componentRepository, - IMediaFileService mediaFileService, - IGameService gameService, - Logger logger) + IMediaFileService mediaFileService, + IGameService gameService, + INoIntroCatalogEntryRepository noIntroCatalogEntryRepository, + IDiskProvider diskProvider, + Logger logger) { _componentRepository = componentRepository; _mediaFileService = mediaFileService; _gameService = gameService; + _noIntroCatalogEntryRepository = noIntroCatalogEntryRepository; + _diskProvider = diskProvider; + _noIntroComponentPlanner = new NoIntroGameComponentPlanner(new NoIntroComponentClassifier()); _logger = logger; } @@ -90,6 +101,8 @@ public void EnsureComponents(Game game) { var existing = _componentRepository.GetByGame(game.Id); var files = _mediaFileService.GetFilesByGame(game.Id); + var noIntroEntries = (_noIntroCatalogEntryRepository.GetByPlatformFamily(game.Platform) ?? Enumerable.Empty()).ToList(); + var noIntroSlots = _noIntroComponentPlanner.GetSlots(game, noIntroEntries); MergeDuplicateDlcSlots(existing, files); @@ -127,9 +140,16 @@ public void EnsureComponents(Game game) } } + foreach (var slot in noIntroSlots) + { + FindOrStage(existing, toInsert, game, slot.ComponentType, slot.Key, slot.Title, monitored: false); + } + + var folderBackedFileSlot = ResolveFolderBackedFileSlot(game, files, noIntroSlots); + foreach (var file in files) { - var component = GetComponentForFile(existing, toInsert, game, file, baseComponent); + var component = GetComponentForFile(existing, toInsert, game, file, baseComponent, noIntroSlots, folderBackedFileSlot); if (component != null && file.ComponentId != component.Id) { @@ -153,7 +173,7 @@ public void EnsureComponents(Game game) foreach (var file in files) { - var component = GetComponentForFile(all, new List(), game, file, all.FirstOrDefault(c => c.ComponentType == GameComponentType.Base)); + var component = GetComponentForFile(all, new List(), game, file, all.FirstOrDefault(c => c.ComponentType == GameComponentType.Base), noIntroSlots, folderBackedFileSlot); if (component is { Id: > 0 } && file.ComponentId != component.Id) { @@ -212,8 +232,15 @@ private void MergeDuplicateDlcSlots(List existing, List } } - private GameComponent GetComponentForFile(List existing, List toInsert, Game game, GameFile file, GameComponent baseComponent) + private GameComponent GetComponentForFile(List existing, List toInsert, Game game, GameFile file, GameComponent baseComponent, List noIntroSlots, NoIntroGameComponentSlot folderBackedFileSlot) { + var noIntroSlot = _noIntroComponentPlanner.FindSlotForFile(noIntroSlots, file) ?? folderBackedFileSlot; + + if (noIntroSlot != null) + { + return FindOrStage(existing, toInsert, game, noIntroSlot.ComponentType, noIntroSlot.Key, noIntroSlot.Title, monitored: false); + } + if (file.RelativePath.IsNullOrWhiteSpace()) { return baseComponent; @@ -268,6 +295,25 @@ private static bool IsMetadataDlcKey(string key) return key.StartsWith("igdb:") || key.StartsWith("steam:"); } + private NoIntroGameComponentSlot ResolveFolderBackedFileSlot(Game game, List files, List noIntroSlots) + { + if (!_diskProvider.FolderExists(game.Path) || !files.Any(file => file.RelativePath.IsNullOrWhiteSpace())) + { + return null; + } + + var matchingSlots = _diskProvider.GetFiles(game.Path, true) + .Select(Path.GetFileName) + .Where(name => name.IsNotNullOrWhiteSpace()) + .Select(name => _noIntroComponentPlanner.FindSlotForFileName(noIntroSlots, name)) + .Where(slot => slot != null) + .GroupBy(slot => slot.Key) + .Select(group => group.First()) + .ToList(); + + return matchingSlots.Count == 1 ? matchingSlots[0] : null; + } + private static GameComponent FindOrStage(List existing, List toInsert, Game game, GameComponentType type, string key, string title, bool monitored) { var found = existing.FirstOrDefault(c => c.ComponentType == type && c.Key == key) ?? diff --git a/src/NzbDrone.Core/Games/Components/NoIntroGameComponentPlanner.cs b/src/NzbDrone.Core/Games/Components/NoIntroGameComponentPlanner.cs new file mode 100644 index 00000000..52254469 --- /dev/null +++ b/src/NzbDrone.Core/Games/Components/NoIntroGameComponentPlanner.cs @@ -0,0 +1,214 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Parser; +using NzbDrone.Core.RomCatalog; + +namespace NzbDrone.Core.Games.Components +{ + public class NoIntroGameComponentPlanner + { + private readonly INoIntroComponentClassifier _componentClassifier; + + public NoIntroGameComponentPlanner(INoIntroComponentClassifier componentClassifier) + { + _componentClassifier = componentClassifier; + } + + public List GetSlots(Game game, List entries) + { + var platformEntries = entries + .Where(entry => NoIntroCatalogDefaults.MatchesGamePlatform(game.Platform, entry.PlatformFamily)) + .ToList(); + var plan = _componentClassifier.BuildCatalogPlan(platformEntries); + var titleKeys = BuildTitleKeys(game); + var gamePlans = plan.Games.Where(x => titleKeys.Contains(CleanTitleKey(x.GameTitle))).ToList(); + + if (gamePlans.Count == 0) + { + return new List(); + } + + var entriesByCanonicalName = platformEntries + .GroupBy(entry => entry.CanonicalName) + .ToDictionary(group => group.Key, group => group.ToList()); + + return gamePlans.SelectMany(gamePlan => gamePlan.RegionLanguageComponents.Concat(gamePlan.DownloadPlayComponents)) + .Select(slot => ToSlot(slot, entriesByCanonicalName)) + .ToList(); + } + + private static HashSet BuildTitleKeys(Game game) + { + var titleKeys = new HashSet(); + + AddTitleKey(titleKeys, game.Title); + AddTitleKey(titleKeys, game.GameMetadata.Value.OriginalTitle); + + foreach (var alternativeTitle in game.GameMetadata.Value.AlternativeTitles) + { + AddTitleKey(titleKeys, alternativeTitle.Title); + } + + return titleKeys; + } + + private static void AddTitleKey(HashSet titleKeys, string title) + { + var titleKey = CleanTitleKey(title); + + if (titleKey.IsNullOrWhiteSpace()) + { + return; + } + + titleKeys.Add(titleKey); + + const string versionSuffix = "version"; + + if (titleKey.EndsWith(versionSuffix)) + { + titleKeys.Add(titleKey.Substring(0, titleKey.Length - versionSuffix.Length)); + } + else + { + titleKeys.Add(titleKey + versionSuffix); + } + } + + private static string CleanTitleKey(string title) + { + return title.CleanGameTitle(); + } + + public NoIntroGameComponentSlot FindSlotForFile(Game game, List entries, GameFile file) + { + return FindSlotForFile(GetSlots(game, entries), file); + } + + public NoIntroGameComponentSlot FindSlotForFileName(Game game, List entries, string fileName) + { + return FindSlotForFileName(GetSlots(game, entries), fileName); + } + + public NoIntroGameComponentSlot FindSlotForFile(List slots, GameFile file) + { + return FindSlotForFileName(slots, GetFileName(file.RelativePath)) ?? + FindSlotForFileName(slots, GetFileName(file.OriginalFilePath)); + } + + public NoIntroGameComponentSlot FindSlotForFileName(List slots, string fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) + { + return null; + } + + return slots.FirstOrDefault(slot => slot.FileNames.Contains(fileName)); + } + + private static string GetFileName(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return null; + } + + return Path.GetFileName(path.Replace('\\', '/')); + } + + private static NoIntroGameComponentSlot ToSlot(NoIntroCatalogComponentSlot slot, Dictionary> entriesByCanonicalName) + { + var componentType = MapComponentType(slot.ComponentType); + entriesByCanonicalName.TryGetValue(slot.CanonicalName, out var entries); + + return new NoIntroGameComponentSlot + { + ComponentType = componentType, + Key = $"nointro:{ComponentKeyPrefix(componentType)}:{Parser.Parser.ToUrlSlug(slot.CanonicalName, true)}", + Title = slot.SlotLabel, + FileNames = BuildFileNames(slot.CanonicalName, entries ?? new List()) + }; + } + + private static HashSet BuildFileNames(string canonicalName, List entries) + { + var fileNames = new HashSet + { + $"{canonicalName}.gb", + $"{canonicalName}.gbc", + $"{canonicalName}.gba", + $"{canonicalName}.nds", + $"{canonicalName}.3ds", + $"{canonicalName}.cia", + $"{canonicalName}.cci", + $"{canonicalName}.cxi", + $"{canonicalName}.n64", + $"{canonicalName}.z64", + $"{canonicalName}.v64", + $"{canonicalName}.nes", + $"{canonicalName}.sfc", + $"{canonicalName}.smc", + $"{canonicalName}.fds", + $"{canonicalName}.vb", + $"{canonicalName}.min", + $"{canonicalName}.iso", + $"{canonicalName}.cso", + $"{canonicalName}.pkg", + $"{canonicalName}.vpk", + $"{canonicalName}.zip" + }; + + foreach (var entry in entries) + { + AddFileName(fileNames, entry.CanonicalFileName); + AddFileName(fileNames, entry.NumberedCanonicalFileName); + } + + return fileNames; + } + + private static void AddFileName(HashSet fileNames, string fileName) + { + if (!string.IsNullOrWhiteSpace(fileName)) + { + fileNames.Add(fileName); + } + } + + private static GameComponentType MapComponentType(NoIntroRomComponentType componentType) + { + return componentType switch + { + NoIntroRomComponentType.RetailRom => GameComponentType.NoIntroRetailRom, + NoIntroRomComponentType.EReaderCards => GameComponentType.NoIntroMultiboot, + NoIntroRomComponentType.Multiboot => GameComponentType.NoIntroMultiboot, + NoIntroRomComponentType.Video => GameComponentType.NoIntroVideo, + NoIntroRomComponentType.Bios => GameComponentType.NoIntroBios, + _ => GameComponentType.NoIntroRomhackOrUnverified + }; + } + + private static string ComponentKeyPrefix(GameComponentType componentType) + { + return componentType switch + { + GameComponentType.NoIntroRetailRom => "retail", + GameComponentType.NoIntroMultiboot => "multiboot", + GameComponentType.NoIntroVideo => "video", + GameComponentType.NoIntroBios => "bios", + _ => "unverified" + }; + } + } + + public class NoIntroGameComponentSlot + { + public GameComponentType ComponentType { get; set; } + public string Key { get; set; } + public string Title { get; set; } + public HashSet FileNames { get; set; } + } +} diff --git a/src/NzbDrone.Core/Games/GamePlatform.cs b/src/NzbDrone.Core/Games/GamePlatform.cs index c7e16399..54b17c26 100644 --- a/src/NzbDrone.Core/Games/GamePlatform.cs +++ b/src/NzbDrone.Core/Games/GamePlatform.cs @@ -27,7 +27,25 @@ public enum PlatformFamily Atari = 6, Mobile = 7, Linux = 8, - Mac = 9 + Mac = 9, + NintendoSwitch = 10, + NintendoWiiU = 11, + NintendoWii = 12, + Nintendo3DS = 13, + NintendoDS = 14, + NintendoGBA = 15, + NintendoGB = 16, + NintendoGBC = 17, + NintendoNES = 18, + NintendoSNES = 19, + NintendoN64 = 20, + NintendoFDS = 21, + NintendoVirtualBoy = 22, + NintendoPokemonMini = 23, + NintendoDSi = 24, + SonyPS3 = 25, + SonyPSP = 26, + SonyPSVita = 27 } /// @@ -60,6 +78,47 @@ public static PlatformFamily MapPlatformFamily(int? igdbFamilyId) }; } + public static bool IsNintendoFamily(PlatformFamily platform) + { + return platform is PlatformFamily.Nintendo or + PlatformFamily.NintendoSwitch or + PlatformFamily.NintendoWiiU or + PlatformFamily.NintendoWii or + PlatformFamily.Nintendo3DS or + PlatformFamily.NintendoDSi or + PlatformFamily.NintendoDS or + PlatformFamily.NintendoGBA or + PlatformFamily.NintendoGB or + PlatformFamily.NintendoGBC or + PlatformFamily.NintendoNES or + PlatformFamily.NintendoSNES or + PlatformFamily.NintendoN64 or + PlatformFamily.NintendoFDS or + PlatformFamily.NintendoVirtualBoy or + PlatformFamily.NintendoPokemonMini; + } + + public static bool IsPlayStationFamily(PlatformFamily platform) + { + return platform is PlatformFamily.PlayStation or + PlatformFamily.SonyPS3 or + PlatformFamily.SonyPSP or + PlatformFamily.SonyPSVita; + } + + public static bool PlatformMatches(PlatformFamily wanted, PlatformFamily actual) + { + if (wanted == actual) + { + return true; + } + + return (IsNintendoFamily(wanted) && actual == PlatformFamily.Nintendo) || + (wanted == PlatformFamily.Nintendo && IsNintendoFamily(actual)) || + (IsPlayStationFamily(wanted) && actual == PlatformFamily.PlayStation) || + (wanted == PlatformFamily.PlayStation && IsPlayStationFamily(actual)); + } + /// /// Common IGDB Platform IDs for reference /// @@ -71,6 +130,7 @@ public static class CommonPlatforms public const int PS5 = 167; public const int PS4 = 48; public const int PS3 = 9; + public const int PSP = 38; public const int PSVita = 46; public const int XboxSeriesX = 169; public const int XboxOne = 49; diff --git a/src/NzbDrone.Core/Jobs/TaskManager.cs b/src/NzbDrone.Core/Jobs/TaskManager.cs index b6cb0924..0b07c426 100644 --- a/src/NzbDrone.Core/Jobs/TaskManager.cs +++ b/src/NzbDrone.Core/Jobs/TaskManager.cs @@ -16,6 +16,7 @@ using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Games.Commands; +using NzbDrone.Core.RomCatalog; using NzbDrone.Core.Update.Commands; namespace NzbDrone.Core.Jobs @@ -107,6 +108,12 @@ public void Handle(ApplicationStartedEvent message) TypeName = typeof(RefreshCollectionsCommand).FullName }, + new ScheduledTask + { + Interval = 24 * 60, + TypeName = typeof(NoIntroCatalogSyncCommand).FullName + }, + new ScheduledTask { Interval = GetBackupInterval(), diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index dd79edb4..656e70e4 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -1017,7 +1017,7 @@ "InstanceNameHelpText": "Instance name in tab and for Syslog app name", "InteractiveImport": "Interactive Import", "InteractiveImportLoadError": "Unable to load manual import items", - "InteractiveImportNoFilesFound": "No video files were found in the selected folder", + "InteractiveImportNoFilesFound": "No game files found", "InteractiveImportNoGame": "Game must be chosen for each selected file", "InteractiveImportNoImportMode": "An import mode must be selected", "InteractiveImportNoLanguage": "Language must be chosen for each selected file", @@ -1700,6 +1700,11 @@ "RenameFiles": "Rename Files", "RenameGames": "Rename Games", "RenameGamesHelpText": "{appName} will use the existing file name if renaming is disabled", + "RenameProfile": "Rename Profile", + "RenameProfileGamarr": "Gamarr", + "RenameProfileHelpText": "Choose how verified ROM filenames are evaluated. Normal Gamarr naming remains the default.", + "RenameProfileNoIntroCanonical": "No-Intro Canonical", + "RenameProfileNoIntroPreserveById": "No-Intro Preserve By-ID", "Renamed": "Renamed", "Reorder": "Reorder", "Repack": "Repack", diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index f82045cd..8b565666 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -409,7 +409,7 @@ private void CompletedScanning(Game game, List possibleExtraFiles) public string[] GetVideoFiles(string path, bool allDirectories = true) { - _logger.Debug("Scanning '{0}' for video files", path); + _logger.Debug("Scanning '{0}' for game files", path); var filesOnDisk = _diskProvider.GetFiles(path, allDirectories).ToList(); @@ -417,14 +417,14 @@ public string[] GetVideoFiles(string path, bool allDirectories = true) .ToList(); _logger.Trace("{0} files were found in {1}", filesOnDisk.Count, path); - _logger.Debug("{0} video files were found in {1}", mediaFileList.Count, path); + _logger.Debug("{0} game files were found in {1}", mediaFileList.Count, path); return mediaFileList.ToArray(); } public string[] GetNonVideoFiles(string path, bool allDirectories = true) { - _logger.Debug("Scanning '{0}' for non-video files", path); + _logger.Debug("Scanning '{0}' for non-game files", path); var filesOnDisk = _diskProvider.GetFiles(path, allDirectories).ToList(); @@ -432,7 +432,7 @@ public string[] GetNonVideoFiles(string path, bool allDirectories = true) .ToList(); _logger.Trace("{0} files were found in {1}", filesOnDisk.Count, path); - _logger.Debug("{0} non-video files were found in {1}", mediaFileList.Count, path); + _logger.Debug("{0} non-game files were found in {1}", mediaFileList.Count, path); return mediaFileList.ToArray(); } diff --git a/src/NzbDrone.Core/MediaFiles/DownloadedGameImportService.cs b/src/NzbDrone.Core/MediaFiles/DownloadedGameImportService.cs index 3955f008..63d7af35 100644 --- a/src/NzbDrone.Core/MediaFiles/DownloadedGameImportService.cs +++ b/src/NzbDrone.Core/MediaFiles/DownloadedGameImportService.cs @@ -331,7 +331,7 @@ private List ProcessFile(FileInfo fileInfo, ImportMode importMode, return new List { - new ImportResult(new ImportDecision(new LocalGame { Path = fileInfo.FullName }, new ImportRejection(ImportRejectionReason.InvalidFilePath, "Invalid video file, filename starts with '._'")), "Invalid video file, filename starts with '._'") + new ImportResult(new ImportDecision(new LocalGame { Path = fileInfo.FullName }, new ImportRejection(ImportRejectionReason.InvalidFilePath, "Invalid game file, filename starts with '._'")), "Invalid game file, filename starts with '._'") }; } @@ -364,8 +364,8 @@ private List ProcessFile(FileInfo fileInfo, ImportMode importMode, return new List { new ImportResult(new ImportDecision(new LocalGame { Path = fileInfo.FullName }, - new ImportRejection(ImportRejectionReason.UnsupportedExtension, $"Invalid video file, unsupported extension: '{extension}'")), - $"Invalid video file, unsupported extension: '{extension}'") + new ImportRejection(ImportRejectionReason.UnsupportedExtension, $"Invalid game file, unsupported extension: '{extension}'")), + $"Invalid game file, unsupported extension: '{extension}'") }; } diff --git a/src/NzbDrone.Core/MediaFiles/MediaFileExtensions.cs b/src/NzbDrone.Core/MediaFiles/MediaFileExtensions.cs index fcbd50ee..63339cbf 100644 --- a/src/NzbDrone.Core/MediaFiles/MediaFileExtensions.cs +++ b/src/NzbDrone.Core/MediaFiles/MediaFileExtensions.cs @@ -39,6 +39,43 @@ static MediaFileExtensions() // FitGirl and other repack installer data { ".dat", Quality.Repack }, + { ".gb", Quality.Unknown }, + { ".gbc", Quality.Unknown }, + { ".gba", Quality.Unknown }, + { ".nds", Quality.Unknown }, + { ".dsi", Quality.Unknown }, + { ".3ds", Quality.Unknown }, + { ".cia", Quality.Unknown }, + { ".nes", Quality.Unknown }, + { ".fds", Quality.Unknown }, + { ".sfc", Quality.Unknown }, + { ".smc", Quality.Unknown }, + { ".n64", Quality.Unknown }, + { ".z64", Quality.Unknown }, + { ".v64", Quality.Unknown }, + { ".xci", Quality.Unknown }, + { ".nsp", Quality.Unknown }, + { ".nsz", Quality.Unknown }, + { ".rvz", Quality.ISO }, + { ".wbfs", Quality.ISO }, + { ".gcz", Quality.ISO }, + { ".ciso", Quality.ISO }, + { ".chd", Quality.ISO }, + { ".sms", Quality.Unknown }, + { ".gg", Quality.Unknown }, + { ".sg", Quality.Unknown }, + { ".md", Quality.Unknown }, + { ".gen", Quality.Unknown }, + { ".32x", Quality.Unknown }, + { ".pce", Quality.Unknown }, + { ".a26", Quality.Unknown }, + { ".a52", Quality.Unknown }, + { ".a78", Quality.Unknown }, + { ".lnx", Quality.Unknown }, + { ".ngp", Quality.Unknown }, + { ".ngc", Quality.Unknown }, + { ".ws", Quality.Unknown }, + { ".wsc", Quality.Unknown }, }; } diff --git a/src/NzbDrone.Core/MetadataSource/RAWG/RawgProxy.cs b/src/NzbDrone.Core/MetadataSource/RAWG/RawgProxy.cs index fbe590fc..dcb5839f 100644 --- a/src/NzbDrone.Core/MetadataSource/RAWG/RawgProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/RAWG/RawgProxy.cs @@ -538,7 +538,47 @@ private PlatformFamily MapPlatformFamily(string slug) return PlatformFamily.Xbox; } - if (slug.Contains("nintendo") || slug.Contains("switch") || slug.Contains("wii") || slug.Contains("3ds")) + if (slug.Contains("switch")) + { + return PlatformFamily.NintendoSwitch; + } + + if (slug.Contains("wii-u") || slug.Contains("wiiu")) + { + return PlatformFamily.NintendoWiiU; + } + + if (slug.Contains("wii")) + { + return PlatformFamily.NintendoWii; + } + + if (slug.Contains("3ds")) + { + return PlatformFamily.Nintendo3DS; + } + + if (slug.Contains("nintendo-ds") || slug.Contains("nds")) + { + return PlatformFamily.NintendoDS; + } + + if (slug.Contains("game-boy-advance") || slug.Contains("gba")) + { + return PlatformFamily.NintendoGBA; + } + + if (slug.Contains("game-boy-color") || slug.Contains("gbc")) + { + return PlatformFamily.NintendoGBC; + } + + if (slug.Contains("game-boy") || slug.Contains("gb")) + { + return PlatformFamily.NintendoGB; + } + + if (slug.Contains("nintendo")) { return PlatformFamily.Nintendo; } diff --git a/src/NzbDrone.Core/Organizer/FileNameBuilder.cs b/src/NzbDrone.Core/Organizer/FileNameBuilder.cs index 4ff2534a..27d4dff6 100644 --- a/src/NzbDrone.Core/Organizer/FileNameBuilder.cs +++ b/src/NzbDrone.Core/Organizer/FileNameBuilder.cs @@ -12,9 +12,11 @@ using NzbDrone.Core.CustomFormats; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Games; +using NzbDrone.Core.Games.Components; using NzbDrone.Core.Games.Translations; using NzbDrone.Core.Parser; using NzbDrone.Core.Qualities; +using NzbDrone.Core.RomCatalog; namespace NzbDrone.Core.Organizer { @@ -31,6 +33,8 @@ public class FileNameBuilder : IBuildFileNames private readonly IQualityDefinitionService _qualityDefinitionService; private readonly IGameTranslationService _gameTranslationService; private readonly ICustomFormatCalculationService _formatCalculator; + private readonly IGameComponentRepository _componentRepository; + private readonly INoIntroCatalogEntryRepository _noIntroEntryRepository; private readonly Logger _logger; private static readonly Regex TitleRegex = new Regex(@"(?\{(?[-{ ._\[(]*)(?:edition-))?\{(?[-{ ._\[(]*)(?(?:[a-z0-9]+)(?:(?[- ._]+)(?:[a-z0-9]+))?)(?::(?[ ,a-z0-9|+-]+(?[-} ._)\]]*)\}", @@ -86,12 +90,16 @@ public FileNameBuilder(INamingConfigService namingConfigService, IQualityDefinitionService qualityDefinitionService, IGameTranslationService gameTranslationService, ICustomFormatCalculationService formatCalculator, + IGameComponentRepository componentRepository, + INoIntroCatalogEntryRepository noIntroEntryRepository, Logger logger) { _namingConfigService = namingConfigService; _qualityDefinitionService = qualityDefinitionService; _gameTranslationService = gameTranslationService; _formatCalculator = formatCalculator; + _componentRepository = componentRepository; + _noIntroEntryRepository = noIntroEntryRepository; _logger = logger; } @@ -107,6 +115,13 @@ public string BuildFileName(Game game, GameFile gameFile, NamingConfig namingCon return GetOriginalTitle(gameFile, false); } + var noIntroFileName = GetNoIntroFileName(gameFile, namingConfig.RenameProfile); + + if (noIntroFileName.IsNotNullOrWhiteSpace()) + { + return noIntroFileName; + } + if (namingConfig.StandardGameFormat.IsNullOrWhiteSpace()) { throw new NamingFormatException("Standard game format cannot be empty"); @@ -148,6 +163,140 @@ public string BuildFileName(Game game, GameFile gameFile, NamingConfig namingCon return Path.Combine(components.ToArray()); } + private string GetNoIntroFileName(GameFile gameFile, RenameProfile renameProfile) + { + if (gameFile == null) + { + return null; + } + + var actualFileName = GetActualFileName(gameFile); + var catalogEntry = ResolveNoIntroCatalogEntry(gameFile, actualFileName); + + if (catalogEntry == null) + { + return null; + } + + if (renameProfile == RenameProfile.Gamarr) + { + return Path.GetFileNameWithoutExtension(actualFileName); + } + + var expectedFileName = NoIntroRenameProfileEvaluator.GetExpectedFileName(catalogEntry, actualFileName, renameProfile); + return Path.GetFileNameWithoutExtension(expectedFileName); + } + + private NoIntroCatalogEntry ResolveNoIntroCatalogEntry(GameFile gameFile, string actualFileName) + { + if (actualFileName.IsNullOrWhiteSpace()) + { + return null; + } + + if (gameFile.ComponentId > 0) + { + var component = _componentRepository.Get(gameFile.ComponentId); + + if (component != null) + { + return FindMatchingCatalogEntry(component, actualFileName); + } + } + + if (gameFile.GameId <= 0) + { + return null; + } + + return _componentRepository.GetByGame(gameFile.GameId) + .Where(IsNoIntroComponent) + .Select(component => FindMatchingCatalogEntry(component, actualFileName)) + .FirstOrDefault(entry => entry != null); + } + + private NoIntroCatalogEntry FindMatchingCatalogEntry(GameComponent component, string actualFileName) + { + if (component == null) + { + return null; + } + + return _noIntroEntryRepository.All() + .Where(entry => IsComponentCatalogMatch(component, entry) && EntryMatchesActualFileName(entry, actualFileName)) + .OrderBy(entry => entry.SystemKey) + .ThenBy(entry => entry.CanonicalName) + .FirstOrDefault(); + } + + private static bool EntryMatchesActualFileName(NoIntroCatalogEntry entry, string actualFileName) + { + if (entry == null || actualFileName.IsNullOrWhiteSpace()) + { + return false; + } + + return string.Equals(entry.CanonicalFileName, actualFileName, StringComparison.Ordinal) || + string.Equals(entry.NumberedCanonicalFileName, actualFileName, StringComparison.Ordinal); + } + + private static bool IsNoIntroComponent(GameComponent component) + { + return component.ComponentType is GameComponentType.NoIntroRetailRom or + GameComponentType.NoIntroMultiboot or + GameComponentType.NoIntroVideo or + GameComponentType.NoIntroBios or + GameComponentType.NoIntroRomhackOrUnverified; + } + + private static bool IsComponentCatalogMatch(string componentTitle, NoIntroCatalogEntry entry) + { + if (entry == null || entry.CanonicalName.IsNullOrWhiteSpace()) + { + return false; + } + + if (!entry.ParentCanonicalName.IsNullOrWhiteSpace() && entry.ParentCanonicalName == componentTitle) + { + return true; + } + + return entry.CanonicalName == componentTitle || entry.CanonicalName.StartsWith($"{componentTitle} (", StringComparison.Ordinal); + } + + private static bool IsComponentCatalogMatch(GameComponent component, NoIntroCatalogEntry entry) + { + if (component == null || entry == null || entry.CanonicalName.IsNullOrWhiteSpace()) + { + return false; + } + + if (IsNoIntroComponent(component)) + { + if (component.Key?.EndsWith($":{Parser.Parser.ToUrlSlug(entry.CanonicalName, true)}", StringComparison.Ordinal) == true) + { + return true; + } + } + + return IsComponentCatalogMatch(component.Title, entry); + } + + private static string GetActualFileName(GameFile gameFile) + { + if (gameFile.OriginalFilePath.IsNotNullOrWhiteSpace()) + { + return Path.GetFileName(gameFile.OriginalFilePath); + } + + if (gameFile.RelativePath.IsNotNullOrWhiteSpace()) + { + return Path.GetFileName(gameFile.RelativePath); + } + + return Path.GetFileName(gameFile.Path); + } + public string BuildFilePath(Game game, string fileName, string extension) { Ensure.That(extension, () => extension).IsNotNullOrWhiteSpace(); diff --git a/src/NzbDrone.Core/Organizer/NamingConfig.cs b/src/NzbDrone.Core/Organizer/NamingConfig.cs index e124bf94..0352490a 100644 --- a/src/NzbDrone.Core/Organizer/NamingConfig.cs +++ b/src/NzbDrone.Core/Organizer/NamingConfig.cs @@ -2,11 +2,19 @@ namespace NzbDrone.Core.Organizer { + public enum RenameProfile + { + Gamarr = 0, + NoIntroPreserveById = 1, + NoIntroCanonical = 2 + } + public class NamingConfig : ModelBase { public static NamingConfig Default => new NamingConfig { RenameGames = false, + RenameProfile = RenameProfile.Gamarr, ReplaceIllegalCharacters = true, ColonReplacementFormat = ColonReplacementFormat.Smart, GameFolderFormat = "{Game Title} ({Release Year})", @@ -14,6 +22,7 @@ public class NamingConfig : ModelBase }; public bool RenameGames { get; set; } + public RenameProfile RenameProfile { get; set; } public bool ReplaceIllegalCharacters { get; set; } public ColonReplacementFormat ColonReplacementFormat { get; set; } public string StandardGameFormat { get; set; } diff --git a/src/NzbDrone.Core/Parser/LanguageParser.cs b/src/NzbDrone.Core/Parser/LanguageParser.cs index 05edfae4..818d7c6d 100644 --- a/src/NzbDrone.Core/Parser/LanguageParser.cs +++ b/src/NzbDrone.Core/Parser/LanguageParser.cs @@ -58,6 +58,7 @@ public static class LanguageParser private static readonly Regex GermanDualLanguageRegex = new (@"(?[a-z]{2,3}(?:,[a-z]{2,3})+)[\)\]]", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex SubtitleLanguageRegex = new Regex(".+?([-_. ](?forced|foreign|default|cc|psdh|sdh))*[-_. ](?[a-z]{2,3})([-_. ](?forced|foreign|default|cc|psdh|sdh))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -455,6 +456,19 @@ public static List ParseLanguages(string title) languages.Add(Language.Russian); } + foreach (Match match in NoIntroLanguageListRegex.Matches(title)) + { + foreach (var code in match.Groups["codes"].Value.Split(',')) + { + var language = IsoLanguages.Find(code)?.Language; + + if (language != null) + { + languages.Add(language); + } + } + } + if (!languages.Any()) { languages.Add(Language.Unknown); diff --git a/src/NzbDrone.Core/Parser/PlatformParser.cs b/src/NzbDrone.Core/Parser/PlatformParser.cs index e128c30c..adc3ca42 100644 --- a/src/NzbDrone.Core/Parser/PlatformParser.cs +++ b/src/NzbDrone.Core/Parser/PlatformParser.cs @@ -68,6 +68,18 @@ public static class PlatformParser @"\b(?:NDS|Nintendo\s*DS)\b|\[(?:NDS|Nintendo\s*DS)\]", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex NintendoGBARegex = new Regex( + @"\b(?:GBA|Game\s*Boy\s*Advance|Nintendo\s*Game\s*Boy\s*Advance)\b|\[(?:GBA|Game\s*Boy\s*Advance|Nintendo\s*Game\s*Boy\s*Advance)\]", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex NintendoGBCRegex = new Regex( + @"\b(?:GBC|Game\s*Boy\s*Color|Nintendo\s*Game\s*Boy\s*Color)\b|\[(?:GBC|Game\s*Boy\s*Color|Nintendo\s*Game\s*Boy\s*Color)\]", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex NintendoGBRegex = new Regex( + @"\b(?:GB|Game\s*Boy|Nintendo\s*Game\s*Boy)\b|\[(?:GB|Game\s*Boy|Nintendo\s*Game\s*Boy)\]", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + // PC/Mac/Linux platforms private static readonly Regex MacRegex = new Regex( @"\b(?:MAC|macOS|OSX|Mac\s*OS)\b|\[(?:MAC|macOS|OSX|Mac\s*OS)\]", @@ -105,19 +117,19 @@ public static PlatformFamily ParsePlatform(string title) if (PlayStation3Regex.IsMatch(title)) { Logger.Trace("Detected PS3 platform in title"); - return PlatformFamily.PlayStation; + return PlatformFamily.SonyPS3; } if (PSVitaRegex.IsMatch(title)) { Logger.Trace("Detected PS Vita platform in title"); - return PlatformFamily.PlayStation; + return PlatformFamily.SonyPSVita; } if (PSPRegex.IsMatch(title)) { Logger.Trace("Detected PSP platform in title"); - return PlatformFamily.PlayStation; + return PlatformFamily.SonyPSP; } // Check Xbox platforms (most specific first) @@ -149,31 +161,49 @@ public static PlatformFamily ParsePlatform(string title) if (SwitchRegex.IsMatch(title)) { Logger.Trace("Detected Nintendo Switch platform in title"); - return PlatformFamily.Nintendo; + return PlatformFamily.NintendoSwitch; } if (WiiURegex.IsMatch(title)) { Logger.Trace("Detected Wii U platform in title"); - return PlatformFamily.Nintendo; + return PlatformFamily.NintendoWiiU; } if (WiiRegex.IsMatch(title)) { Logger.Trace("Detected Wii platform in title"); - return PlatformFamily.Nintendo; + return PlatformFamily.NintendoWii; } if (Nintendo3DSRegex.IsMatch(title)) { Logger.Trace("Detected Nintendo 3DS platform in title"); - return PlatformFamily.Nintendo; + return PlatformFamily.Nintendo3DS; } if (NintendoDSRegex.IsMatch(title)) { Logger.Trace("Detected Nintendo DS platform in title"); - return PlatformFamily.Nintendo; + return PlatformFamily.NintendoDS; + } + + if (NintendoGBARegex.IsMatch(title)) + { + Logger.Trace("Detected Game Boy Advance platform in title"); + return PlatformFamily.NintendoGBA; + } + + if (NintendoGBCRegex.IsMatch(title)) + { + Logger.Trace("Detected Game Boy Color platform in title"); + return PlatformFamily.NintendoGBC; + } + + if (NintendoGBRegex.IsMatch(title)) + { + Logger.Trace("Detected Game Boy platform in title"); + return PlatformFamily.NintendoGB; } // Check Mac/Linux @@ -278,6 +308,21 @@ public static string ParsePlatformString(string title) return "NDS"; } + if (NintendoGBARegex.IsMatch(title)) + { + return "GBA"; + } + + if (NintendoGBCRegex.IsMatch(title)) + { + return "GBC"; + } + + if (NintendoGBRegex.IsMatch(title)) + { + return "GB"; + } + // Mac/Linux if (MacRegex.IsMatch(title)) { diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogDefaults.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogDefaults.cs new file mode 100644 index 00000000..7ae11d5f --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogDefaults.cs @@ -0,0 +1,154 @@ +using System.Collections.Generic; +using NzbDrone.Core.Games; + +namespace NzbDrone.Core.RomCatalog +{ + public static class NoIntroCatalogDefaults + { + public static readonly IReadOnlyList Sources = new List + { + new NoIntroCatalogSourceSeed("No-Intro Nintendo Game Boy", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Game%20Boy.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo Game Boy Color", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Game%20Boy%20Color.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo Game Boy Advance", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Game%20Boy%20Advance.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo DS", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20DS.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo 3DS", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%203DS.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo 3DS Digital", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%203DS%20%28Digital%29.dat"), + new NoIntroCatalogSourceSeed("No-Intro New Nintendo 3DS", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20New%20Nintendo%203DS.dat"), + new NoIntroCatalogSourceSeed("No-Intro New Nintendo 3DS Digital", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20New%20Nintendo%203DS%20%28Digital%29.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo DS Download Play", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20DS%20%28Download%20Play%29.dat"), + + // DAT-o-MATIC-only niche systems (GBA Multiboot/e-Reader/Play-Yan/Video, + // DSvision) are intentionally not seeded while the datomatic:// scrape + // path is broken — see FetchDatOMaticNumbered. Restore once it works. + + new NoIntroCatalogSourceSeed("No-Intro Nintendo Entertainment System", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20Entertainment%20System.dat"), + new NoIntroCatalogSourceSeed("No-Intro Super Nintendo Entertainment System", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Super%20Nintendo%20Entertainment%20System.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo 64", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%2064.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo 64DD", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%2064DD.dat"), + new NoIntroCatalogSourceSeed("No-Intro Family Computer Disk System", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Family%20Computer%20Disk%20System.dat"), + new NoIntroCatalogSourceSeed("No-Intro Virtual Boy", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Virtual%20Boy.dat"), + new NoIntroCatalogSourceSeed("No-Intro Pokemon Mini", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Pokemon%20Mini.dat"), + new NoIntroCatalogSourceSeed("No-Intro Satellaview", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Satellaview.dat"), + new NoIntroCatalogSourceSeed("No-Intro Sufami Turbo", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Sufami%20Turbo.dat"), + new NoIntroCatalogSourceSeed("No-Intro Nintendo DSi", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Nintendo%20DSi.dat"), + new NoIntroCatalogSourceSeed("No-Intro Wii Digital", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Wii%20%28Digital%29.dat"), + new NoIntroCatalogSourceSeed("No-Intro Wii U Digital", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Nintendo%20-%20Wii%20U%20%28Digital%29.dat"), + new NoIntroCatalogSourceSeed("No-Intro Sony PlayStation Portable", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Portable.dat"), + new NoIntroCatalogSourceSeed("No-Intro Sony PlayStation Portable PSN", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Portable%20%28PSN%29.dat"), + new NoIntroCatalogSourceSeed("No-Intro Sony PlayStation Portable PSX2PSP", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Portable%20%28PSX2PSP%29.dat"), + new NoIntroCatalogSourceSeed("No-Intro Sony PlayStation Vita", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Vita.dat"), + new NoIntroCatalogSourceSeed("No-Intro Sony PlayStation Vita PSN", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%20Vita%20%28PSN%29.dat"), + new NoIntroCatalogSourceSeed("No-Intro Sony PlayStation 3 PSN", "https://raw.githubusercontent.com/libretro/libretro-database/master/metadat/no-intro/Sony%20-%20PlayStation%203%20%28PSN%29.dat") + }; + + public static PlatformFamily MapPlatformFamily(string systemKey) + { + return systemKey switch + { + "nintendo---game-boy" => PlatformFamily.NintendoGB, + "nintendo---game-boy-color" => PlatformFamily.NintendoGBC, + "nintendo---game-boy-advance" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance-multiboot" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance--multiboot" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance-e-reader" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance--e-reader" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance-play-yan" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance--play-yan" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance-video" => PlatformFamily.NintendoGBA, + "nintendo---game-boy-advance--video" => PlatformFamily.NintendoGBA, + "nintendo---nintendo-ds" => PlatformFamily.NintendoDS, + "nintendo---nintendo-dsi" => PlatformFamily.NintendoDSi, + "nintendo---nintendo-ds-download-play" => PlatformFamily.NintendoDS, + "nintendo---nintendo-ds--download-play" => PlatformFamily.NintendoDS, + "nintendo---nintendo-ds-dsvision-sd-cards" => PlatformFamily.NintendoDS, + "nintendo---nintendo-ds--dsvision-sd-cards" => PlatformFamily.NintendoDS, + "nintendo---nintendo-3ds" => PlatformFamily.Nintendo3DS, + "nintendo---nintendo-3ds-digital" => PlatformFamily.Nintendo3DS, + "nintendo---nintendo-3ds--digital" => PlatformFamily.Nintendo3DS, + "nintendo---new-nintendo-3ds" => PlatformFamily.Nintendo3DS, + "nintendo---new-nintendo-3ds-digital" => PlatformFamily.Nintendo3DS, + "nintendo---new-nintendo-3ds--digital" => PlatformFamily.Nintendo3DS, + "nintendo---nintendo-entertainment-system" => PlatformFamily.NintendoNES, + "nintendo---super-nintendo-entertainment-system" => PlatformFamily.NintendoSNES, + "nintendo---nintendo-64" => PlatformFamily.NintendoN64, + "nintendo---nintendo-64dd" => PlatformFamily.NintendoN64, + "nintendo---family-computer-disk-system" => PlatformFamily.NintendoFDS, + "nintendo---virtual-boy" => PlatformFamily.NintendoVirtualBoy, + "nintendo---pokemon-mini" => PlatformFamily.NintendoPokemonMini, + "nintendo---satellaview" => PlatformFamily.NintendoSNES, + "nintendo---sufami-turbo" => PlatformFamily.NintendoSNES, + "nintendo---wii--digital" => PlatformFamily.NintendoWii, + "nintendo---wii-u--digital" => PlatformFamily.NintendoWiiU, + "sony---playstation-3--psn" => PlatformFamily.SonyPS3, + "sony---playstation-portable" => PlatformFamily.SonyPSP, + "sony---playstation-portable--psn" => PlatformFamily.SonyPSP, + "sony---playstation-portable--psx2psp" => PlatformFamily.SonyPSP, + "sony---playstation-vita" => PlatformFamily.SonyPSVita, + "sony---playstation-vita--psn" => PlatformFamily.SonyPSVita, + _ => PlatformFamily.Unknown + }; + } + + public static bool MatchesGamePlatform(PlatformFamily gamePlatform, PlatformFamily catalogPlatform) + { + if (gamePlatform == catalogPlatform) + { + return true; + } + + return (gamePlatform == PlatformFamily.Nintendo && GamePlatform.IsNintendoFamily(catalogPlatform)) || + (gamePlatform == PlatformFamily.PlayStation && GamePlatform.IsPlayStationFamily(catalogPlatform)); + } + + public static IReadOnlyList GetRelevantPlatformFamilies(PlatformFamily gamePlatform) + { + if (gamePlatform == PlatformFamily.Nintendo) + { + return new[] + { + PlatformFamily.Nintendo, + PlatformFamily.NintendoSwitch, + PlatformFamily.NintendoWiiU, + PlatformFamily.NintendoWii, + PlatformFamily.Nintendo3DS, + PlatformFamily.NintendoDSi, + PlatformFamily.NintendoDS, + PlatformFamily.NintendoGBA, + PlatformFamily.NintendoGB, + PlatformFamily.NintendoGBC, + PlatformFamily.NintendoNES, + PlatformFamily.NintendoSNES, + PlatformFamily.NintendoN64, + PlatformFamily.NintendoFDS, + PlatformFamily.NintendoVirtualBoy, + PlatformFamily.NintendoPokemonMini + }; + } + + if (gamePlatform == PlatformFamily.PlayStation) + { + return new[] + { + PlatformFamily.PlayStation, + PlatformFamily.SonyPS3, + PlatformFamily.SonyPSP, + PlatformFamily.SonyPSVita + }; + } + + return new[] { gamePlatform }; + } + } + + public class NoIntroCatalogSourceSeed + { + public NoIntroCatalogSourceSeed(string name, string sourceUrl) + { + Name = name; + SourceUrl = sourceUrl; + } + + public string Name { get; } + public string SourceUrl { get; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogDocumentClient.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogDocumentClient.cs new file mode 100644 index 00000000..dfac969c --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogDocumentClient.cs @@ -0,0 +1,124 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text.RegularExpressions; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroCatalogDocumentClient + { + string Fetch(string sourceUrl); + string FetchDatOMaticNumbered(int systemId); + string FetchAdvanscene(string sourceUrl); + } + + public class NoIntroCatalogDocumentClient : INoIntroCatalogDocumentClient + { + private const string DatOMaticSourceUrlPrefix = "datomatic://system/"; + private static readonly Regex PrepareFieldRegex = new Regex(@"name=""(?dat_dl_[^""]+)""\s+value=""Prepare""", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex DownloadTokenRegex = new Regex("[0-9a-f]{32})\" value=\"Download!!\"", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private readonly IHttpClient _httpClient; + + public NoIntroCatalogDocumentClient(IHttpClient httpClient) + { + _httpClient = httpClient; + } + + public string Fetch(string sourceUrl) + { + if (sourceUrl.StartsWith(DatOMaticSourceUrlPrefix, StringComparison.OrdinalIgnoreCase)) + { + if (!int.TryParse(sourceUrl.AsSpan(DatOMaticSourceUrlPrefix.Length), out var systemId)) + { + throw new InvalidOperationException($"Invalid DAT-o-MATIC source url: {sourceUrl}"); + } + + return FetchDatOMaticNumbered(systemId); + } + + return _httpClient.Get(new HttpRequest(sourceUrl)).Content; + } + + public string FetchDatOMaticNumbered(int systemId) + { + var prepareUrl = $"https://datomatic.no-intro.org/index.php?page=download&op=dat&s={systemId}"; + var preparePage = _httpClient.Get(new HttpRequest(prepareUrl)); + var prepareField = PrepareFieldRegex.Match(preparePage.Content).Groups["field"].Value; + + if (string.IsNullOrWhiteSpace(prepareField)) + { + throw new InvalidOperationException("DAT-o-MATIC did not expose a numbered DAT prepare field"); + } + + var prepareRequest = new HttpRequestBuilder(prepareUrl) + .Post() + .AddFormParameter("system_selection", systemId) + .AddFormParameter("sys_list_order", 1) + .AddFormParameter("format", 0) + .AddFormParameter("naming", 0) + .AddFormParameter("numbered", 1) + .AddFormParameter("inc_bios", 1) + .AddFormParameter("release_1", 1) + .AddFormParameter("release_2", 1) + .AddFormParameter("license_1", 1) + .AddFormParameter("license_2", 1) + .AddFormParameter("license_0", 1) + .AddFormParameter("lifespan_1", 1) + .AddFormParameter("inc_xroms", 1) + .AddFormParameter("inc_zroms", 1) + .AddFormParameter("storage_1", 1) + .AddFormParameter("storage_2", 1) + .AddFormParameter("inc_nodump", 0) + .AddFormParameter("inc_mia", 1) + .AddFormParameter(prepareField, "Prepare") + .Build(); + + var prepareResponse = _httpClient.Post(prepareRequest); + var token = DownloadTokenRegex.Matches(prepareResponse.Content) + .Cast() + .Select(x => x.Groups["token"].Value) + .LastOrDefault(); + + if (string.IsNullOrWhiteSpace(token)) + { + throw new InvalidOperationException("DAT-o-MATIC did not return a numbered DAT download token"); + } + + var downloadRequest = new HttpRequestBuilder(prepareResponse.Request.Url.FullUri) + .Post() + .AddFormParameter(token, "Download!!") + .Build(); + var downloadResponse = _httpClient.Post(downloadRequest); + + return ExtractDat(downloadResponse.ResponseData); + } + + public string FetchAdvanscene(string sourceUrl) + { + return ExtractDat(_httpClient.Get(new HttpRequest(sourceUrl)).ResponseData); + } + + private static string ExtractDat(byte[] data) + { + if (data.Length >= 2 && data[0] == 'P' && data[1] == 'K') + { + using var stream = new MemoryStream(data); + using var archive = new ZipArchive(stream, ZipArchiveMode.Read); + var entry = archive.Entries.FirstOrDefault(x => x.Name.EndsWith(".dat", StringComparison.OrdinalIgnoreCase)) ?? archive.Entries.FirstOrDefault(); + + if (entry == null) + { + throw new InvalidOperationException("Catalog archive was empty"); + } + + using var entryStream = entry.Open(); + using var reader = new StreamReader(entryStream); + return reader.ReadToEnd(); + } + + return System.Text.Encoding.UTF8.GetString(data); + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogEntry.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogEntry.cs new file mode 100644 index 00000000..b1ce521a --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogEntry.cs @@ -0,0 +1,17 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Games; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroCatalogEntry : ModelBase + { + public int CatalogSourceId { get; set; } + public string SystemKey { get; set; } + public string CanonicalName { get; set; } + public string ParentCanonicalName { get; set; } + public string CanonicalFileName { get; set; } + public string ReleaseNumber { get; set; } + public string NumberedCanonicalFileName { get; set; } + public PlatformFamily PlatformFamily { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogEntryRepository.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogEntryRepository.cs new file mode 100644 index 00000000..a4a5051b --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogEntryRepository.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using System.Linq; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Games; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroCatalogEntryRepository : IBasicRepository + { + List GetBySourceId(int catalogSourceId); + List GetByPlatformFamily(PlatformFamily platformFamily); + void DeleteBySourceId(int catalogSourceId); + } + + public class NoIntroCatalogEntryRepository : BasicRepository, INoIntroCatalogEntryRepository + { + public NoIntroCatalogEntryRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + + public List GetBySourceId(int catalogSourceId) + { + return Query(x => x.CatalogSourceId == catalogSourceId); + } + + public List GetByPlatformFamily(PlatformFamily platformFamily) + { + var relevantFamilies = NoIntroCatalogDefaults.GetRelevantPlatformFamilies(platformFamily).ToHashSet(); + return Query(x => relevantFamilies.Contains(x.PlatformFamily)); + } + + public void DeleteBySourceId(int catalogSourceId) + { + Delete(x => x.CatalogSourceId == catalogSourceId); + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogHash.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogHash.cs new file mode 100644 index 00000000..0c556df2 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogHash.cs @@ -0,0 +1,13 @@ +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroCatalogHash : ModelBase + { + public int CatalogEntryId { get; set; } + public string HashType { get; set; } + public string HashValue { get; set; } + public bool IsPrimary { get; set; } + public bool IsBadDump { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogHashRepository.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogHashRepository.cs new file mode 100644 index 00000000..228298ca --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogHashRepository.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroCatalogHashRepository : IBasicRepository + { + List GetByEntryIds(List entryIds); + void DeleteByEntryIds(List entryIds); + } + + public class NoIntroCatalogHashRepository : BasicRepository, INoIntroCatalogHashRepository + { + public NoIntroCatalogHashRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + + public List GetByEntryIds(List entryIds) + { + return entryIds.Count == 0 ? new List() : Query(x => entryIds.Contains(x.CatalogEntryId)); + } + + public void DeleteByEntryIds(List entryIds) + { + if (entryIds.Count == 0) + { + return; + } + + Delete(x => entryIds.Contains(x.CatalogEntryId)); + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogPlan.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogPlan.cs new file mode 100644 index 00000000..82d3af6e --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogPlan.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroCatalogPlan + { + public List Games { get; set; } = new List(); + public List StandaloneGames { get; set; } = new List(); + } + + public class NoIntroCatalogGamePlan + { + public string SystemKey { get; set; } + public string GameTitle { get; set; } + public List RegionLanguageComponents { get; set; } = new List(); + public List DownloadPlayComponents { get; set; } = new List(); + } + + public class NoIntroCatalogStandalonePlan + { + public string Title { get; set; } + public NoIntroRomComponentType ComponentType { get; set; } + } + + public class NoIntroCatalogComponentSlot + { + public string SlotLabel { get; set; } + public string CanonicalName { get; set; } + public NoIntroRomComponentType ComponentType { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSnapshot.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSnapshot.cs new file mode 100644 index 00000000..e5c36616 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSnapshot.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroCatalogSnapshot + { + public string CatalogVersion { get; set; } + public string SystemKey { get; set; } + public List Entries { get; set; } = new List(); + } + + public class NoIntroCatalogSnapshotEntry + { + public string CanonicalName { get; set; } + public string ParentCanonicalName { get; set; } + public string CanonicalFileName { get; set; } + public string ReleaseNumber { get; set; } + public string NumberedCanonicalFileName { get; set; } + public List Hashes { get; set; } = new List(); + } + + public class NoIntroCatalogSnapshotHash + { + public string HashType { get; set; } + public string HashValue { get; set; } + public bool IsPrimary { get; set; } + public bool IsBadDump { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSnapshotParser.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSnapshotParser.cs new file mode 100644 index 00000000..d133d2d6 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSnapshotParser.cs @@ -0,0 +1,190 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; +using System.Xml.Linq; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroCatalogSnapshotParser + { + NoIntroCatalogSnapshot Parse(string content); + } + + public class NoIntroCatalogSnapshotParser : INoIntroCatalogSnapshotParser + { + private static readonly Regex ClrMameHeaderRegex = new Regex(@"clrmamepro\s*\((?.*?)\)\s*game\s*\(", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Singleline); + private static readonly Regex ClrMameGameRegex = new Regex(@"game\s*\((?.*?)(?=\n\s*game\s*\(|\z)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Singleline); + private static readonly Regex ClrMameRomRegex = new Regex(@"rom\s*\((?[^\r\n]*)\)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex ClrMameFieldRegex = new Regex("(?[a-z0-9_]+)\\s+(?:(?:\"(?[^\"]*)\")|(?[^\\s\\)]+))", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex NumberedNameRegex = new Regex(@"^(?(?:xB|x|z)\d{3,4}|\d{4})\s+-\s+(?.+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + public NoIntroCatalogSnapshot Parse(string content) + { + if (!content.TrimStart().StartsWith("<", StringComparison.Ordinal)) + { + return ParseClrMamePro(content); + } + + var document = XDocument.Parse(content); + var root = document.Element("datafile") ?? throw new InvalidOperationException("Unsupported No-Intro catalog format"); + var header = root.Element("header"); + + var systemName = header?.Element("name")?.Value ?? "unknown"; + var version = header?.Element("version")?.Value; + + var snapshot = new NoIntroCatalogSnapshot + { + CatalogVersion = version, + SystemKey = NormalizeSystemKey(systemName) + }; + + foreach (var game in root.Elements("game")) + { + var canonicalName = game.Attribute("name")?.Value; + + if (string.IsNullOrWhiteSpace(canonicalName)) + { + continue; + } + + var entry = CreateEntry(canonicalName, + game.Attribute("cloneof")?.Value, + game.Elements("rom").FirstOrDefault()?.Attribute("name")?.Value ?? canonicalName, + game.Attribute("id")?.Value); + + foreach (var rom in game.Elements("rom")) + { + AddHash(entry, "crc32", rom.Attribute("crc")?.Value, true, IsBadDump(rom)); + AddHash(entry, "md5", rom.Attribute("md5")?.Value, false, IsBadDump(rom)); + AddHash(entry, "sha1", rom.Attribute("sha1")?.Value, false, IsBadDump(rom)); + } + + snapshot.Entries.Add(entry); + } + + return snapshot; + } + + private static NoIntroCatalogSnapshot ParseClrMamePro(string content) + { + var header = ClrMameHeaderRegex.Match(content); + var headerFields = header.Success ? ParseFields(header.Groups["body"].Value) : new System.Collections.Generic.Dictionary(); + var systemName = headerFields.TryGetValue("name", out var name) ? name : "unknown"; + var version = headerFields.TryGetValue("version", out var headerVersion) ? headerVersion : null; + + var snapshot = new NoIntroCatalogSnapshot + { + CatalogVersion = version, + SystemKey = NormalizeSystemKey(systemName) + }; + + foreach (Match gameMatch in ClrMameGameRegex.Matches(content)) + { + var body = gameMatch.Groups["body"].Value; + var romMatch = ClrMameRomRegex.Match(body); + var gameFieldsBody = romMatch.Success ? body.Substring(0, romMatch.Index) : body; + var fields = ParseFields(gameFieldsBody); + + if (!fields.TryGetValue("name", out var canonicalName) || string.IsNullOrWhiteSpace(canonicalName)) + { + continue; + } + + var romFields = romMatch.Success ? ParseFields(romMatch.Groups["body"].Value) : new System.Collections.Generic.Dictionary(); + + var entry = CreateEntry(canonicalName, + fields.TryGetValue("cloneof", out var parent) ? parent : null, + romFields.TryGetValue("name", out var fileName) ? fileName : canonicalName, + fields.TryGetValue("id", out var id) ? id : null); + + AddHash(entry, "crc32", romFields.TryGetValue("crc", out var crc) ? crc : null, true, false); + AddHash(entry, "md5", romFields.TryGetValue("md5", out var md5) ? md5 : null, false, false); + AddHash(entry, "sha1", romFields.TryGetValue("sha1", out var sha1) ? sha1 : null, false, false); + + snapshot.Entries.Add(entry); + } + + return snapshot; + } + + private static System.Collections.Generic.Dictionary ParseFields(string body) + { + var fields = new System.Collections.Generic.Dictionary(); + + foreach (Match match in ClrMameFieldRegex.Matches(body)) + { + fields[match.Groups["name"].Value.ToLowerInvariant()] = match.Groups["quoted"].Success ? match.Groups["quoted"].Value : match.Groups["bare"].Value; + } + + return fields; + } + + private static NoIntroCatalogSnapshotEntry CreateEntry(string name, string parentName, string fileName, string id) + { + var nameMatch = NumberedNameRegex.Match(name); + var fileNameMatch = NumberedNameRegex.Match(fileName); + var releaseNumber = id; + var canonicalName = name; + var canonicalFileName = fileName; + + if (nameMatch.Success) + { + releaseNumber = nameMatch.Groups["number"].Value; + canonicalName = nameMatch.Groups["name"].Value; + } + + if (fileNameMatch.Success) + { + releaseNumber ??= fileNameMatch.Groups["number"].Value; + canonicalFileName = fileNameMatch.Groups["name"].Value; + } + + return new NoIntroCatalogSnapshotEntry + { + CanonicalName = canonicalName, + ParentCanonicalName = StripNumberPrefix(parentName), + CanonicalFileName = canonicalFileName, + ReleaseNumber = releaseNumber, + NumberedCanonicalFileName = releaseNumber != null ? fileName : null + }; + } + + private static string StripNumberPrefix(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return value; + } + + var match = NumberedNameRegex.Match(value); + return match.Success ? match.Groups["name"].Value : value; + } + + private static void AddHash(NoIntroCatalogSnapshotEntry entry, string hashType, string hashValue, bool isPrimary, bool isBadDump) + { + if (string.IsNullOrWhiteSpace(hashValue)) + { + return; + } + + entry.Hashes.Add(new NoIntroCatalogSnapshotHash + { + HashType = hashType, + HashValue = hashValue, + IsPrimary = isPrimary, + IsBadDump = isBadDump + }); + } + + private static bool IsBadDump(XElement rom) + { + var status = rom.Attribute("status")?.Value; + return status != null && !status.Equals("verified", StringComparison.OrdinalIgnoreCase) && !status.Equals("good", StringComparison.OrdinalIgnoreCase); + } + + private static string NormalizeSystemKey(string value) + { + return string.Concat(value.Trim().ToLowerInvariant().Select(c => char.IsLetterOrDigit(c) ? c : '-')).Trim('-'); + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSource.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSource.cs new file mode 100644 index 00000000..799afb28 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSource.cs @@ -0,0 +1,16 @@ +using System; +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroCatalogSource : ModelBase + { + public string Name { get; set; } + public string SourceUrl { get; set; } + public string PinnedRevision { get; set; } + public string CatalogVersion { get; set; } + public DateTime? LastSuccessfulSync { get; set; } + public DateTime? LastAttemptedSync { get; set; } + public string LastSyncError { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSourceRepository.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSourceRepository.cs new file mode 100644 index 00000000..4e0abd6a --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSourceRepository.cs @@ -0,0 +1,17 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroCatalogSourceRepository : IBasicRepository + { + } + + public class NoIntroCatalogSourceRepository : BasicRepository, INoIntroCatalogSourceRepository + { + public NoIntroCatalogSourceRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSyncCommand.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSyncCommand.cs new file mode 100644 index 00000000..115dbf58 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSyncCommand.cs @@ -0,0 +1,21 @@ +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroCatalogSyncCommand : Command + { + public int? CatalogSourceId { get; set; } + + public NoIntroCatalogSyncCommand() + { + } + + public NoIntroCatalogSyncCommand(int? catalogSourceId) + { + CatalogSourceId = catalogSourceId; + } + + public override bool SendUpdatesToClient => true; + public override bool IsTypeExclusive => true; + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSyncService.cs b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSyncService.cs new file mode 100644 index 00000000..8f777c44 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroCatalogSyncService.cs @@ -0,0 +1,325 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; +using NLog; +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroCatalogSyncService + { + void Sync(int? catalogSourceId = null); + } + + public class NoIntroCatalogSyncService : IExecute, INoIntroCatalogSyncService + { + private readonly NoIntroCatalogSourceRepository _sourceRepository; + private readonly NoIntroCatalogEntryRepository _entryRepository; + private readonly NoIntroCatalogHashRepository _hashRepository; + private readonly INoIntroCatalogDocumentClient _documentClient; + private readonly NoIntroCatalogSnapshotParser _snapshotParser; + private readonly Logger _logger; + + public NoIntroCatalogSyncService( + NoIntroCatalogSourceRepository sourceRepository, + NoIntroCatalogEntryRepository entryRepository, + NoIntroCatalogHashRepository hashRepository, + INoIntroCatalogDocumentClient documentClient, + NoIntroCatalogSnapshotParser snapshotParser, + Logger logger) + { + _sourceRepository = sourceRepository; + _entryRepository = entryRepository; + _hashRepository = hashRepository; + _documentClient = documentClient; + _snapshotParser = snapshotParser; + _logger = logger; + } + + public void Execute(NoIntroCatalogSyncCommand message) + { + Sync(message.CatalogSourceId); + } + + public void Sync(int? catalogSourceId = null) + { + var sources = catalogSourceId.HasValue + ? new List { _sourceRepository.Get(catalogSourceId.Value) } + : _sourceRepository.All().ToList(); + + if (!catalogSourceId.HasValue) + { + EnsureDefaultSources(sources); + } + + foreach (var source in sources) + { + try + { + SyncSource(source); + } + catch + { + if (catalogSourceId.HasValue) + { + throw; + } + } + } + } + + private void SyncSource(NoIntroCatalogSource source) + { + source.LastAttemptedSync = DateTime.UtcNow; + _sourceRepository.Update(source); + + try + { + var snapshot = _snapshotParser.Parse(_documentClient.Fetch(source.SourceUrl)); + EnrichWithNumberedCatalog(snapshot); + ReplaceCatalog(source, snapshot); + + source.CatalogVersion = snapshot.CatalogVersion; + source.LastSuccessfulSync = DateTime.UtcNow; + source.LastSyncError = null; + _sourceRepository.Update(source); + } + catch (Exception ex) + { + source.LastSyncError = ex.Message; + _sourceRepository.Update(source); + _logger.Warn(ex, "Failed syncing No-Intro catalog source {0}", source.SourceUrl); + throw; + } + } + + private void ReplaceCatalog(NoIntroCatalogSource source, NoIntroCatalogSnapshot snapshot) + { + var existingEntries = _entryRepository.GetBySourceId(source.Id); + var existingEntryIds = existingEntries.Select(x => x.Id).ToList(); + + if (existingEntryIds.Count > 0) + { + _hashRepository.DeleteByEntryIds(existingEntryIds); + _entryRepository.DeleteBySourceId(source.Id); + } + + var entries = snapshot.Entries.Select(entry => new NoIntroCatalogEntry + { + CatalogSourceId = source.Id, + SystemKey = snapshot.SystemKey, + CanonicalName = entry.CanonicalName, + ParentCanonicalName = entry.ParentCanonicalName, + PlatformFamily = NoIntroCatalogDefaults.MapPlatformFamily(snapshot.SystemKey), + CanonicalFileName = entry.CanonicalFileName, + ReleaseNumber = entry.ReleaseNumber, + NumberedCanonicalFileName = entry.NumberedCanonicalFileName + }).ToList(); + + _entryRepository.InsertMany(entries); + + var hashes = new List(); + + for (var i = 0; i < entries.Count; i++) + { + hashes.AddRange(snapshot.Entries[i].Hashes.Select(hash => new NoIntroCatalogHash + { + CatalogEntryId = entries[i].Id, + HashType = hash.HashType, + HashValue = hash.HashValue, + IsPrimary = hash.IsPrimary, + IsBadDump = hash.IsBadDump + })); + } + + if (hashes.Count > 0) + { + _hashRepository.InsertMany(hashes); + } + } + + private void EnsureDefaultSources(List sources) + { + foreach (var source in NoIntroCatalogDefaults.Sources) + { + AddDefaultSource(sources, source.Name, source.SourceUrl); + } + } + + private void AddDefaultSource(List sources, string name, string sourceUrl) + { + if (sources.Any(source => source.SourceUrl == sourceUrl)) + { + return; + } + + sources.Add(_sourceRepository.Insert(new NoIntroCatalogSource + { + Name = name, + SourceUrl = sourceUrl + })); + } + + private void EnrichWithNumberedCatalog(NoIntroCatalogSnapshot snapshot) + { + var systemId = GetDatOMaticSystemId(snapshot.SystemKey); + + if (!systemId.HasValue) + { + return; + } + + try + { + var numberedContent = _documentClient.FetchDatOMaticNumbered(systemId.Value); + + if (string.IsNullOrWhiteSpace(numberedContent)) + { + EnrichWithAdvansceneCatalog(snapshot); + return; + } + + var numberedSnapshot = _snapshotParser.Parse(numberedContent); + var numberedByHash = numberedSnapshot.Entries + .SelectMany(entry => entry.Hashes.Select(hash => new { Key = HashKey(hash), Entry = entry })) + .GroupBy(x => x.Key) + .ToDictionary(x => x.Key, x => x.First().Entry); + + foreach (var entry in snapshot.Entries) + { + var numberedEntry = entry.Hashes + .Select(hash => numberedByHash.GetValueOrDefault(HashKey(hash))) + .FirstOrDefault(match => match?.NumberedCanonicalFileName != null); + + if (numberedEntry == null) + { + continue; + } + + entry.ReleaseNumber = numberedEntry.ReleaseNumber; + entry.NumberedCanonicalFileName = numberedEntry.NumberedCanonicalFileName; + } + } + catch (InvalidOperationException ex) + { + _logger.Debug(ex, "DAT-o-MATIC numbered DAT unavailable for {0}; falling back to ADVANsCEne", snapshot.SystemKey); + EnrichWithAdvansceneCatalog(snapshot); + return; + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed enriching No-Intro catalog {0} with DAT-o-MATIC numbered filenames", snapshot.SystemKey); + } + + EnrichWithAdvansceneCatalog(snapshot); + } + + private void EnrichWithAdvansceneCatalog(NoIntroCatalogSnapshot snapshot) + { + var sourceUrl = GetAdvansceneSourceUrl(snapshot.SystemKey); + + if (sourceUrl == null) + { + return; + } + + var unnumbered = snapshot.Entries.Where(x => x.NumberedCanonicalFileName == null).ToList(); + + // Nothing left to number (DAT-o-MATIC covered everything) — skip + // the download entirely. + if (unnumbered.Count == 0) + { + return; + } + + try + { + var advansceneContent = _documentClient.FetchAdvanscene(sourceUrl); + + if (string.IsNullOrWhiteSpace(advansceneContent)) + { + return; + } + + var releaseNumbersByCrc = ParseAdvansceneReleaseNumbers(advansceneContent); + + foreach (var entry in unnumbered) + { + var crc = entry.Hashes.FirstOrDefault(x => x.HashType.Equals("crc32", StringComparison.OrdinalIgnoreCase)); + + if (crc == null || !releaseNumbersByCrc.TryGetValue(crc.HashValue.ToUpperInvariant(), out var releaseNumber)) + { + continue; + } + + entry.ReleaseNumber = releaseNumber; + entry.NumberedCanonicalFileName = $"{releaseNumber} - {entry.CanonicalFileName}"; + } + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed enriching No-Intro catalog {0} with ADVANsCEne release numbers", snapshot.SystemKey); + } + } + + private static Dictionary ParseAdvansceneReleaseNumbers(string content) + { + var document = XDocument.Parse(content); + + return document.Descendants("game") + .Select(game => new + { + ReleaseNumber = FormatReleaseNumber(game.Element("releaseNumber")?.Value), + Crc = game.Element("files")?.Elements("romCRC").FirstOrDefault()?.Value + }) + .Where(x => x.ReleaseNumber != null && !string.IsNullOrWhiteSpace(x.Crc)) + .GroupBy(x => x.Crc.ToUpperInvariant()) + .ToDictionary(x => x.Key, x => x.First().ReleaseNumber); + } + + private static string FormatReleaseNumber(string value) + { + return int.TryParse(value, out var number) ? number.ToString("0000") : null; + } + + private static string HashKey(NoIntroCatalogSnapshotHash hash) + { + return $"{hash.HashType}:{hash.HashValue}".ToLowerInvariant(); + } + + private static int? GetDatOMaticSystemId(string systemKey) + { + return systemKey switch + { + "nintendo---game-boy" => 46, + "nintendo---game-boy-color" => 47, + "nintendo---game-boy-advance" => 23, + "nintendo---game-boy-advance-multiboot" => 137, + "nintendo---game-boy-advance--multiboot" => 137, + "nintendo---game-boy-advance-e-reader" => 41, + "nintendo---game-boy-advance--e-reader" => 41, + "nintendo---game-boy-advance-play-yan" => 148, + "nintendo---game-boy-advance--play-yan" => 148, + "nintendo---game-boy-advance-video" => 297, + "nintendo---game-boy-advance--video" => 297, + "nintendo---nintendo-ds" => 28, + "nintendo---nintendo-ds-download-play" => 65, + "nintendo---nintendo-ds--download-play" => 65, + "nintendo---nintendo-ds-dsvision-sd-cards" => 319, + "nintendo---nintendo-ds--dsvision-sd-cards" => 319, + _ => null + }; + } + + private static string GetAdvansceneSourceUrl(string systemKey) + { + return systemKey switch + { + "nintendo---game-boy-advance" => "https://advanscene.com/offline/datas/ADVANsCEne_GBA.zip", + "nintendo---nintendo-ds" => "https://advanscene.com/offline/datas/ADVANsCEne_NDS_S.zip", + _ => null + }; + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroComponentClassification.cs b/src/NzbDrone.Core/RomCatalog/NoIntroComponentClassification.cs new file mode 100644 index 00000000..83d08735 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroComponentClassification.cs @@ -0,0 +1,8 @@ +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroComponentClassification + { + public NoIntroRomComponentType ComponentType { get; set; } + public bool IsFallback { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroComponentClassifier.cs b/src/NzbDrone.Core/RomCatalog/NoIntroComponentClassifier.cs new file mode 100644 index 00000000..f785b586 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroComponentClassifier.cs @@ -0,0 +1,362 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroComponentClassifier + { + NoIntroComponentClassification Classify(string relativePath, string fileName); + NoIntroCatalogPlan BuildCatalogPlan(IEnumerable entries); + } + + public class NoIntroComponentClassifier : INoIntroComponentClassifier + { + public NoIntroComponentClassification Classify(string relativePath, string fileName) + { + var path = relativePath ?? string.Empty; + var name = fileName ?? string.Empty; + var combined = $"{path}/{name}"; + + if (Contains(combined, "[BIOS]") || Contains(combined, "/BIOS/") || EndsWithFolder(path, "BIOS")) + { + return Exact(NoIntroRomComponentType.Bios); + } + + if (Contains(combined, "Romhack") || Contains(combined, "/Romhacks/")) + { + return Exact(NoIntroRomComponentType.RomhackOrUnverified); + } + + if (Contains(path, "GBA (e-Reader)")) + { + return Exact(NoIntroRomComponentType.EReaderCards); + } + + if (Contains(path, "GBA (Multiboot)") || Contains(path, "Download Play")) + { + return Exact(NoIntroRomComponentType.Multiboot); + } + + if (Contains(path, "GBA (Video)") || Contains(path, "GBA (Play-Yan)") || Contains(path, "DSvision SD cards")) + { + return Exact(NoIntroRomComponentType.Video); + } + + if (Contains(path, "GBA") || + Contains(path, "GBA (by-id)") || + Contains(path, "3DS") || + Contains(path, "PSP") || + Contains(path, "PlayStation Portable") || + Contains(path, "PlayStation Vita") || + Contains(path, "PlayStation 3") || + Contains(path, "/zip/") || + Contains(path, "/nds/") || + Contains(path, "/3ds/") || + Contains(path, "/cia/") || + Contains(path, "/iso/") || + Contains(path, "/pkg/") || + Contains(path, "/vpk/") || + EndsWithFolder(path, "zip") || + EndsWithFolder(path, "nds") || + EndsWithFolder(path, "3ds") || + EndsWithFolder(path, "cia") || + EndsWithFolder(path, "iso") || + EndsWithFolder(path, "pkg") || + EndsWithFolder(path, "vpk")) + { + return Exact(NoIntroRomComponentType.RetailRom); + } + + return Fallback(NoIntroRomComponentType.RomhackOrUnverified); + } + + public NoIntroCatalogPlan BuildCatalogPlan(IEnumerable entries) + { + var plan = new NoIntroCatalogPlan(); + + foreach (var entry in entries ?? Enumerable.Empty()) + { + AddEntry(plan, entry); + } + + return plan; + } + + private static void AddEntry(NoIntroCatalogPlan plan, NoIntroCatalogEntry entry) + { + var canonicalName = entry.CanonicalName ?? string.Empty; + + if (IsDownloadPlaySource(entry.SystemKey)) + { + AddDownloadPlaySourceEntry(plan, entry, canonicalName); + return; + } + + if (IsDownloadPlay(canonicalName)) + { + AddDownloadPlay(plan, entry, canonicalName); + return; + } + + if (IsStandaloneProduct(canonicalName)) + { + plan.StandaloneGames.Add(new NoIntroCatalogStandalonePlan + { + Title = canonicalName, + ComponentType = ClassifyStandaloneProduct(canonicalName) + }); + + return; + } + + var region = TryParseRegionRelease(canonicalName); + + if (region == null) + { + plan.StandaloneGames.Add(new NoIntroCatalogStandalonePlan + { + Title = canonicalName, + ComponentType = NoIntroRomComponentType.RetailRom + }); + + return; + } + + GetOrAddGame(plan, entry.SystemKey, region.GameTitle).RegionLanguageComponents.Add(new NoIntroCatalogComponentSlot + { + SlotLabel = region.SlotLabel, + CanonicalName = canonicalName, + ComponentType = NoIntroRomComponentType.RetailRom + }); + } + + private static void AddDownloadPlay(NoIntroCatalogPlan plan, NoIntroCatalogEntry entry, string canonicalName) + { + if (string.IsNullOrWhiteSpace(entry.ParentCanonicalName)) + { + plan.StandaloneGames.Add(new NoIntroCatalogStandalonePlan + { + Title = canonicalName, + ComponentType = NoIntroRomComponentType.Multiboot + }); + + return; + } + + GetOrAddGame(plan, entry.SystemKey, entry.ParentCanonicalName).DownloadPlayComponents.Add(new NoIntroCatalogComponentSlot + { + SlotLabel = "Download Play", + CanonicalName = canonicalName, + ComponentType = NoIntroRomComponentType.Multiboot + }); + } + + private static NoIntroCatalogGamePlan GetOrAddGame(NoIntroCatalogPlan plan, string systemKey, string gameTitle) + { + var game = plan.Games.SingleOrDefault(x => + x.SystemKey.Equals(systemKey, StringComparison.OrdinalIgnoreCase) && + x.GameTitle.Equals(gameTitle, StringComparison.Ordinal)); + + if (game != null) + { + return game; + } + + game = new NoIntroCatalogGamePlan + { + SystemKey = systemKey, + GameTitle = gameTitle + }; + + plan.Games.Add(game); + return game; + } + + private static RegionRelease TryParseRegionRelease(string canonicalName) + { + var tags = ParseTrailingTags(canonicalName); + + if (tags.Count == 0) + { + return null; + } + + var title = canonicalName.Substring(0, tags[0].OpenIndex).TrimEnd(); + + if (string.IsNullOrWhiteSpace(title)) + { + return null; + } + + return new RegionRelease + { + GameTitle = NormalizeGameTitle(title, tags), + SlotLabel = BuildSlotLabel(tags) + }; + } + + private static string NormalizeGameTitle(string title, List tags) + { + if (tags.Any(tag => Contains(tag.Label, "Kiosk")) && title.EndsWith(" Demo", StringComparison.Ordinal)) + { + return title.Substring(0, title.Length - " Demo".Length); + } + + return title; + } + + private static List ParseTrailingTags(string canonicalName) + { + var tags = new List(); + var cursor = canonicalName.Length; + + while (cursor > 0 && canonicalName[cursor - 1] == ')') + { + var openIndex = canonicalName.LastIndexOf('(', cursor - 1); + + if (openIndex < 1 || openIndex >= cursor - 1) + { + break; + } + + var label = canonicalName.Substring(openIndex + 1, cursor - openIndex - 2); + + if (string.IsNullOrWhiteSpace(label)) + { + break; + } + + tags.Insert(0, new ReleaseTag + { + OpenIndex = openIndex, + Label = label + }); + + cursor = openIndex; + + while (cursor > 0 && canonicalName[cursor - 1] == ' ') + { + cursor--; + } + } + + return tags; + } + + private static string BuildSlotLabel(List tags) + { + var first = tags[0].Label; + var rest = tags.Skip(1).Select(tag => $"({tag.Label})"); + + return string.Join(" ", new[] { first }.Concat(rest)); + } + + private static bool IsDownloadPlay(string canonicalName) + { + return Contains(canonicalName, "Download Play"); + } + + private static bool IsDownloadPlaySource(string systemKey) + { + return Contains(systemKey ?? string.Empty, "download-play"); + } + + private static void AddDownloadPlaySourceEntry(NoIntroCatalogPlan plan, NoIntroCatalogEntry entry, string canonicalName) + { + var region = TryParseRegionRelease(canonicalName); + + if (region == null) + { + plan.StandaloneGames.Add(new NoIntroCatalogStandalonePlan + { + Title = canonicalName, + ComponentType = NoIntroRomComponentType.Multiboot + }); + + return; + } + + GetOrAddGame(plan, entry.SystemKey, region.GameTitle).RegionLanguageComponents.Add(new NoIntroCatalogComponentSlot + { + SlotLabel = region.SlotLabel, + CanonicalName = canonicalName, + ComponentType = NoIntroRomComponentType.Multiboot + }); + } + + private static bool IsStandaloneProduct(string canonicalName) + { + if (Contains(canonicalName, "(Kiosk)")) + { + return false; + } + + return Contains(canonicalName, "Game Boy Advance Video") || + Contains(canonicalName, "Play-Yan") || + Contains(canonicalName, "DSvision") || + Contains(canonicalName, "[BIOS]") || + Contains(canonicalName, "(BIOS)") || + Contains(canonicalName, " Demo") || + Contains(canonicalName, " Prototype") || + Contains(canonicalName, "Not for Resale"); + } + + private static NoIntroRomComponentType ClassifyStandaloneProduct(string canonicalName) + { + if (Contains(canonicalName, "[BIOS]") || Contains(canonicalName, "(BIOS)")) + { + return NoIntroRomComponentType.Bios; + } + + if (Contains(canonicalName, "Game Boy Advance Video") || Contains(canonicalName, "Play-Yan") || Contains(canonicalName, "DSvision")) + { + return NoIntroRomComponentType.Video; + } + + return NoIntroRomComponentType.RomhackOrUnverified; + } + + private static NoIntroComponentClassification Exact(NoIntroRomComponentType componentType) + { + return new NoIntroComponentClassification + { + ComponentType = componentType, + IsFallback = false + }; + } + + private static NoIntroComponentClassification Fallback(NoIntroRomComponentType componentType) + { + return new NoIntroComponentClassification + { + ComponentType = componentType, + IsFallback = true + }; + } + + private static bool Contains(string value, string pattern) + { + return value.Contains(pattern, StringComparison.OrdinalIgnoreCase); + } + + private static bool EndsWithFolder(string path, string folderName) + { + return path.EndsWith($"/{folderName}", StringComparison.OrdinalIgnoreCase) || + path.EndsWith($"\\{folderName}", StringComparison.OrdinalIgnoreCase) || + path.Equals(folderName, StringComparison.OrdinalIgnoreCase); + } + + private class RegionRelease + { + public string GameTitle { get; set; } + public string SlotLabel { get; set; } + } + + private class ReleaseTag + { + public int OpenIndex { get; set; } + public string Label { get; set; } + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroRenameProfileEvaluator.cs b/src/NzbDrone.Core/RomCatalog/NoIntroRenameProfileEvaluator.cs new file mode 100644 index 00000000..77e21529 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroRenameProfileEvaluator.cs @@ -0,0 +1,51 @@ +using System; +using NzbDrone.Core.Organizer; + +namespace NzbDrone.Core.RomCatalog +{ + public static class NoIntroRenameProfileEvaluator + { + public static string GetExpectedFileName(NoIntroCatalogEntry catalogEntry, string actualFileName, RenameProfile renameProfile) + { + if (renameProfile == RenameProfile.NoIntroPreserveById && !string.IsNullOrWhiteSpace(catalogEntry.NumberedCanonicalFileName)) + { + return catalogEntry.NumberedCanonicalFileName; + } + + if (renameProfile == RenameProfile.NoIntroPreserveById && IsByIdFileName(actualFileName, catalogEntry.CanonicalFileName)) + { + return actualFileName; + } + + return catalogEntry.CanonicalFileName; + } + + public static bool MatchesProfile(NoIntroCatalogEntry catalogEntry, string actualFileName, RenameProfile renameProfile) + { + if (actualFileName.Equals(catalogEntry.CanonicalFileName, StringComparison.Ordinal)) + { + return true; + } + + if (renameProfile != RenameProfile.NoIntroPreserveById) + { + return false; + } + + return !string.IsNullOrWhiteSpace(catalogEntry.NumberedCanonicalFileName) + ? actualFileName.Equals(catalogEntry.NumberedCanonicalFileName, StringComparison.Ordinal) + : IsByIdFileName(actualFileName, catalogEntry.CanonicalFileName); + } + + private static bool IsByIdFileName(string actualFileName, string canonicalFileName) + { + if (string.IsNullOrWhiteSpace(actualFileName) || string.IsNullOrWhiteSpace(canonicalFileName)) + { + return false; + } + + var prefix = $" - {canonicalFileName}"; + return actualFileName.EndsWith(prefix, StringComparison.Ordinal); + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroRomComponentType.cs b/src/NzbDrone.Core/RomCatalog/NoIntroRomComponentType.cs new file mode 100644 index 00000000..9aa303ac --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroRomComponentType.cs @@ -0,0 +1,12 @@ +namespace NzbDrone.Core.RomCatalog +{ + public enum NoIntroRomComponentType + { + RetailRom = 0, + EReaderCards = 1, + Multiboot = 2, + Video = 3, + Bios = 4, + RomhackOrUnverified = 5 + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroRomHasher.cs b/src/NzbDrone.Core/RomCatalog/NoIntroRomHasher.cs new file mode 100644 index 00000000..83f6d423 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroRomHasher.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using ICSharpCode.SharpZipLib.Checksum; + +namespace NzbDrone.Core.RomCatalog +{ + public static class NoIntroRomHasher + { + public static NoIntroHashTriplet Compute(Stream stream) + { + using var md5 = MD5.Create(); + using var sha1 = SHA1.Create(); + var crc = new Crc32(); + var buffer = new byte[8192]; + + int bytesRead; + + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + md5.TransformBlock(buffer, 0, bytesRead, null, 0); + sha1.TransformBlock(buffer, 0, bytesRead, null, 0); + crc.Update(new ArraySegment(buffer, 0, bytesRead)); + } + + md5.TransformFinalBlock(Array.Empty(), 0, 0); + sha1.TransformFinalBlock(Array.Empty(), 0, 0); + + return new NoIntroHashTriplet + { + Md5 = ToHex(md5.Hash), + Sha1 = ToHex(sha1.Hash), + Crc32 = crc.Value.ToString("X8"), + PreferredHashType = "sha1", + PreferredHashValue = ToHex(sha1.Hash) + }; + } + + private static string ToHex(byte[] bytes) + { + return BitConverter.ToString(bytes).Replace("-", string.Empty); + } + } + + public class NoIntroHashTriplet + { + public string Md5 { get; set; } + public string Sha1 { get; set; } + public string Crc32 { get; set; } + public string PreferredHashType { get; set; } + public string PreferredHashValue { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroSystemMapping.cs b/src/NzbDrone.Core/RomCatalog/NoIntroSystemMapping.cs new file mode 100644 index 00000000..9de93df5 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroSystemMapping.cs @@ -0,0 +1,13 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Games; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroSystemMapping : ModelBase + { + public string SystemKey { get; set; } + public string DisplayName { get; set; } + public PlatformFamily PlatformFamily { get; set; } + public string RootRelativePathPattern { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationResult.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationResult.cs new file mode 100644 index 00000000..02c7e070 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationResult.cs @@ -0,0 +1,23 @@ +using System; +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroVerificationResult : ModelBase + { + public int SnapshotId { get; set; } + public int VerificationSetId { get; set; } + public int? CatalogEntryId { get; set; } + public string RelativePath { get; set; } + public string ArchivePath { get; set; } + public string MemberPath { get; set; } + public string ActualFileName { get; set; } + public string ExpectedFileName { get; set; } + public string HashType { get; set; } + public string HashValue { get; set; } + public NoIntroVerificationStatus VerificationStatus { get; set; } + public bool IsDuplicate { get; set; } + public bool IsMissing { get; set; } + public DateTime VerifiedAt { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationResultRepository.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationResultRepository.cs new file mode 100644 index 00000000..57957c24 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationResultRepository.cs @@ -0,0 +1,17 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroVerificationResultRepository : IBasicRepository + { + } + + public class NoIntroVerificationResultRepository : BasicRepository, INoIntroVerificationResultRepository + { + public NoIntroVerificationResultRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationService.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationService.cs new file mode 100644 index 00000000..8fca9606 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationService.cs @@ -0,0 +1,285 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using ICSharpCode.SharpZipLib.Zip; +using NzbDrone.Core.Organizer; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroVerificationService + { + NoIntroVerificationSnapshot Verify(int verificationSetId, IEnumerable filePaths); + } + + public class NoIntroVerificationService : INoIntroVerificationService + { + private readonly NoIntroVerificationSetRepository _verificationSetRepository; + private readonly NoIntroVerificationSnapshotRepository _snapshotRepository; + private readonly NoIntroVerificationResultRepository _resultRepository; + private readonly NoIntroCatalogEntryRepository _entryRepository; + private readonly NoIntroCatalogHashRepository _hashRepository; + private readonly INamingConfigService _namingConfigService; + + public NoIntroVerificationService( + NoIntroVerificationSetRepository verificationSetRepository, + NoIntroVerificationSnapshotRepository snapshotRepository, + NoIntroVerificationResultRepository resultRepository, + NoIntroCatalogEntryRepository entryRepository, + NoIntroCatalogHashRepository hashRepository, + INamingConfigService namingConfigService) + { + _verificationSetRepository = verificationSetRepository; + _snapshotRepository = snapshotRepository; + _resultRepository = resultRepository; + _entryRepository = entryRepository; + _hashRepository = hashRepository; + _namingConfigService = namingConfigService; + } + + public NoIntroVerificationSnapshot Verify(int verificationSetId, IEnumerable filePaths) + { + var verificationSet = _verificationSetRepository.Get(verificationSetId); + var snapshot = _snapshotRepository.Insert(new NoIntroVerificationSnapshot + { + VerificationSetId = verificationSet.Id, + CatalogSourceId = verificationSet.CatalogSourceId, + CatalogRevision = string.Empty, + StartedAt = DateTime.UtcNow + }); + + var entries = _entryRepository.GetBySourceId(verificationSet.CatalogSourceId); + var entryIds = entries.Select(x => x.Id).ToList(); + var hashes = _hashRepository.GetByEntryIds(entryIds); + var renameProfile = _namingConfigService.GetConfig().RenameProfile; + var hashMap = hashes.GroupBy(x => $"{x.HashType}:{x.HashValue}".ToLowerInvariant()) + .ToDictionary(x => x.Key, x => x.First()); + var entryMap = entries.ToDictionary(x => x.Id); + + var results = filePaths.Select(path => VerifyPath(snapshot, verificationSet, path, hashMap, entryMap, renameProfile)).ToList(); + MarkDuplicates(results); + results.AddRange(BuildMissingResults(snapshot, verificationSet, entries, results)); + + _resultRepository.InsertMany(results); + + snapshot.CompletedAt = DateTime.UtcNow; + return _snapshotRepository.Update(snapshot); + } + + private static NoIntroVerificationResult VerifyPath( + NoIntroVerificationSnapshot snapshot, + NoIntroVerificationSet verificationSet, + string path, + Dictionary hashMap, + Dictionary entryMap, + RenameProfile renameProfile) + { + try + { + return Path.GetExtension(path).Equals(".zip", StringComparison.OrdinalIgnoreCase) + ? VerifyArchive(snapshot, verificationSet, path, hashMap, entryMap, renameProfile) + : VerifyFile(snapshot, verificationSet, path, hashMap, entryMap, renameProfile); + } + catch (Exception) + { + return BuildUnknownResult(snapshot, verificationSet, path, Path.GetFileName(path), null, null, null); + } + } + + private static NoIntroVerificationResult VerifyFile( + NoIntroVerificationSnapshot snapshot, + NoIntroVerificationSet verificationSet, + string path, + Dictionary hashMap, + Dictionary entryMap, + RenameProfile renameProfile) + { + using var stream = File.OpenRead(path); + var hashes = ComputeHashes(stream); + return BuildMatchedResult(snapshot, verificationSet, path, null, null, Path.GetFileName(path), hashes, hashMap, entryMap, renameProfile); + } + + private static NoIntroVerificationResult VerifyArchive( + NoIntroVerificationSnapshot snapshot, + NoIntroVerificationSet verificationSet, + string path, + Dictionary hashMap, + Dictionary entryMap, + RenameProfile renameProfile) + { + using var fileStream = File.OpenRead(path); + using var zipFile = new ZipFile(fileStream); + var fileEntries = zipFile.Cast().Where(x => x.IsFile).ToList(); + + if (fileEntries.Count != 1) + { + return new NoIntroVerificationResult + { + SnapshotId = snapshot.Id, + VerificationSetId = verificationSet.Id, + CatalogEntryId = null, + RelativePath = GetRelativePath(verificationSet.RootPath, path), + ArchivePath = GetRelativePath(verificationSet.RootPath, path), + MemberPath = null, + ActualFileName = Path.GetFileName(path), + ExpectedFileName = null, + HashType = null, + HashValue = null, + VerificationStatus = NoIntroVerificationStatus.Unknown, + IsDuplicate = false, + IsMissing = false, + VerifiedAt = DateTime.UtcNow + }; + } + + var entry = fileEntries[0]; + using var memberStream = zipFile.GetInputStream(entry); + var hashes = ComputeHashes(memberStream); + + return BuildMatchedResult(snapshot, verificationSet, path, GetRelativePath(verificationSet.RootPath, path), entry.Name, Path.GetFileName(path), hashes, hashMap, entryMap, renameProfile); + } + + private static NoIntroVerificationResult BuildMatchedResult( + NoIntroVerificationSnapshot snapshot, + NoIntroVerificationSet verificationSet, + string fullPath, + string archivePath, + string memberPath, + string actualFileName, + NoIntroHashTriplet hashes, + Dictionary hashMap, + Dictionary entryMap, + RenameProfile renameProfile) + { + var matchedHash = FindMatch(hashes, hashMap); + + if (matchedHash == null) + { + return BuildUnknownResult(snapshot, verificationSet, fullPath, actualFileName, archivePath, memberPath, hashes); + } + + var catalogEntry = entryMap[matchedHash.CatalogEntryId]; + var expectedFileName = NoIntroRenameProfileEvaluator.GetExpectedFileName(catalogEntry, actualFileName, renameProfile); + var verificationStatus = matchedHash.IsBadDump + ? NoIntroVerificationStatus.BadDump + : NoIntroRenameProfileEvaluator.MatchesProfile(catalogEntry, actualFileName, renameProfile) + ? NoIntroVerificationStatus.Verified + : NoIntroVerificationStatus.NameMismatch; + + return new NoIntroVerificationResult + { + SnapshotId = snapshot.Id, + VerificationSetId = verificationSet.Id, + CatalogEntryId = catalogEntry.Id, + RelativePath = GetRelativePath(verificationSet.RootPath, fullPath), + ArchivePath = archivePath, + MemberPath = memberPath, + ActualFileName = actualFileName, + ExpectedFileName = expectedFileName, + HashType = matchedHash.HashType, + HashValue = matchedHash.HashValue, + VerificationStatus = verificationStatus, + IsDuplicate = false, + IsMissing = false, + VerifiedAt = DateTime.UtcNow + }; + } + + private static NoIntroCatalogHash FindMatch(NoIntroHashTriplet hashes, Dictionary hashMap) + { + return TryGetHash("sha1", hashes.Sha1, hashMap) ?? + TryGetHash("md5", hashes.Md5, hashMap) ?? + TryGetHash("crc32", hashes.Crc32, hashMap); + } + + private static NoIntroCatalogHash TryGetHash(string hashType, string hashValue, Dictionary hashMap) + { + if (string.IsNullOrWhiteSpace(hashValue)) + { + return null; + } + + hashMap.TryGetValue($"{hashType}:{hashValue}".ToLowerInvariant(), out var matchedHash); + return matchedHash; + } + + private static void MarkDuplicates(List results) + { + var duplicates = results.Where(x => x.CatalogEntryId.HasValue).GroupBy(x => x.CatalogEntryId.Value).Where(x => x.Count() > 1); + + foreach (var duplicateGroup in duplicates) + { + foreach (var result in duplicateGroup) + { + result.IsDuplicate = true; + } + } + } + + private static List BuildMissingResults( + NoIntroVerificationSnapshot snapshot, + NoIntroVerificationSet verificationSet, + List entries, + List results) + { + var matchedEntryIds = results.Where(x => x.CatalogEntryId.HasValue).Select(x => x.CatalogEntryId.Value).ToHashSet(); + + return entries.Where(entry => !matchedEntryIds.Contains(entry.Id)).Select(entry => new NoIntroVerificationResult + { + SnapshotId = snapshot.Id, + VerificationSetId = verificationSet.Id, + CatalogEntryId = entry.Id, + RelativePath = string.Empty, + ArchivePath = null, + MemberPath = null, + ActualFileName = string.Empty, + ExpectedFileName = entry.CanonicalFileName, + HashType = null, + HashValue = null, + VerificationStatus = NoIntroVerificationStatus.Unknown, + IsDuplicate = false, + IsMissing = true, + VerifiedAt = DateTime.UtcNow + }).ToList(); + } + + private static string GetRelativePath(string rootPath, string fullPath) + { + return fullPath.StartsWith(rootPath, StringComparison.OrdinalIgnoreCase) + ? fullPath.Substring(rootPath.Length).TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + : fullPath; + } + + private static NoIntroVerificationResult BuildUnknownResult( + NoIntroVerificationSnapshot snapshot, + NoIntroVerificationSet verificationSet, + string fullPath, + string actualFileName, + string archivePath, + string memberPath, + NoIntroHashTriplet hashes) + { + return new NoIntroVerificationResult + { + SnapshotId = snapshot.Id, + VerificationSetId = verificationSet.Id, + RelativePath = GetRelativePath(verificationSet.RootPath, fullPath), + ArchivePath = archivePath, + MemberPath = memberPath, + ActualFileName = actualFileName, + ExpectedFileName = null, + HashType = hashes?.PreferredHashType, + HashValue = hashes?.PreferredHashValue, + VerificationStatus = NoIntroVerificationStatus.Unknown, + IsDuplicate = false, + IsMissing = false, + VerifiedAt = DateTime.UtcNow + }; + } + + private static NoIntroHashTriplet ComputeHashes(Stream stream) + { + return NoIntroRomHasher.Compute(stream); + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSet.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSet.cs new file mode 100644 index 00000000..f6f03972 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSet.cs @@ -0,0 +1,12 @@ +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroVerificationSet : ModelBase + { + public int CatalogSourceId { get; set; } + public string SystemKey { get; set; } + public string RootPath { get; set; } + public bool Enabled { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSetRepository.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSetRepository.cs new file mode 100644 index 00000000..61bf1928 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSetRepository.cs @@ -0,0 +1,17 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroVerificationSetRepository : IBasicRepository + { + } + + public class NoIntroVerificationSetRepository : BasicRepository, INoIntroVerificationSetRepository + { + public NoIntroVerificationSetRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSnapshot.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSnapshot.cs new file mode 100644 index 00000000..4d9e4b2d --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSnapshot.cs @@ -0,0 +1,14 @@ +using System; +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.RomCatalog +{ + public class NoIntroVerificationSnapshot : ModelBase + { + public int VerificationSetId { get; set; } + public int CatalogSourceId { get; set; } + public string CatalogRevision { get; set; } + public DateTime StartedAt { get; set; } + public DateTime? CompletedAt { get; set; } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSnapshotRepository.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSnapshotRepository.cs new file mode 100644 index 00000000..24efcb2f --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationSnapshotRepository.cs @@ -0,0 +1,17 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.RomCatalog +{ + public interface INoIntroVerificationSnapshotRepository : IBasicRepository + { + } + + public class NoIntroVerificationSnapshotRepository : BasicRepository, INoIntroVerificationSnapshotRepository + { + public NoIntroVerificationSnapshotRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/RomCatalog/NoIntroVerificationStatus.cs b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationStatus.cs new file mode 100644 index 00000000..375a9715 --- /dev/null +++ b/src/NzbDrone.Core/RomCatalog/NoIntroVerificationStatus.cs @@ -0,0 +1,10 @@ +namespace NzbDrone.Core.RomCatalog +{ + public enum NoIntroVerificationStatus + { + Verified = 0, + NameMismatch = 1, + Unknown = 2, + BadDump = 3 + } +}