Implement cardano-config migrate

README.md

@@ -10,8 +10,8 @@ The goal is one shared parser for applications that need the node's configuratio
 such as [`cardano-cli`](https://github.com/IntersectMBO/cardano-cli),
 [`dmq-node`](https://github.com/IntersectMBO/dmq-node/) and
 [the `ouroboros-consensus` tools](https://github.com/IntersectMBO/ouroboros-consensus/tree/main/ouroboros-consensus-cardano#consensus-db-tools).
-The bundled `cardano-config` executable exposes the same via its `resolve` and
-`schema` subcommands.
+The bundled `cardano-config` executable exposes the same via its `resolve`,
+`schema` and `migrate` subcommands.
 
 ## Recommended format
 
@@ -42,10 +42,37 @@ pointing to that component's schema (e.g. a `StorageConfig` sub-file uses `schem
 so editors and validators pick up the right schema for the sub-file. The key is
 an annotation: the parser accepts and ignores it.
 
-To port an old config to the new format, group the component keys under their
-sections inside `Configuration` and add the `Version` / `MinNodeVersion`
-envelope. `cardano-config schema` documents the recommended form;
-`--legacy-one-file` documents the flat form.
+To port an old config to the new format, run `cardano-config migrate` (it reads
+`-` as stdin, so you can fetch and convert in one step):
+
+```console
+$ cardano-config migrate old-config.json > config.json
+$ curl -sL <url-of-old-config> | cardano-config migrate - > config.json
+```
+
+It reshapes the document into the envelope as JSON: it adds `$schema` and
+`Version`, carries `MinNodeVersion` through, and groups each component's keys
+under its section inside `Configuration`. It also brings field names up to date:
+the parser rejects the old names, so `migrate` rewrites the ones that were
+renamed (`hardLimit`/`softLimit`/`delay` → `HardLimit`/`SoftLimit`/`Delay`,
+`EnableRpc`/`RpcSocketPath` → `EnableGrpc`/`GrpcSocketPath`, `TargetNumberOf*` →
+`DeadlineTargetNumberOf*`) and drops the ones that were removed
+(`PBftSignatureThreshold`, `LastKnownBlockVersion-Major`/`-Minor`/`-Alt`, now
+supplied by consensus defaults; the vestigial `Protocol`; and
+`MaxKnownMajorProtocolVersion`, a dead key the node never read). Apart from that
+it preserves the values as written and does not fill in defaults, inline
+referenced sub-files, or read genesis files; follow it with `resolve` to check
+the result.
+
+A genuinely unrecognised key (a typo, say) is **kept** rather than silently
+dropped, so nothing is lost - but it remains unrecognised and so still surfaces
+as an `UnrecognisedKeys` warning on the next parse. Remove it by hand if you want
+a warning-free config.
+
+(To port by hand instead: group the component keys under their sections inside
+`Configuration` and add the `Version` / `MinNodeVersion` envelope. `cardano-config
+schema` documents the recommended form; `--legacy-one-file` documents the flat
+form.)
 
 ## Defaults and layering
 

app/Main.hs

@@ -1,4 +1,4 @@
--- | The @cardano-config@ command-line tool. It exposes two subcommands:
+-- | The @cardano-config@ command-line tool. It exposes three subcommands:
 --
 --   * @cardano-config resolve@ resolves a @cardano-node@ configuration
 --     (per-component defaults, the configuration file and the CLI flags),
@@ -7,11 +7,17 @@
 --
 --   * @cardano-config schema@ dumps the configuration JSON Schema (for the
 --     whole configuration or a single component), derived from the same codecs.
+--
+--   * @cardano-config migrate@ reshapes a configuration into the recommended
+--     @{ $schema, Version, MinNodeVersion, Configuration }@ envelope and prints
+--     it as JSON (a purely structural migration, preserving the values).
 module Main (main) where
 
 import Cardano.Configuration (parseConfigurationFiles, renderConfigWarning, resolveConfiguration)
 import Cardano.Configuration.CliArgs (CliArgs, configFilePath, parseCliArgs)
 import Cardano.Configuration.File (componentDefaults)
+import Cardano.Configuration.File.Merge (decodeValueFile)
+import Cardano.Configuration.File.Migrate (migrate)
 import Cardano.Configuration.Render (GenesisRendering (..), nodeConfigurationToJSON)
 import Cardano.Configuration.Schema
   ( configurationSchemas
@@ -28,6 +34,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import Data.Foldable (for_)
 import Data.List (intercalate)
 import qualified Data.Text as T
+import Data.Yaml (decodeThrow)
 import Data.Yaml.Pretty (encodePretty, setConfCompare, setConfDropNull)
 import qualified Data.Yaml.Pretty as Yaml
 import Options.Applicative
@@ -42,6 +49,8 @@ data Command
     Resolve CliArgs GenesisRendering
   | -- | @schema@: dump a JSON Schema.
     Schema SchemaCmd
+  | -- | @migrate@: reshape a configuration file into the Version1 envelope.
+    Migrate FilePath
 
 -- | What the @schema@ subcommand should print.
 data SchemaCmd
@@ -91,6 +100,20 @@ commandParser =
                   <> footerDoc (Just schemaValidationHelp)
               )
           )
+        <> command
+          "migrate"
+          ( info
+              ( Migrate
+                  <$> strArgument
+                    (metavar "CONFIG" <> help "Configuration file to migrate (JSON or YAML), or - for stdin.")
+              )
+              ( progDesc
+                  ( "Reshape a configuration into the recommended "
+                      <> "{ $schema, Version, MinNodeVersion, Configuration } envelope and print it as JSON. "
+                      <> "Preserves the values as written (no defaults are filled, no sub-files inlined)."
+                  )
+              )
+          )
     )
 
 -- | How to validate a configuration against the schema, shown under
@@ -148,6 +171,19 @@ withGenesesFlag =
 run :: Command -> IO ()
 run (Resolve cli geneses) = runResolve cli geneses
 run (Schema cmd) = runSchema cmd
+run (Migrate path) = runMigrate path
+
+-- | Read a configuration and print it, reshaped into the Version1 envelope, as
+-- JSON. A purely structural migration: it does not resolve, default or validate.
+-- A path of @-@ reads the configuration from stdin (so it composes with @curl@).
+runMigrate :: FilePath -> IO ()
+runMigrate path = handleAny (die . displayException) $ do
+  raw <- case path of
+    "-" -> BS.getContents >>= decodeThrow
+    _ -> decodeValueFile Nothing path
+  let (migrated, warnings) = migrate raw
+  for_ warnings $ hPutStrLn stderr . ("Warning: " <>) . renderConfigWarning
+  dump migrated
 
 -- | Resolve a configuration and print it as YAML.
 runResolve :: CliArgs -> GenesisRendering -> IO ()

cardano-config.cabal

@@ -53,6 +53,7 @@ library internal
     Cardano.Configuration.File.Lint
     Cardano.Configuration.File.Mempool
     Cardano.Configuration.File.Merge
+    Cardano.Configuration.File.Migrate
     Cardano.Configuration.File.Network
     Cardano.Configuration.File.Protocol
     Cardano.Configuration.File.Storage

src/Cardano/Configuration/File.hs

@@ -54,7 +54,6 @@ import Cardano.Configuration.File.Consensus
 import Cardano.Configuration.File.Error (ConfigurationParsingError (..))
 import Cardano.Configuration.File.Lint
   ( ConfigWarning (..)
-  , checkEnvelope
   , configWarnings
   , renderConfigWarning
   )
@@ -67,6 +66,7 @@ import Cardano.Configuration.File.Merge
   , sectionUserLayer
   , splitEnvelope
   )
+import Cardano.Configuration.File.Migrate (migrate)
 import Cardano.Configuration.File.Network
 import Cardano.Configuration.File.Protocol
 import Cardano.Configuration.File.Storage
@@ -184,11 +184,21 @@ componentDefaults =
 parseConfigurationFiles ::
   HasCallStack => FilePath -> IO (NodeConfigurationFromFile, [ConfigWarning])
 parseConfigurationFiles cfgFile = do
-  mainValue <- decodeValueFile Nothing cfgFile
+  rawValue <- decodeValueFile Nothing cfgFile
+  -- Every document is run through 'migrate' before parsing, so an already-enveloped
+  -- configuration that still uses a pre-rename field name (or carries a stray
+  -- top-level sibling) is brought up to the current shape too — not only a
+  -- non-enveloped one. Migration is idempotent, so a configuration already in the
+  -- canonical form is left untouched; a 'MigratedToCurrentFormat' warning is raised
+  -- only when migration actually changed the document (@migrated /= rawValue@, an
+  -- order-independent comparison). 'migrate' also returns its own warnings for the
+  -- fields it had to reconcile. If the migrated document still cannot be parsed, the
+  -- parse error surfaces as usual.
+  let (mainValue, migrateWarnings) = migrate rawValue
+      migrationWarnings =
+        [MigratedToCurrentFormat | mainValue /= rawValue] <> migrateWarnings
   (version, minNodeVer, configValue) <- splitEnvelope mainValue
-  -- 'checkEnvelope' inspects the raw top-level document (is it the Version1
-  -- envelope?); the rest inspect the unwrapped configuration object.
-  let warnings = checkEnvelope mainValue <> configWarnings configValue
+  let warnings = migrationWarnings <> configWarnings configValue
       root = takeDirectory cfgFile
   config <- case version of
     1 -> parseConfigurationVersion1 root minNodeVer configValue

src/Cardano/Configuration/File/Lint.hs

@@ -1,8 +1,6 @@
--- | Linting of the top-level configuration keys: detecting unrecognised keys,
--- keys shadowed by a component supplied as its own section, and use of the legacy
--- single-file form. This is the only part of the configuration-file handling that
--- depends on the schema (for the set of recognised keys and the per-component key
--- listing).
+-- | Linting of the top-level configuration keys: detecting unrecognised keys.
+-- This is the only part of the configuration-file handling that depends on the
+-- schema (for the set of recognised keys).
 --
 -- The checks are pure and return structured 'ConfigWarning's; how (or whether) to
 -- surface them — print, log via a tracer, treat as fatal — is left to the caller.
@@ -11,12 +9,9 @@ module Cardano.Configuration.File.Lint
   , renderConfigWarning
   , configWarnings
   , checkUnknownKeys
-  , checkShadowedKeys
-  , checkLegacyFormat
-  , checkEnvelope
   ) where
 
-import Cardano.Configuration.Schema (componentPropertyNames, recognisedKeys)
+import Cardano.Configuration.Schema (recognisedKeys)
 import Data.Aeson (Value (..))
 import qualified Data.Aeson.Key as K
 import qualified Data.Aeson.KeyMap as KM
@@ -29,19 +24,25 @@ import qualified Data.Text as T
 -- to @stderr@; another consumer might log them through its own tracer, or treat
 -- them as errors).
 data ConfigWarning
-  = -- | Top-level keys that no parser recognises (typically typos); they are
-    -- ignored.
+  = -- | Top-level keys that no parser recognises: typos, or a component property
+    -- placed flat under @Configuration@ instead of under its section. They are
+    -- ignored (not resolved into a section).
     UnrecognisedKeys [String]
-  | -- | Top-level keys ignored because their component was also given as its
-    -- own section (which wins). Each pair is @(section, key)@.
-    ShadowedKeys [(Text, Text)]
-  | -- | The configuration uses the legacy single-file form (component keys at the
-    -- top level) rather than the recommended split-file form.
-    LegacySingleFileFormat
-  | -- | The document is not in the recommended Version1 envelope form: one or
-    -- more of the top-level @$schema@, @Version@, @MinNodeVersion@ and @Configuration@ keys
-    -- is absent. Carries the missing key names.
-    NotVersion1Envelope [Text]
+  | -- | The document was not in the current canonical format, so migrating it (see
+    -- @Cardano.Configuration.File.Migrate.migrate@) changed it before parsing —
+    -- either it was not in the Version1 envelope, or it still used a pre-rename
+    -- field name, or it carried an obsolete key. Run @cardano-config migrate@ to
+    -- update the file on disk.
+    MigratedToCurrentFormat
+  | -- | While migrating, both the old and the current name of a renamed field
+    -- were present at the same level (@(old, new)@). The value under the current
+    -- name is kept and the one under the old name is dropped. Reconcile the two by
+    -- hand if the dropped value was the one you meant to keep.
+    RenamedKeyCollision Text Text
+  | -- | While migrating, a key appeared both as a top-level sibling of the
+    -- @Configuration@ envelope and inside it. The value inside @Configuration@ (the
+    -- canonical location) is kept and the top-level one is dropped.
+    EnvelopeKeyCollision Text
   | -- | A consistency check of warning severity did not hold on the resolved
     -- configuration (e.g. the Mithril snapshot policy under the V2LSM backend
     -- without an @LSMExportPath@). The configuration is still accepted; the
@@ -55,75 +56,45 @@ renderConfigWarning :: ConfigWarning -> String
 renderConfigWarning = \case
   UnrecognisedKeys ks ->
     "unrecognised configuration key(s): " <> intercalate ", " ks <> " (ignored)"
-  ShadowedKeys sks ->
-    "top-level configuration key(s) ignored because their component is given as a separate section: "
-      <> intercalate ", " (map describe sks)
-   where
-    describe (section, key) = T.unpack key <> " (shadowed by the " <> T.unpack section <> " section)"
-  LegacySingleFileFormat ->
-    "the configuration uses the legacy single-file form (component keys at the top level); "
-      <> "consider porting it to the split-file form (each component under its section key)"
-  NotVersion1Envelope missing ->
-    "the configuration is not in the Version1 envelope form; missing top-level key(s): "
-      <> intercalate ", " (map T.unpack missing)
-      <> " (expected { $schema, Version, MinNodeVersion, Configuration })"
+  MigratedToCurrentFormat ->
+    "the configuration was not in the current canonical format; "
+      <> "it was migrated before parsing "
+      <> "(run `cardano-config migrate` to update the file)"
+  RenamedKeyCollision old new ->
+    "both the old key \""
+      <> T.unpack old
+      <> "\" and its current name \""
+      <> T.unpack new
+      <> "\" are present; keeping \""
+      <> T.unpack new
+      <> "\" and dropping \""
+      <> T.unpack old
+      <> "\""
+  EnvelopeKeyCollision key ->
+    "the key \""
+      <> T.unpack key
+      <> "\" appears both at the top level and inside Configuration; "
+      <> "keeping the value inside Configuration"
   ConsistencyWarning msg -> msg
 
 -- | All warnings for an (unwrapped) configuration object.
+--
+-- With the parser accepting only the Version1 format (a document that is not is
+-- migrated first, which groups every component under its section), the only key
+-- warning left is for keys that none of the parsers recognise — typos, or a
+-- component property placed flat under @Configuration@ rather than under its
+-- section. There is no longer any \"shadowed\" or \"legacy single-file\" handling:
+-- a misplaced key is simply unrecognised, not resolved.
 configWarnings :: Value -> [ConfigWarning]
-configWarnings value =
-  checkUnknownKeys value <> checkShadowedKeys value <> checkLegacyFormat value
+configWarnings = checkUnknownKeys
 
--- | Top-level keys that none of the parsers recognise.
+-- | Top-level keys that none of the parsers recognise. Only the section keys, the
+-- tracing keys and the envelope annotations are recognised at the @Configuration@
+-- level; a component's own property names are recognised only under its section,
+-- so one placed flat here is reported (and ignored, not resolved).
 checkUnknownKeys :: Value -> [ConfigWarning]
 checkUnknownKeys = \case
   Object o ->
     let unknown = [K.toString k | k <- KM.keys o, K.toText k `notElem` recognisedKeys]
      in [UnrecognisedKeys unknown | not (null unknown)]
   _ -> []
-
--- | Top-level keys shadowed by a component supplied as its own section: when a
--- section key (e.g. @TestingConfig@) is present, that component's keys are read
--- from the section, so any sibling top-level key belonging to the same component
--- (e.g. a top-level @DijkstraGenesisFile@) is silently ignored.
---
--- This looks only at the keys the user wrote in this object; the per-component
--- base defaults are merged separately inside
--- 'Cardano.Configuration.File.Merge.parseSection' and never appear here, so they
--- cannot trigger it.
-checkShadowedKeys :: Value -> [ConfigWarning]
-checkShadowedKeys = \case
-  Object o ->
-    let present = map K.toText (KM.keys o)
-        shadowed =
-          [ (section, key)
-          | (section, keys) <- componentPropertyNames
-          , section `elem` present
-          , key <- keys
-          , key `elem` present
-          ]
-     in [ShadowedKeys shadowed | not (null shadowed)]
-  _ -> []
-
--- | Whether the document is in the recommended Version1 envelope form, i.e. has
--- the top-level @$schema@, @Version@, @MinNodeVersion@ and @Configuration@ keys. Operates on
--- the /raw/ top-level value (before the envelope is split off), unlike the other
--- checks here which see the unwrapped configuration object.
-checkEnvelope :: Value -> [ConfigWarning]
-checkEnvelope = \case
-  Object o ->
-    let missing =
-          [ k | k <- ["$schema", "Version", "MinNodeVersion", "Configuration"], not (KM.member (K.fromText k) o)
-          ]
-     in [NotVersion1Envelope missing | not (null missing)]
-  _ -> []
-
--- | Whether the configuration uses the legacy single-file form — i.e. any
--- component's keys appear flat at the top level rather than under its section.
-checkLegacyFormat :: Value -> [ConfigWarning]
-checkLegacyFormat = \case
-  Object o ->
-    let present = map K.toText (KM.keys o)
-        flat = [key | (_, keys) <- componentPropertyNames, key <- keys, key `elem` present]
-     in [LegacySingleFileFormat | not (null flat)]
-  _ -> []

src/Cardano/Configuration/File/Merge.hs

@@ -138,15 +138,20 @@ loadBaseDefault section = do
     then Just <$> decodeValueFile (Just section) fp
     else pure Nothing
 
--- | The configuration layer the user supplied for a section: the top-level
--- object when the section key is absent (its keys live there), an inline object,
--- or a referenced sub-file.
+-- | The configuration layer the user supplied for a section: an inline object,
+-- or a referenced sub-file. A component is read only from its own section key; a
+-- section that is absent contributes no user layer (so the component takes its
+-- base defaults). Component keys placed flat under @Configuration@ are /not/
+-- resolved into their section — they are left unrecognised (see
+-- 'Cardano.Configuration.File.Lint.checkUnknownKeys'). Non-Version1 documents,
+-- where the keys are flat, are migrated (grouped into sections) before reaching
+-- here.
 sectionUserLayer :: FilePath -> Value -> String -> IO Value
 sectionUserLayer root configValue section =
   case configValue of
     Object o ->
       case KM.lookup (K.fromString section) o of
-        Nothing -> pure configValue
+        Nothing -> pure (Object KM.empty)
         Just source -> loadSectionSource root section source
     _ ->
       throwIO $