Store schema v2: historic codes, complete field coverage, strict identification

.gitignore

@@ -17,3 +17,7 @@ pom.xml.asc
 .cpcache/
 /*.db
 .lsp/.cache/db.transit.json
+api-key.txt
+cache/
+*.zip
+.eastwood

CHANGELOG.md

@@ -1,5 +1,27 @@
 # CHANGELOG
 
+# Unreleased (store schema v2)
+
+Databases must be rebuilt: v1 databases (and older legacy formats) are
+rejected at open with a clear error.
+
+* Import the HISTORIC_CODES (deprecated code → live code map) and
+  VTM_INGREDIENTS files from the supplementary (bonus) distribution
+* Store all fields defined by the dm+d schema that were previously dropped,
+  including invalidity flags on every product type, previous identifiers,
+  ingredient denominator strengths, licensing authority changes, flavour,
+  colour, and lookup code history (CDDT/CDPREV)
+* New history APIs: `fetch-history`, `previous-ids`, `current-ids` map
+  deprecated identifiers to their current equivalents and vice versa
+* New `status` API returning store version, creation time, release date,
+  TRUD provenance, source file inventory and entity counts
+* Strict `open-store`: validates SQLite application_id (0x646D2B64 'dm+d')
+  and store version, with descriptive errors; read-only pooled DataSource
+* New full-text product name search via `search` (FTS5; tokenized prefix
+  matching); new `plan-products` for streaming iteration over all products;
+  `lookup-types` enumerates all lookup tables
+* Remove unused pathom3 graph API and unused dependencies
+
 # v1.0.208 - 2025-01-23
 
 * Use a SQLite DataSource rather than a Connection in case used by multiple threads concurrently

README.md

@@ -545,6 +545,24 @@ com.eldrix/dmd {:git/url "https://github.com/wardle/dmd.git"
                 :sha     "XXX"}
 ```
 
+A dm+d database file is self-describing. `dmd-database?` identifies a file as
+a dm+d database from its SQLite header (application_id 0x646D2B64, "dm+d")
+without opening it; `open-store` validates both that marker and the store
+schema version, returning a read-only pooled DataSource; and `status` returns
+the store version, dm+d release date, source TRUD provenance and entity
+counts. Each release should be built into a fresh database file; databases
+created by older versions of this library are rejected at open and should be
+rebuilt.
+
+```clojure
+(require '[com.eldrix.dmd.core :as dmd])
+(def st (dmd/open-store "dmd-2025-01-16.db"))
+(dmd/status st)                       ;; => {:version 2 :release #object[java.time.LocalDate "2025-01-16"] ...}
+(dmd/search st "amlodip" :types #{:VMP})  ;; full-text product name search
+(dmd/previous-ids st 86389004)        ;; => #{9906511000001107} - deprecated ids for a live code
+(dmd/current-ids st 9906511000001107) ;; => #{86389004} - live ids for a deprecated code
+```
+
 # Frequently asked questions
 
 ### What is the use of `dmd`?

cmd/com/eldrix/dmd/cli.clj

@@ -63,9 +63,10 @@
 (defn run-server
   [{:keys [db port] :or {port "8080"}}]
   (if db
-    (let [st (dmd/open-store db)]
+    (let [st (dmd/open-store db)
+          {:keys [version release counts]} (dmd/status st)]
       (log/info "starting server using " db " on port" port)
-      (log/info "UK dm+d release:" (.format java.time.format.DateTimeFormatter/ISO_DATE (dmd/fetch-release-date st)))
+      (log/info "UK dm+d release:" (str release) "(store version" (str version ")") counts)
       (try (serve/start-server st (Integer/parseInt port))
            (catch NumberFormatException e
              (log/error "invalid port:" port))))

deps.edn

@@ -2,8 +2,6 @@
 
  :deps    {org.clojure/clojure               {:mvn/version "1.12.0"}
            org.clojure/core.async            {:mvn/version "1.7.701"}
-           org.clojure/core.match            {:mvn/version "1.1.0"}
-           org.clojure/data.csv              {:mvn/version "1.1.0"}
            org.clojure/data.json             {:mvn/version "2.5.1"}
            org.clojure/data.xml              {:mvn/version "0.2.0-alpha9"}
            org.clojure/data.zip              {:mvn/version "1.1.0"}
@@ -14,8 +12,7 @@
            com.github.seancorfield/honeysql  {:mvn/version "2.6.1270"}
            com.github.seancorfield/next.jdbc {:mvn/version "1.3.981"}
            org.xerial/sqlite-jdbc            {:mvn/version "3.48.0.0"}
-           com.zaxxer/HikariCP               {:mvn/version "6.2.1"}
-           com.wsscode/pathom3               {:mvn/version "2025.01.16-alpha"}}
+           com.zaxxer/HikariCP               {:mvn/version "6.2.1"}}
 
  :aliases {:build
            {:deps       {io.github.clojure/tools.build          {:git/tag "v0.10.6" :git/sha "52cf7d6"}

src/com/eldrix/dmd/core.clj

@@ -40,9 +40,13 @@
          releases (dl/download-release api-key cache-dir release-date)
          _ (log/info "Downloaded dm+d releases " releases)
          unzipped (doall (map #(trud/unzip-nested (:archiveFilePath %)) releases))
-         filename' (if filename filename (str "dmd-" (.format DateTimeFormatter/ISO_LOCAL_DATE (:releaseDate (first releases))) ".db"))]
+         filename' (if filename filename (str "dmd-" (.format DateTimeFormatter/ISO_LOCAL_DATE (:releaseDate (first releases))) ".db"))
+         trud-info (mapv (fn [release]
+                           (-> (select-keys release [:itemIdentifier :id :name :releaseDate :archiveFileName])
+                               (update :releaseDate str)))
+                         releases)]
      (log/info "Creating dm+d file-based database :" filename')
-     (install-from-dirs filename' (map #(.toFile ^java.nio.file.Path %) unzipped))
+     (install-from-dirs filename' (map #(.toFile ^java.nio.file.Path %) unzipped) :trud trud-info)
      ;(zipfile/delete-paths unzipped)
      (log/info "Created dm+d file-based database :" filename'))))
 
@@ -53,28 +57,95 @@
   (install-release api-key-file cache-dir))
 
 (defn open-store
-  "Open a dm+d store. Returns what should be regarded as an opaque handle, 
-  that should be closed using `close`. Currently this is a DataSource but 
-  this is subject to change. `f` can be anything coercible to a file using 
+  "Open a dm+d store. Returns what should be regarded as an opaque handle,
+  that should be closed using `close`. Currently this is a DataSource but
+  this is subject to change. `f` can be anything coercible to a file using
   [[clojure.java.io/as-file]]. Throws an exception if the file does not exist."
   [f]
   (st4/open-store f))
 
+(defn sqlite-database?
+  "Returns true if `f` is a SQLite 3 database file."
+  [f]
+  (st4/sqlite-database? f))
+
+(defn dmd-database?
+  "Returns true if `f` is a dm+d SQLite database created by this library.
+  Strict: legacy dm+d files predating the application_id marker return false."
+  [f]
+  (st4/dmd-database? f))
+
 (defn close [st]
   (st4/close st))
 
 (defn fetch-release-date [store]
   (st4/fetch-release-date store))
 
+(defn status
+  "Returns a structured description of an open dm+d store, including store
+  schema version, creation date, dm+d release date, source TRUD release
+  information and file inventory when available, and entity counts."
+  [store]
+  (st4/status store))
+
 (defn fetch-product [store product-id]
   (st4/fetch-product store product-id))
 
 (defn fetch-product-by-exact-name [conn nm]
   (st4/fetch-product-by-exact-name conn nm))
 
+(defn search
+  "Search product names, returning a sequence of maps of :SEARCH/ID,
+  :SEARCH/TYPE and :SEARCH/NM, best matches first. Each token is matched as
+  a prefix; multiple tokens must all match.
+  Options:
+  - :types - product types to include e.g. #{:VMP :AMP}; default, all
+  - :limit - maximum number of results; default 100"
+  [conn s & {:keys [_types _limit] :as opts}]
+  (st4/search conn s opts))
+
 (defn fetch-lookup [conn lookup-kind]
   (st4/fetch-all-lookup conn lookup-kind))
 
+(def lookup-types
+  "Set of lookup types, as keywords, usable with [[fetch-lookup]]."
+  st4/lookup-types)
+
+(defn fetch-history
+  "Returns all history entries in which `id` is the current identifier,
+  ordered by start date, including 'self' entries recording the period of
+  validity of the current identifier itself."
+  [conn id]
+  (st4/fetch-history conn id))
+
+(defn previous-ids
+  "Returns the set of prior identifiers for the given current identifier."
+  [conn id]
+  (st4/previous-ids conn id))
+
+(defn current-ids
+  "Returns the set of identifiers in current use for the given (usually
+  historic) identifier."
+  [conn id]
+  (st4/current-ids conn id))
+
+(defn vtm-ingredients
+  "Returns ingredient (ISID) identifiers for the given VTM."
+  [conn vtmid]
+  (st4/vtm-ingredients conn vtmid))
+
+(defn vtms-for-ingredient
+  "Returns VTM identifiers for the given ingredient."
+  [conn isid]
+  (st4/vtms-for-ingredient conn isid))
+
+(defn plan-products
+  "Returns a reducible over all rows of the given product type (:VTM :VMP
+  :AMP :VMPP or :AMPP), for streaming iteration; each row is a `next.jdbc`
+  row abstraction with columns accessible by keyword."
+  [conn product-type]
+  (st4/plan-products conn product-type))
+
 (defn ^:deprecated vmps-from-atc
   "DEPRECATED: use [[vpids-from-atc]] instead."
   [conn atc]