Consolidate index metadata extraction and prepare for nameservice-based index tracking

src/fluree/db/branch.cljc

@@ -3,6 +3,7 @@
             [fluree.db.async-db :as async-db]
             [fluree.db.dbproto :as dbproto]
             [fluree.db.flake.commit-data :as commit-data]
+            [fluree.db.index.metadata :as index.meta]
             [fluree.db.indexer :as indexer]
             [fluree.db.json-ld.policy :as policy]
             [fluree.db.nameservice :as nameservice]
@@ -27,14 +28,14 @@
 
 (defn same-index?
   [commit-x commit-y]
-  (let [x-index-t (commit-data/index-t commit-x)
-        y-index-t (commit-data/index-t commit-y)]
+  (let [x-index-t (index.meta/index-t commit-x)
+        y-index-t (index.meta/index-t commit-y)]
     (= x-index-t y-index-t)))
 
 (defn newer-index?
   [commit-x commit-y]
-  (let [x-index-t (commit-data/index-t commit-x)
-        y-index-t (commit-data/index-t commit-y)]
+  (let [x-index-t (index.meta/index-t commit-x)
+        y-index-t (index.meta/index-t commit-y)]
     (and (some? x-index-t)
          (or (nil? y-index-t)
              (> x-index-t y-index-t)))))

src/fluree/db/connection.cljc

@@ -5,6 +5,7 @@
             [clojure.string :as str]
             [fluree.db.commit.storage :as commit-storage]
             [fluree.db.constants :as const]
+            [fluree.db.index.metadata :as index.meta]
             [fluree.db.indexer.garbage :as garbage]
             [fluree.db.ledger :as ledger]
             [fluree.db.nameservice :as nameservice]
@@ -479,8 +480,7 @@
   [{:keys [index-catalog] :as _conn} latest-commit]
   (go-try
     (let [storage       (:storage index-catalog)
-          index-address (some-> (util/get-first latest-commit const/iri-index)
-                                (util/get-first-value const/iri-address))]
+          {index-address :address} (index.meta/index-metadata latest-commit)]
       (when index-address
         (log/debug "Dropping index" index-address)
         (let [{:keys [spot psot opst post tspo]} (<? (storage/read-json storage index-address true))

src/fluree/db/flake/commit_data.cljc

@@ -254,36 +254,6 @@
   [commit-map]
   (-> commit-map :data :t))
 
-(defn index-t
-  "Given a commit map, returns the t value of the index (if exists)."
-  [commit-map]
-  (-> commit-map :index :data :t))
-
-(defn use-latest-index
-  "Checks if old-commit has a more current index than new-commit and
-  if so, updates new-commit to contain the updated index.
-
-  This can happen when processing a new commit while an asynchronous
-  indexing process complete giving it a newer index point than the
-  new commit"
-  [{new-index :index :as new-commit} {old-index :index :as old-commit}]
-  (if (not= (index-t new-commit)
-            (index-t old-commit))
-    (cond
-      ;; there is no old index, just return new commit
-      (nil? old-index)
-      new-commit
-
-      ;; new-index is nil but there is an old index, or old index is newer
-      (or (nil? new-index)
-          (flake/t-before? (index-t new-commit) (index-t old-commit)))
-      (assoc new-commit :index old-index)
-
-      ;; index in new-commit is newer, no changes to new commit
-      :else
-      new-commit)
-    new-commit))
-
 (defn new-db-commit
   "Returns the :data portion of the commit map for a new db commit."
   [dbid t db-address prev-data flakes size]

src/fluree/db/flake/flake_db.cljc

@@ -18,6 +18,7 @@
             [fluree.db.flake.match :as match]
             [fluree.db.flake.reasoner :as flake.reasoner]
             [fluree.db.flake.transact :as flake.transact]
+            [fluree.db.index.metadata :as index.meta]
             [fluree.db.indexer :as indexer]
             [fluree.db.json-ld.iri :as iri]
             [fluree.db.json-ld.policy :as policy]
@@ -88,7 +89,7 @@
 (defn newer-index?
   [commit {data-map :data, :as _commit-index}]
   (if data-map
-    (let [commit-index-t (commit-data/index-t commit)
+    (let [commit-index-t (index.meta/index-t commit)
           index-t        (:t data-map)]
       (or (nil? commit-index-t)
           (flake/t-after? index-t commit-index-t)))
@@ -615,7 +616,7 @@
      (let [commit-t    (-> commit-jsonld
                            (get-first const/iri-data)
                            (get-first-value const/iri-fluree-t))
-           root-map    (if-let [{:keys [address]} (:index commit-map)]
+           root-map    (if-let [{:keys [address]} (index.meta/index-metadata commit-map)]
                          (<? (index-storage/read-db-root index-catalog address))
                          (genesis-root-map ledger-alias))
            max-ns-code (-> root-map :namespace-codes iri/get-max-namespace-code)

src/fluree/db/flake/index/storage.cljc

@@ -5,6 +5,7 @@
             [fluree.db.cache :as cache]
             [fluree.db.flake :as flake]
             [fluree.db.flake.index :as index]
+            [fluree.db.index.metadata :as index.meta]
             [fluree.db.json-ld.iri :as iri]
             [fluree.db.json-ld.vocab :as vocab]
             [fluree.db.serde :as serde]
@@ -93,8 +94,7 @@
                   reindex-min-bytes reindex-max-bytes max-old-indexes]}
           db
 
-          prev-idx-t    (-> commit :index :data :t)
-          prev-idx-addr (-> commit :index :address)
+          {prev-idx-addr :address prev-idx-t :t} (index.meta/index-metadata commit)
           vg-addresses  (<? (write-vg-map index-catalog vg))
           data          (cond-> {:ledger-alias alias
                                  :t               t

src/fluree/db/index/metadata.cljc

@@ -0,0 +1,40 @@
+(ns fluree.db.index.metadata
+  (:require [fluree.db.constants :as const]
+            [fluree.db.util :as util :refer [get-first get-first-value]]))
+
+#?(:clj (set! *warn-on-reflection* true))
+
+(defn index-metadata
+  "Returns a canonical index metadata map from either an internal commit map
+  or an expanded JSON-LD commit. The result is either nil (no index) or
+  {:address <string> :t <int>}.
+
+  Accepts either:
+  - Internal commit map: {:index {:address <addr> :data {:t <int>}}}
+  - Expanded JSON-LD commit: f:index → { f:address, f:data → { f:t }}"
+  [commit-or-jsonld]
+  (let [internal-address (get-in commit-or-jsonld [:index :address])
+        internal-t       (get-in commit-or-jsonld [:index :data :t])]
+    (if (or internal-address internal-t)
+      (when internal-address
+        {:address internal-address
+         :t       internal-t})
+      (let [idx           (get-first commit-or-jsonld const/iri-index)
+            jsonld-addr   (some-> idx (get-first-value const/iri-address))
+            jsonld-data   (get-first idx const/iri-data)
+            jsonld-t      (some-> jsonld-data (get-first-value const/iri-fluree-t))]
+        (when jsonld-addr
+          {:address jsonld-addr
+           :t       jsonld-t})))))
+
+(defn index-address
+  "Returns the index address from a commit map or JSON-LD commit, or nil."
+  [commit-or-jsonld]
+  (some-> (index-metadata commit-or-jsonld) :address))
+
+(defn index-t
+  "Returns the index t from a commit map or JSON-LD commit, or nil."
+  [commit-or-jsonld]
+  (some-> (index-metadata commit-or-jsonld) :t))
+
+

src/fluree/db/indexer/garbage.cljc

@@ -1,6 +1,7 @@
 (ns fluree.db.indexer.garbage
   (:require [clojure.core.async :as async :refer [<! go]]
             [fluree.db.flake.index.storage :as storage]
+            [fluree.db.index.metadata :as index.meta]
             [fluree.db.util :as util]
             [fluree.db.util.async :refer [go-try]]
             [fluree.db.util.log :as log]))
@@ -118,5 +119,5 @@
   dependent on the frequency and size of updates that is ledger-specific
   against the ledger's 'reindex-min-bytes' setting."
   [{:keys [index-catalog commit] :as _db} max-indexes]
-  (let [index-address (-> commit :index :address)]
+  (let [{index-address :address} (index.meta/index-metadata commit)]
     (clean-garbage* index-catalog index-address max-indexes)))

src/fluree/db/ledger.cljc

@@ -6,6 +6,7 @@
             [fluree.db.did :as did]
             [fluree.db.flake :as flake]
             [fluree.db.flake.commit-data :as commit-data]
+            [fluree.db.index.metadata :as index.meta]
             [fluree.db.json-ld.credential :as credential]
             [fluree.db.json-ld.iri :as iri]
             [fluree.db.nameservice :as nameservice]
@@ -273,7 +274,8 @@
 (defn write-commit
   [commit-storage alias {:keys [did private]} commit]
   (go-try
-    (let [commit-jsonld (commit-data/->json-ld commit)
+    (let [commit-jsonld (-> (commit-data/->json-ld commit)
+                            (dissoc "index"))
           ;; For credential/generate, we need a DID map with public key
           did-map (when (and did private)
                     (if (map? did)
@@ -381,7 +383,7 @@
        (log/debug "commit!: publish-commit done" {:ledger ledger-alias})
 
        (if (track/track-txn? opts)
-         (let [index-t (commit-data/index-t commit-map)
+         (let [index-t (index.meta/index-t commit-map)
                novelty-size (get-in db* [:novelty :size] 0)
                ;; Always read threshold from realized FlakeDB; db* may be AsyncDB
                reindex-min-bytes (or (:reindex-min-bytes db) 1000000)]