From 41ea2932adead6e9118eb7ba3696670b286396a8 Mon Sep 17 00:00:00 2001 From: Esity Date: Fri, 3 Jul 2026 16:58:42 -0500 Subject: [PATCH 1/5] feat(schema): drop (conversation_id, seq) unique on llm_messages A conversation is a tree, not a line: a single parent can legitimately have two children at the same depth (e.g. a failed branch and the branch that continued), so two distinct messages can share (conversation_id, seq) and that is truth, not a duplicate. The unique constraint rejected those legitimate siblings and forced the ledger into a racy MAX(seq)+1 generation that collided under concurrent writers (PG::UniqueViolation on llm_messages_conversation_id_seq_key at scale). Message identity is already guaranteed by the unique uuid; seq becomes descriptive tree-depth with a non-unique composite index for conversation-ordered reads. SQLite emulates the constraint drop by rebuilding the table (which drops the inline uuid UNIQUE), so we re-assert the uuid unique index; on PostgreSQL DROP CONSTRAINT is surgical and it is a no-op. --- .../136_drop_llm_messages_seq_unique.rb | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb diff --git a/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb b/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb new file mode 100644 index 0000000..b7a3df5 --- /dev/null +++ b/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# The (conversation_id, seq) UNIQUE constraint modeled a conversation as a flat +# line, but a conversation is a tree: a single parent can legitimately have two +# children at the same depth (e.g. a failed branch and the branch that +# succeeded), so two distinct messages can share the same (conversation_id, seq) +# and that is TRUTH, not a duplicate. The unique constraint rejected those +# legitimate siblings and forced the ledger into a racy MAX(seq)+1 generation +# that collided under concurrent writers. +# +# Message identity is guaranteed by the unique `uuid` (producer-derived, stable +# across redelivery). seq/parent become descriptive ordering data, so we keep a +# NON-unique composite index for conversation-ordered reads. +Sequel.migration do + up do + alter_table(:llm_messages) do + drop_constraint(:llm_messages_conversation_id_seq_key, type: :unique) # rubocop:disable Legion/Llm/TaxonomyEnum + add_index %i[conversation_id seq], name: :idx_llm_messages_conversation_seq + end + # SQLite emulates the constraint drop by rebuilding the table, which drops + # the inline `uuid` UNIQUE. Re-assert it so message identity/dedup is never + # lost. On PostgreSQL DROP CONSTRAINT is surgical and the original uuid index + # already exists, so this is a no-op there. + alter_table(:llm_messages) do + add_index :uuid, unique: true, name: :idx_llm_messages_uuid_unique, if_not_exists: true + end + end + + down do + alter_table(:llm_messages) do + drop_index :uuid, name: :idx_llm_messages_uuid_unique, if_exists: true + drop_index %i[conversation_id seq], name: :idx_llm_messages_conversation_seq + add_unique_constraint %i[conversation_id seq], name: :llm_messages_conversation_id_seq_key + end + end +end From 84f83f8a7b7b31eb46e474e6c188996abd419268 Mon Sep 17 00:00:00 2001 From: Esity Date: Fri, 3 Jul 2026 17:10:46 -0500 Subject: [PATCH 2/5] chore: bump version to 1.10.6 and update CHANGELOG for migration 136 --- CHANGELOG.md | 5 +++++ lib/legion/data/version.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da202db..2719249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Legion::Data Changelog +## [1.10.6] - 2026-07-03 + +### Changed +- Migration 136: drops the `(conversation_id, seq)` UNIQUE constraint on `llm_messages`, replacing it with a NON-unique composite index. A conversation is a tree, not a line — a single parent can legitimately have two children at the same depth (e.g. a failed branch and the branch that continued), so two distinct messages can share `(conversation_id, seq)` and that is truth, not a duplicate. The unique constraint rejected legitimate siblings and forced the centralized ledger into a racy `MAX(seq)+1` generation that collided under concurrent writers (`PG::UniqueViolation` on `llm_messages_conversation_id_seq_key`). Message identity remains guaranteed by the unique `uuid`; `seq` becomes descriptive tree-depth. On SQLite the constraint drop rebuilds the table (dropping the inline `uuid` UNIQUE), so the migration re-asserts the uuid unique index — a no-op on PostgreSQL where `DROP CONSTRAINT` is surgical. + ## [1.10.5] - 2026-06-16 ### Added diff --git a/lib/legion/data/version.rb b/lib/legion/data/version.rb index fcff62b..0e29ec5 100755 --- a/lib/legion/data/version.rb +++ b/lib/legion/data/version.rb @@ -2,6 +2,6 @@ module Legion module Data - VERSION = '1.10.5' + VERSION = '1.10.6' end end From a22c409a2bdc9437a8d9a606c32f66063ffdcb5c Mon Sep 17 00:00:00 2001 From: Esity Date: Fri, 3 Jul 2026 17:12:21 -0500 Subject: [PATCH 3/5] fixing rubocop errors --- lib/legion/data/settings.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/legion/data/settings.rb b/lib/legion/data/settings.rb index 8795428..5327589 100755 --- a/lib/legion/data/settings.rb +++ b/lib/legion/data/settings.rb @@ -41,8 +41,8 @@ def self.default name: nil, # Logging - log: false, - query_log: false, + log: true, + query_log: true, log_connection_info: false, log_warn_duration: 1, sql_log_level: 'debug', From d854830aa21efc8e76399e8b9ae0be39c3e7e888 Mon Sep 17 00:00:00 2001 From: Esity Date: Fri, 3 Jul 2026 17:34:54 -0500 Subject: [PATCH 4/5] fix: restore log/query_log defaults to false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The debug toggle (log: true, query_log: true) had leaked into committed defaults. With query_log true it wins over log in Connection#setup, installing QueryFileLogger instead of SlowQueryLogger and skipping log_warn_duration — breaking connection_spec. Restore both to false (opt-in only). --- lib/legion/data/settings.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/legion/data/settings.rb b/lib/legion/data/settings.rb index 5327589..8795428 100755 --- a/lib/legion/data/settings.rb +++ b/lib/legion/data/settings.rb @@ -41,8 +41,8 @@ def self.default name: nil, # Logging - log: true, - query_log: true, + log: false, + query_log: false, log_connection_info: false, log_warn_duration: 1, sql_log_level: 'debug', From 65495bf097d433d62b564695a0ddb50376a6e0ae Mon Sep 17 00:00:00 2001 From: Esity Date: Fri, 3 Jul 2026 17:37:54 -0500 Subject: [PATCH 5/5] style: fix rubocop offenses across legion-data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RescueLogLevel: :debug -> :warn in rescue handle_exception calls (data.rb, extract/handlers/base.rb, models/function.rb, models/node.rb) - NoLoopDo: archiver batch loop uses bounded `while` instead of `loop do` - NoUnderscorePrefixedKwargs / UnusedMethodArgument: unused **_opts -> ** (connection_replicas_spec, tls_spec) - TaxonomyEnum: exclude migrations and specs — the cop validates LLM lane taxonomy (:tier/:type/:circuit_state) but `type:` in those paths is a Sequel column type (foreign_key type: :uuid) or the extract API, not LLM taxonomy. Removes the now-redundant inline disable in migration 136. --- .rubocop.yml | 7 +++++++ lib/legion/data.rb | 2 +- lib/legion/data/archiver.rb | 21 ++++++++----------- lib/legion/data/extract/handlers/base.rb | 2 +- .../136_drop_llm_messages_seq_unique.rb | 2 +- lib/legion/data/models/function.rb | 2 +- lib/legion/data/models/node.rb | 4 ++-- spec/legion/data/connection_replicas_spec.rb | 2 +- spec/legion/data/tls_spec.rb | 4 ++-- 9 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 5de70dc..52143a4 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -23,6 +23,13 @@ Naming/VariableNumber: - lib/legion/data/connection.rb Legion/Framework/EagerSequelModel: Enabled: false +# TaxonomyEnum validates LLM lane taxonomy (:tier/:type/:circuit_state). In +# these paths `type:` is a Sequel column type (e.g. foreign_key type: :uuid) or +# the extract API (type: :auto/:text) — not LLM taxonomy — so the cop misfires. +Legion/Llm/TaxonomyEnum: + Exclude: + - 'lib/legion/data/migrations/**/*' + - 'spec/**/*' Metrics/BlockLength: Max: 100 Exclude: diff --git a/lib/legion/data.rb b/lib/legion/data.rb index 2e09dab..d2f615b 100755 --- a/lib/legion/data.rb +++ b/lib/legion/data.rb @@ -112,7 +112,7 @@ def stats def connected? Legion::Settings[:data][:connected] == true rescue StandardError => e - handle_exception(e, level: :debug, handled: true, operation: :connected?) + handle_exception(e, level: :warn, handled: true, operation: :connected?) false end diff --git a/lib/legion/data/archiver.rb b/lib/legion/data/archiver.rb index 5793d28..c2c64dd 100644 --- a/lib/legion/data/archiver.rb +++ b/lib/legion/data/archiver.rb @@ -97,18 +97,15 @@ def archive_batches(conn:, table:, cutoff:, batch_size:, storage_backend:) total_rows = 0 paths = [] - loop do - batch_result = archive_batch( - conn: conn, - table: table, - cutoff: cutoff, - batch_size: batch_size, - batch_n: batches + 1, - now: now, - storage_backend: storage_backend - ) - break unless batch_result - + while (batch_result = archive_batch( + conn: conn, + table: table, + cutoff: cutoff, + batch_size: batch_size, + batch_n: batches + 1, + now: now, + storage_backend: storage_backend + )) batches += 1 total_rows += batch_result[:row_count] paths << batch_result[:path] diff --git a/lib/legion/data/extract/handlers/base.rb b/lib/legion/data/extract/handlers/base.rb index 3417ba4..46ae3d1 100644 --- a/lib/legion/data/extract/handlers/base.rb +++ b/lib/legion/data/extract/handlers/base.rb @@ -53,7 +53,7 @@ def available? require gem_name true rescue LoadError => e - handle_exception(e, level: :debug, handled: true, operation: :extract_handler_available, handler: name, gem: gem_name) + handle_exception(e, level: :warn, handled: true, operation: :extract_handler_available, handler: name, gem: gem_name) false end end diff --git a/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb b/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb index b7a3df5..3c7e946 100644 --- a/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb +++ b/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb @@ -14,7 +14,7 @@ Sequel.migration do up do alter_table(:llm_messages) do - drop_constraint(:llm_messages_conversation_id_seq_key, type: :unique) # rubocop:disable Legion/Llm/TaxonomyEnum + drop_constraint(:llm_messages_conversation_id_seq_key, type: :unique) add_index %i[conversation_id seq], name: :idx_llm_messages_conversation_seq end # SQLite emulates the constraint drop by rebuilding the table, which drops diff --git a/lib/legion/data/models/function.rb b/lib/legion/data/models/function.rb index da35b62..c038a21 100755 --- a/lib/legion/data/models/function.rb +++ b/lib/legion/data/models/function.rb @@ -18,7 +18,7 @@ def embedding_vector ::JSON.parse(embedding) rescue ::JSON::ParserError => e - handle_exception(e, level: :debug, handled: true, operation: :embedding_vector, id: self[:id]) + handle_exception(e, level: :warn, handled: true, operation: :embedding_vector, id: self[:id]) nil end diff --git a/lib/legion/data/models/node.rb b/lib/legion/data/models/node.rb index 20093f4..01de384 100755 --- a/lib/legion/data/models/node.rb +++ b/lib/legion/data/models/node.rb @@ -17,7 +17,7 @@ def parsed_metrics Legion::JSON.load(metrics) rescue StandardError => e - handle_exception(e, level: :debug, handled: true, operation: :parsed_metrics, id: self[:id]) + handle_exception(e, level: :warn, handled: true, operation: :parsed_metrics, id: self[:id]) nil end @@ -26,7 +26,7 @@ def parsed_hosted_worker_ids Legion::JSON.load(hosted_worker_ids) rescue StandardError => e - handle_exception(e, level: :debug, handled: true, operation: :parsed_hosted_worker_ids, id: self[:id]) + handle_exception(e, level: :warn, handled: true, operation: :parsed_hosted_worker_ids, id: self[:id]) [] end end diff --git a/spec/legion/data/connection_replicas_spec.rb b/spec/legion/data/connection_replicas_spec.rb index 3a9a007..ea7b0be 100644 --- a/spec/legion/data/connection_replicas_spec.rb +++ b/spec/legion/data/connection_replicas_spec.rb @@ -30,7 +30,7 @@ end # Build a minimal Sequel::Database double with the methods we call. - def fake_sequel_db(**_opts) + def fake_sequel_db(**) db = instance_double(Sequel::Database) allow(db).to receive(:extension) allow(db).to receive(:add_servers) diff --git a/spec/legion/data/tls_spec.rb b/spec/legion/data/tls_spec.rb index df8c64e..ffaf56f 100644 --- a/spec/legion/data/tls_spec.rb +++ b/spec/legion/data/tls_spec.rb @@ -9,7 +9,7 @@ before do stub_const('Legion::Crypt::TLS', Module.new do - def self.resolve(config, **_opts) + def self.resolve(config, **) if config[:enabled] { enabled: true, verify: :peer, ca: '/etc/ssl/ca.pem', cert: nil, key: nil } else @@ -69,7 +69,7 @@ def self.resolve(config, **_opts) ) stub_const('Legion::Crypt::TLS', Module.new do - def self.resolve(_config, **_opts) + def self.resolve(_config, **) { enabled: true, verify: :none, ca: nil, cert: nil, key: nil } end end)