diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95686406..ed00e2e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,13 +67,13 @@ jobs: fi } - set_output canister '^(Cargo\.lock|crates/(vfs_canister|vfs_runtime|vfs_types|vfs_store|wiki_domain)/|docs/DB_LIFECYCLE\.md|scripts/build-vfs-canister\.sh)' + set_output canister '^(Cargo\.lock|crates/(vfs_canister|vfs_runtime|vfs_types|vfs_store|wiki_domain)/|docs/DB_LIFECYCLE\.md|scripts/(build-vfs-canister|check-build-vfs-canister-guard)\.(sh|mjs))' set_output rust_all '^(Cargo\.toml|crates/(vfs_canister|vfs_runtime|vfs_types|vfs_cli_app|vfs_cli_core|vfs_client|vfs_store|wiki_domain|ic_sqlite_vfs_probe)/|scripts/(kinic_vfs_cli_release_version|package_kinic_vfs_cli)\.sh)' set_output cli '^(crates/(vfs_cli_app|vfs_cli_core|vfs_client)/|docs/(CLI|PUBLIC_SMOKE)\.md|scripts/(local|smoke|mainnet|kinic_vfs_cli_release_version|package_kinic_vfs_cli))' set_output wikibrowser '^(wikibrowser/|crates/vfs_canister/vfs\.did)' set_output extension '^(extensions/wiki-clipper/|crates/vfs_canister/vfs\.did)' - set_output wiki_generator '^workers/wiki-generator/' - set_output skill_registry_web '^(skill-registry-web/|wikibrowser/app/skills/|wikibrowser/scripts/check-skill-registry\.mjs|skills/)' + set_output wiki_generator '^(workers/wiki-generator/|crates/vfs_canister/vfs\.did)' + set_output skill_registry_web '^(skill-registry-web/|wikibrowser/app/skills/|wikibrowser/scripts/check-skill-registry\.mjs|skills/|crates/vfs_canister/vfs\.did)' regression-groups-check: runs-on: ubuntu-latest @@ -267,6 +267,8 @@ jobs: run: | node scripts/build-icp-cli-login.mjs git diff --exit-code crates/vfs_canister/src/icp_cli_login.html + - name: Check build environment guard + run: node scripts/check-build-vfs-canister-guard.mjs - name: Cache cargo registry uses: actions/cache@v5 with: diff --git a/Cargo.lock b/Cargo.lock index 86014c95..254167d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3522,6 +3522,7 @@ dependencies = [ "kinic-wiki-domain", "proptest", "rusqlite", + "serde_json", "sha2 0.10.9", "tempfile", "vfs-store", diff --git a/MARKETPLACE_PLAN.md b/MARKETPLACE_PLAN.md new file mode 100644 index 00000000..d5918b6e --- /dev/null +++ b/MARKETPLACE_PLAN.md @@ -0,0 +1,493 @@ +# Marketplace Plan + +Kinic Wiki に DB 閲覧権マーケットプレイスを追加するための計画。 + +目的は、複雑な金融商品ではなく「価値ある private DB を、購入者だけが読める」状態を最小実装で作ること。 + +## 結論 + +最初は「DB 全体の永続 Reader 相当閲覧権」を販売する。 + +- 売るもの: DB の閲覧権 +- 売らないもの: Writer 権、Owner 権、DB 所有権、二次流通権 +- 購入前に見せるもの: title、description、LLM summary、verified stats、sample excerpts、category graph、最終更新日、tags、price +- 購入後に許可するもの: read、list、search、graph +- 権限正本: `database_members` ではなく `market_entitlements` +- API 名: `market_` prefix +- 購入権: 永続。active 判定は `status = 'active'` のみ。 +- DB delete/archive: owner 権限を維持し、禁止しない。UI/CLI で購入者影響を警告する。 +- 決済: ユーザー別 KINIC balance を正本にし、購入時は外部 ledger を呼ばない。 +- cashout: MVP では実装しない。seller 売上は内部 balance までに留める。 + +## なぜこの形にするか + +`database_members` に reader を直接追加すると、owner が後から revoke できる。 +購入者から見ると「買った権利が消える」ため、販売契約として弱い。 + +購入権は `market_entitlements` に分離する。 +通常 ACL と購入権を分けることで、手動 grant、owner 管理、paid access、期限管理を混ぜずに扱える。 + +決済は buyer から seller への direct transfer ではなく、deposit 済み残高で処理する。 +閲覧権購入時に ledger call を行うと、ledger await と entitlement 付与が同じ update に混在し、曖昧結果、重複実行、local apply failure の復旧分岐が増える。 +購入は canister 内の SQLite transaction だけで buyer debit、seller credit、entitlement grant を確定する。 + +## 決済前提 + +- deposit は `approve + icrc2_transfer_from` で buyer account から canister account へ KINIC を移す。 +- 閲覧権購入時は外部 ledger を呼ばず、内部 balance transaction だけで確定する。 +- deposit の ledger 曖昧結果は `kinic_pending_operations` で扱う。 + +## MVP Scope + +MVP で作るもの: + +- DB owner が listing を作成できる。 +- listing は DB 全体に紐付く。 +- listing は公開 metadata を持つ。 +- buyer は KINIC を deposit して内部 balance を持つ。 +- buyer は内部 balance で閲覧権を購入する。 +- purchase 成功後、buyer に active entitlement を付与する。 +- seller 売上は seller の内部 balance に credit する。 +- read 系認可は `database_members` または active `market_entitlements` を許可する。 +- active entitlement が残る DB の delete/archive は許可し、UI/CLI で警告する。 +- listing 検索は本文検索と分離する。 +- delete/archive 前に active entitlement count を返す query を提供する。 + +MVP で作らないもの: + +- Writer / Owner 権限販売 +- prefix 単位販売 +- snapshot 固定販売 +- platform fee split +- refund +- resale +- ZKP +- 購入者ごとの LLM 要約生成 +- 複数 token +- seller withdraw +- kinic pending operation repair/cancel + +## Buyer Experience + +購入者は Marketplace 一覧から listing を探す。 +詳細画面では本文全体ではなく、購入判断材料だけを見る。 + +表示するもの: + +- title +- description +- LLM summary +- sample excerpts +- topics / tags +- node count +- logical size +- last updated +- seller +- price + +購入前に balance が不足する場合、deposit 画面へ誘導する。 +購入後は My purchases に DB が表示され、通常の WikiBrowser と同じ read/search/graph UI で読む。 + +## Seller Experience + +seller は既存 DB 管理画面から「Sell this DB」を押す。 +公開情報を入力し、必要なら LLM summary を生成する。 + +seller が入力するもの: + +- title +- description +- tags +- price +- sample excerpts + +LLM summary は任意。 +生成する場合も listing publish 時だけ実行し、閲覧ごとには生成しない。 +summary には `summary_snapshot_revision` を持たせる。 +DB 更新で listing snapshot と current snapshot がズレた場合、購入は止めずに stale 警告を表示する。 + +## Value Signals + +DB 本文は購入前に全面公開しない。 +価値証明は暗号ではなく、購入判断材料として提示する。 + +使う signal: + +- LLM summary +- seller description +- sample excerpts +- verified stats +- top-level contents sample +- category graph +- prefix stats +- last updated +- provenance +- purchase count +- report count + +ZKP は使わない。 +「内容が有益」という性質は暗号で証明しにくい。 +最初は preview、sample、統計、評判で判断させる。 + +## Entitlement Model + +`market_entitlements` を追加する。 + +```sql +CREATE TABLE market_entitlements ( + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + listing_id TEXT NOT NULL, + order_id TEXT NOT NULL, + purchased_at_ms INTEGER NOT NULL, + status TEXT NOT NULL, + PRIMARY KEY (database_id, buyer_principal, listing_id) +); +``` + +DB 全体閲覧権販売なので、同じ DB を relist しても二重購入させない。 + +```sql +CREATE UNIQUE INDEX market_entitlements_database_buyer_idx + ON market_entitlements(database_id, buyer_principal) + WHERE status = 'active'; +``` + +認可判定: + +```text +can_read = database_members has reader/writer/owner + OR market_entitlements has active entitlement +``` + +Writer 系 update は entitlement では許可しない。 +owner revoke は entitlement に影響しない。 +`list_database_members`、cycles history、Owner/Writer update は entitlement を見ない。 + +実装では `load_member_role` と `role_allows` を変更しない。 +`require_database_read_access` などの新規 helper を作り、member role が通る場合は許可し、明示した read surface だけ active entitlement を許可する。 + +active entitlement 判定: + +```sql +status = 'active' +``` + +anonymous principal は entitlement 対象外にする。 + +購入済み entitlement で許可する read surface は以下に限定する。 + +- `read_node` +- `list_children` +- `list_nodes` +- `search_nodes` +- `search_node_paths` +- `incoming_links` +- `outgoing_links` +- `graph_links` +- `graph_neighborhood` +- `read_node_context` + +`export_snapshot`、`fetch_updates`、source generation、URL ingest、write/update 系、member/cycles 管理系は entitlement では許可しない。 + +## Payment Model + +KINIC は canister 内部 KINIC balance に deposit してから使う。 + +購入時に外部 ledger は呼ばない。 +購入 transaction は index DB だけを更新する。 + +MVP invariant: + +- deposit は既存 cycles purchase と同じく `approve + icrc2_transfer_from` で canister account に入れる。 +- deposit の ledger await は購入処理と分離する。 +- seller 売上の withdraw は実装しない。 + +deposit は buyer から canister default account への `transfer_from` として扱う。 +既に `database_cycle_pending_operations` が ledger await と曖昧結果を扱う設計を持つため、KINIC deposit も同じ pattern の pending table を持つ。 + +### Deposit Flow + +1. wallet が KINIC ledger で canister を spender に `icrc2_approve` する。 +2. buyer が `kinic_deposit_balance(amount_e8s, expected_fee_e8s)` を呼ぶ。 +3. canister が `kinic_pending_operations` に `deposit` を作成する。 +4. canister が `icrc2_transfer_from` で buyer account から canister account へ KINIC を移す。 +5. ledger success または `Duplicate` success なら buyer internal balance に credit する。 +6. `kinic_ledger` に `deposit` を記録し、pending operation を削除する。 + +同一 caller の deposit は guard で直列化する。 +ledger 明示 error では pending operation を削除し、内部 balance は更新しない。 +ledger 結果曖昧、または ledger success 後 local apply 前に失敗した場合は pending operation を残す。 +MVP では deposit repair/cancel API は作らず、既存 cycles purchase と同じく caller と billing authority が pending 状態を確認して運用する。 + +### Internal Balance + +`kinic_accounts` を追加する。 +これは marketplace 専用ではなく、将来ほかの KINIC 利用でも参照できる canister 内部 account である。 + +```sql +CREATE TABLE kinic_accounts ( + principal TEXT PRIMARY KEY, + balance_e8s INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); +``` + +`kinic_ledger` を内部 balance の正本履歴にする。 +履歴の `source` で marketplace 由来か別用途由来かを区別する。 + +```sql +CREATE TABLE kinic_ledger ( + entry_id INTEGER PRIMARY KEY AUTOINCREMENT, + principal TEXT NOT NULL, + source TEXT NOT NULL, + kind TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + balance_after_e8s INTEGER NOT NULL, + counterparty TEXT, + listing_id TEXT, + order_id TEXT, + external_block_index INTEGER, + created_at_ms INTEGER NOT NULL +); +``` + +`kind` は MVP では `deposit`, `purchase`, `sale` だけにする。 + +### Purchase Flow + +`market_purchase_access(listing_id, price_e8s)` は ledger を呼ばない。 + +1. caller 認証。anonymous は拒否。 +2. listing が active で、price が一致することを確認。 +3. buyer balance が price 以上であることを確認。 +4. buyer から price を debit。 +5. seller に price を credit。 +6. `market_orders` と buyer/seller の `kinic_ledger` を記録。 +7. `market_entitlements` を付与。 + +全手順を 1 SQLite transaction で行う。 +同じ buyer が同じ listing を再購入した場合は拒否する。 + +### Future Withdraw + +MVP では withdraw を実装しない。 +理由は、canister から外部 ledger へ outgoing transfer する場合、fee、ambiguous transfer repair、同一 TransferArg retry が同時に必要になるためである。 + +withdraw は別 PR とし、fee、created_at_time、`Duplicate` 成功扱い、同一 TransferArg retry、repair/cancel API を同時に設計する。 + +## Listing Model + +`market_listings` を追加する。 + +```sql +CREATE TABLE market_listings ( + listing_id TEXT PRIMARY KEY, + seller_principal TEXT NOT NULL, + database_id TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + llm_summary TEXT, + summary_snapshot_revision TEXT, + sample_excerpts_json TEXT NOT NULL, + tags_json TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + status TEXT NOT NULL, + revision INTEGER NOT NULL, + purchase_count INTEGER NOT NULL, + report_count INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); +``` + +制約: + +- seller は対象 DB の owner だけ。 +- `price_e8s > 0`。 +- `status` は `draft`, `active`, `paused`。 +- active listing の公開 metadata / price 変更は revision を進める。 + +## Order Model + +`market_orders` を追加する。 + +```sql +CREATE TABLE market_orders ( + order_id TEXT PRIMARY KEY, + listing_id TEXT NOT NULL, + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + seller_principal TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + listing_revision INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL +); +``` + +`order_id` は canister 側生成にする。 +購入 API が ledger を呼ばないため、重複購入は entitlement existence で判定する。 + +## Pending Operations + +MVP では `kinic_pending_operations` を deposit の曖昧結果に使う。 +purchase は外部 ledger を呼ばないため pending operation を持たない。 + +構造は既存 `database_cycle_pending_operations` と同じ考え方にする。 +既存 table は DB cycles 専用であり、KINIC 内部 balance には流用しない。 + +```sql +CREATE TABLE kinic_pending_operations ( + operation_id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + caller TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + from_owner TEXT, + from_subaccount BLOB, + to_owner TEXT, + to_subaccount BLOB, + ledger_fee_e8s INTEGER, + operation_status TEXT NOT NULL, + external_block_index INTEGER, + ledger_created_at_time_ns INTEGER, + created_at_ms INTEGER NOT NULL +); +``` + +`operation_status` は `in_flight`, `ambiguous`, `completed`。 +MVP では repair/cancel method は作らず、caller と billing authority が pending 状態を確認できる query を用意する。 +withdraw PR では repair/cancel method も同時に実装する。 + +## API + +内部 KINIC balance は `kinic_` prefix、listing / order / entitlement は `market_` prefix を付ける。 + +MVP: + +- `kinic_get_balance` +- `kinic_deposit_balance` +- `kinic_fund_database_cycles` +- `market_create_listing` +- `market_update_listing` +- `market_publish_listing` +- `market_pause_listing` +- `market_list_listings` +- `market_list_database_listings` +- `market_get_listing` +- `market_preview_purchase` +- `market_purchase_access` +- `market_list_entitlements` +- `market_list_orders` +- `market_count_active_entitlements` +- `kinic_list_pending_operations` + +`kinic_list_pending_operations` は `kinic_pending_operations` のうち KINIC deposit に関係する pending だけを返す。 + +MVP では作らない: + +- `kinic_withdraw_balance` +- `kinic_repair_withdraw_complete` +- `kinic_repair_withdraw_cancel` +- `market_refund_purchase` + +後続候補: + +- `market_list_seller_listings` +- `market_report_listing` +- `market_update_summary` +- platform fee 設定 + +## UI + +最初の画面は 4 つにする。 + +- Marketplace: listing 一覧 +- Listing detail: preview と purchase +- My purchases: 購入済み DB 一覧 +- KINIC balance: deposit、balance、履歴 + +seller 操作は既存 DB 管理画面に追加する。 + +- Sell this DB +- Edit listing +- Publish listing +- Pause listing +- Delete/archive warning + +## Delete And Archive Rule + +active entitlement がある DB でも owner は delete/archive できる。 + +理由: + +- 著作権と DB 管理権は作者・owner 側に残す。 +- marketplace entitlement は Reader 相当の閲覧権であり、owner 権限を奪わない。 +- refund や escrow は MVP に入れない。 + +UI/CLI は delete/archive 前に active entitlement 数を表示し、購入者が影響を受けることを警告する。 +canister API は既存 owner 権限を維持する。 + +DB delete 後は listing と entitlement rows を削除する。 +market_orders は購入履歴・監査ログとして残す。 + +Marketplace write operation は DB storage metering ではなく KINIC internal account と listing metadata の管理面 write として扱い、`with_unmetered_update` のままにする。 +listing metadata は validator のサイズ上限を持ち、seller は DB owner に限定する。 + +## Abuse Controls + +MVP は DB owner が listing を作成できる。 +seller 制限 table は作らない。 + +必要な後続管理操作: + +- listing pause +- abusive listing delist +- report count display + +## Open Questions + +- price は seller 入力か、初期は固定価格か。 +- purchase count と report count を公開するか。 +- LLM summary generation は Worker 経由か、seller 手書きだけで始めるか。 +- active entitlement が残る DB の rename は許可するか。 +- seller cashout を後続で入れる場合の outgoing transfer repair 方式。 +- seller cashout を入れる前に seller 内部 balance を何に使えるようにするか。 + +## Implementation Order + +1. `market_` public types を `vfs_types` に追加する。 +2. index schema に `kinic_accounts`、`kinic_ledger`、`kinic_pending_operations`、listing、order、entitlement を追加する。 +3. canister runtime に deposit transfer_from、balance credit、listing CRUD を追加する。 +4. purchase なしで entitlement seed helper を作り、read surface の認可テストを追加する。 +5. read/list/search/graph 認可に entitlement 判定を追加する。 +6. market purchase transaction を追加する。 +7. delete/archive の UI/CLI warning を追加する。canister guard は追加しない。 +8. CLI に最小 market commands を追加する。 +9. WikiBrowser に Marketplace、Listing detail、My purchases、KINIC balance を追加する。 +10. IDL generator、hand-written TS IDL、Rust client、candid shape tests を同期する。 +11. unit tests と e2e smoke を追加する。 + +## Verification + +最低限の検証: + +- DB owner は listing 作成可能。 +- DB owner 以外は listing 作成不可。 +- listing publish 後、anonymous は preview だけ読める。 +- 未購入 buyer は DB 本文を読めない。 +- `kinic_deposit_balance` は pending operation を作り、ledger success 後だけ buyer balance を credit する。 +- ledger `Duplicate` は同一 pending operation の success として扱う。 +- deposit の ambiguous / completed pending は repair API なしで caller と billing authority が確認する。 +- purchase は外部 ledger を呼ばず、buyer balance を debit し seller balance を credit する。 +- purchase 成功後、buyer は read/list/search/graph できる。 +- purchase entitlement では write できない。 +- owner revoke は entitlement に影響しない。 +- 永続 entitlement は期限切れしない。 +- DB 更新後、listing は stale 警告を表示するが購入可能。 +- sample excerpt は検証済み listing metadata から返り、DB 本文 API を匿名に開かない。 +- category graph は top-level category のみ返し、node path、link text、raw href は返さない。 +- `export_snapshot` と `fetch_updates` は entitlement では拒否される。 +- seller が対象 DB owner であることは create/update/publish/purchase 時に再確認される。 +- active entitlement があっても owner は delete/archive できる。 +- UI/CLI は delete/archive 前に購入者影響を警告する。 diff --git a/README.md b/README.md index e283a477..6484ac81 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ https://wiki.kinic.xyz The official Kinic Wiki database is: -https://wiki.kinic.xyz/db_kva4v2twg6jv/Wiki +https://wiki.kinic.xyz/db/db_kva4v2twg6jv/Wiki Database ID: diff --git a/crates/vfs_canister/build.rs b/crates/vfs_canister/build.rs new file mode 100644 index 00000000..a8d07f0d --- /dev/null +++ b/crates/vfs_canister/build.rs @@ -0,0 +1,6 @@ +// Where: crates/vfs_canister/build.rs +// What: Rebuild the canister when local II origin compilation changes. +// Why: The certified ii-alternative-origins body is selected at compile time. +fn main() { + println!("cargo:rerun-if-env-changed=KINIC_VFS_LOCAL_II_ORIGINS"); +} diff --git a/crates/vfs_canister/src/icp_cli_login.html b/crates/vfs_canister/src/icp_cli_login.html index eca15c90..f3ab70dd 100644 --- a/crates/vfs_canister/src/icp_cli_login.html +++ b/crates/vfs_canister/src/icp_cli_login.html @@ -83,7 +83,7 @@

CLI login

diff --git a/crates/vfs_canister/src/lib.rs b/crates/vfs_canister/src/lib.rs index 2916e432..a7be9a07 100644 --- a/crates/vfs_canister/src/lib.rs +++ b/crates/vfs_canister/src/lib.rs @@ -36,7 +36,7 @@ use ic_stable_structures::memory_manager::{MemoryId, MemoryManager}; use vfs_runtime::STORAGE_BILLING_INTERVAL_MS; use vfs_runtime::{ CyclesPendingLedgerDetailsInput, DatabaseCyclesPurchaseWithLedgerDetails, DatabaseMeta, - RequiredRole, VfsService, cycles_for_payment_amount_e8s, + KinicDepositWithLedgerDetails, RequiredRole, VfsService, cycles_for_payment_amount_e8s, }; use vfs_types::{ AppendNodeRequest, CanisterHealth, CanonicalRole, ChildNode, CreateDatabaseRequest, @@ -47,17 +47,21 @@ use vfs_types::{ DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, GlobNodeHit, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, IncomingLinksRequest, - IndexSqlJsonQueryResult, KINIC_DECIMALS, KINIC_LEDGER_FEE_E8S, LinkEdge, ListChildrenRequest, - ListNodesRequest, MemoryCapability, MemoryManifest, MemoryRoot, MkdirNodeRequest, - MkdirNodeResult, MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, - Node, NodeContext, NodeContextRequest, NodeEntry, OpsAnswerSessionCheckRequest, - OpsAnswerSessionCheckResult, OpsAnswerSessionRequest, OutgoingLinksRequest, QueryContext, - QueryContextRequest, RenameDatabaseRequest, SearchNodeHit, SearchNodePathsRequest, - SearchNodesRequest, SourceEvidence, SourceEvidenceRequest, SourceRunSessionCheckRequest, - Status, StorageBillingBatchRequest, StorageBillingBatchResult, - UrlIngestTriggerSessionCheckRequest, UrlIngestTriggerSessionRequest, WriteNodeRequest, - WriteNodeResult, WriteNodesRequest, WriteSourceForGenerationRequest, - WriteSourceForGenerationResult, kinic_base_units_per_token, + IndexSqlJsonQueryResult, KINIC_DECIMALS, KINIC_LEDGER_FEE_E8S, KinicBalance, + KinicDepositRequest, KinicDepositResult, KinicFundDatabaseCyclesRequest, + KinicFundDatabaseCyclesResult, KinicPendingOperationsPage, KinicPendingOperationsPageRequest, + LinkEdge, ListChildrenRequest, ListNodesRequest, MarketCreateListingRequest, + MarketEntitlementPage, MarketListing, MarketListingDetail, MarketListingPage, MarketOrder, + MarketOrderPage, MarketPurchasePreview, MarketPurchaseRequest, MarketUpdateListingRequest, + MemoryCapability, MemoryManifest, MemoryRoot, MkdirNodeRequest, MkdirNodeResult, + MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeContext, + NodeContextRequest, NodeEntry, OpsAnswerSessionCheckRequest, OpsAnswerSessionCheckResult, + OpsAnswerSessionRequest, OutgoingLinksRequest, QueryContext, QueryContextRequest, + RenameDatabaseRequest, SearchNodeHit, SearchNodePathsRequest, SearchNodesRequest, + SourceEvidence, SourceEvidenceRequest, SourceRunSessionCheckRequest, Status, + StorageBillingBatchRequest, StorageBillingBatchResult, UrlIngestTriggerSessionCheckRequest, + UrlIngestTriggerSessionRequest, WriteNodeRequest, WriteNodeResult, WriteNodesRequest, + WriteSourceForGenerationRequest, WriteSourceForGenerationResult, kinic_base_units_per_token, }; #[cfg(not(target_arch = "wasm32"))] @@ -65,7 +69,8 @@ const INDEX_DB_PATH: &str = "./DB/index.sqlite3"; #[cfg(not(target_arch = "wasm32"))] const DATABASES_DIR: &str = "./DB/databases"; const II_ALTERNATIVE_ORIGINS_PATH: &str = "/.well-known/ii-alternative-origins"; -const II_ALTERNATIVE_ORIGINS_BODY: &str = r#"{"alternativeOrigins":["https://wiki.kinic.xyz","https://kinic.xyz","chrome-extension://jcfniiflikojmbfnaoamlbbddlikchaj","chrome-extension://hbnicbmdodpmihmcnfgejcdgbfmemoci","chrome-extension://moebdnadaffhlddnhifmmdoecifhcbdi"]}"#; +const II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY: &str = r#"{"alternativeOrigins":["https://wiki.kinic.xyz","https://kinic.xyz","chrome-extension://jcfniiflikojmbfnaoamlbbddlikchaj","chrome-extension://hbnicbmdodpmihmcnfgejcdgbfmemoci","chrome-extension://moebdnadaffhlddnhifmmdoecifhcbdi"]}"#; +const II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY: &str = r#"{"alternativeOrigins":["https://wiki.kinic.xyz","https://kinic.xyz","chrome-extension://jcfniiflikojmbfnaoamlbbddlikchaj","chrome-extension://hbnicbmdodpmihmcnfgejcdgbfmemoci","chrome-extension://moebdnadaffhlddnhifmmdoecifhcbdi","http://localhost:3000","http://127.0.0.1:3010","http://localhost:3010","http://127.0.0.1:3100","http://localhost:3100"]}"#; const ICP_CLI_LOGIN_DISCOVERY_PATH: &str = "/.well-known/ic-cli-login"; const ICP_CLI_LOGIN_PATH: &str = "/login"; const ICP_CLI_LOGIN_HTML: &str = include_str!("icp_cli_login.html"); @@ -434,49 +439,79 @@ fn icrc10_supported_standards() -> Vec { fn icrc21_canister_call_consent_message( request: Icrc21ConsentMessageRequest, ) -> Icrc21ConsentMessageResponse { - if request.method != "purchase_database_cycles" { - return icrc21_unsupported(format!("unsupported canister call: {}", request.method)); - } - let purchase = match Decode!(&request.arg, DatabaseCyclesPurchaseRequest) { - Ok(decoded) => decoded, - Err(error) => { - return icrc21_unavailable(format!( - "purchase_database_cycles argument decode failed: {error}" - )); - } - }; - let cycles = match with_service(|service| { - let config = service.cycles_billing_config()?; - let cycles = cycles_for_payment_amount_e8s(purchase.payment_amount_e8s, &config)?; - service.validate_database_cycles_purchase_with_minimum( - &purchase.database_id, - purchase.payment_amount_e8s, - purchase.min_expected_cycles, - )?; - Ok(cycles) - }) { - Ok(cycles) => cycles, - Err(error) => return icrc21_unsupported(error), - }; let language = if request.user_preferences.metadata.language.trim().is_empty() { "en".to_string() } else { request.user_preferences.metadata.language }; - Icrc21ConsentMessageResponse::Ok(Icrc21ConsentInfo { - metadata: Icrc21ConsentMessageMetadata { - language, - utc_offset_minutes: request.user_preferences.metadata.utc_offset_minutes, - }, - consent_message: Icrc21ConsentMessage::GenericDisplayMessage(format!( - "# Purchase Kinic database cycles\n\nDatabase: `{database_id}`\n\nCycles: `{cycles}`\n\nPayment: `{payment}` KINIC\n\nLedger transfer fee in allowance: `{fee}` KINIC\n\nSpender canister: `{spender}`", - database_id = purchase.database_id, - cycles = format_cycles(cycles), - payment = format_e8s(purchase.payment_amount_e8s), - fee = format_e8s(KINIC_LEDGER_FEE_E8S), - spender = canister_principal().to_text() - )), - }) + let metadata = Icrc21ConsentMessageMetadata { + language, + utc_offset_minutes: request.user_preferences.metadata.utc_offset_minutes, + }; + match request.method.as_str() { + "purchase_database_cycles" => { + let purchase = match Decode!(&request.arg, DatabaseCyclesPurchaseRequest) { + Ok(decoded) => decoded, + Err(error) => { + return icrc21_unavailable(format!( + "purchase_database_cycles argument decode failed: {error}" + )); + } + }; + let cycles = match with_service(|service| { + let config = service.cycles_billing_config()?; + let cycles = cycles_for_payment_amount_e8s(purchase.payment_amount_e8s, &config)?; + service.validate_database_cycles_purchase_with_minimum( + &purchase.database_id, + purchase.payment_amount_e8s, + purchase.min_expected_cycles, + )?; + Ok(cycles) + }) { + Ok(cycles) => cycles, + Err(error) => return icrc21_unsupported(error), + }; + Icrc21ConsentMessageResponse::Ok(Icrc21ConsentInfo { + metadata, + consent_message: Icrc21ConsentMessage::GenericDisplayMessage(format!( + "# Purchase Kinic database cycles\n\nDatabase: `{database_id}`\n\nCycles: `{cycles}`\n\nPayment: `{payment}` KINIC\n\nLedger transfer fee in allowance: `{fee}` KINIC\n\nSpender canister: `{spender}`", + database_id = purchase.database_id, + cycles = format_cycles(cycles), + payment = format_e8s(purchase.payment_amount_e8s), + fee = format_e8s(KINIC_LEDGER_FEE_E8S), + spender = canister_principal().to_text() + )), + }) + } + "kinic_deposit_balance" => { + let deposit = match Decode!(&request.arg, KinicDepositRequest) { + Ok(decoded) => decoded, + Err(error) => { + return icrc21_unavailable(format!( + "kinic_deposit_balance argument decode failed: {error}" + )); + } + }; + if deposit.amount_e8s == 0 { + return icrc21_unsupported("KINIC deposit amount must be positive".to_string()); + } + if i64::try_from(deposit.amount_e8s).is_err() + || i64::try_from(deposit.expected_fee_e8s).is_err() + { + return icrc21_unsupported("KINIC deposit amount exceeds i64".to_string()); + } + Icrc21ConsentMessageResponse::Ok(Icrc21ConsentInfo { + metadata, + consent_message: Icrc21ConsentMessage::GenericDisplayMessage(format!( + "# Deposit KINIC to KINIC balance\n\nAmount: `{amount}` KINIC\n\nLedger transfer fee in allowance: `{fee}` KINIC\n\nSpender canister: `{spender}`\n\nPurpose: canister-held KINIC balance", + amount = format_e8s(deposit.amount_e8s), + fee = format_e8s(deposit.expected_fee_e8s), + spender = canister_principal().to_text() + )), + }) + } + method => icrc21_unsupported(format!("unsupported canister call: {method}")), + } } #[update] @@ -709,6 +744,229 @@ fn list_database_cycles_pending_purchases( }) } +#[query] +fn kinic_get_balance() -> Result { + with_service(|service| service.kinic_get_balance(&caller_text())) +} + +#[update] +fn kinic_fund_database_cycles( + request: KinicFundDatabaseCyclesRequest, +) -> Result { + require_authenticated_caller()?; + with_unmetered_update( + "kinic_fund_database_cycles", + Some(request.database_id.clone()), + |service, caller, now| service.kinic_fund_database_cycles(caller, request, now), + ) +} + +#[update] +async fn kinic_deposit_balance(request: KinicDepositRequest) -> Result { + require_authenticated_caller()?; + let caller = caller_text(); + let now = now_millis(); + let config = with_service(|service| service.cycles_billing_config())?; + let ledger = Principal::from_text(&config.kinic_ledger_canister_id) + .map_err(|error| format!("invalid KINIC ledger canister id: {error}"))?; + let ledger_created_at_time_ns = now_nanos(); + let canister_owner = canister_principal().to_text(); + let deposit_start = with_service(|service| { + service.begin_kinic_deposit_with_ledger_details(KinicDepositWithLedgerDetails { + caller: &caller, + amount_e8s: request.amount_e8s, + ledger: CyclesPendingLedgerDetailsInput { + from_owner: &caller, + from_subaccount: None, + to_owner: &canister_owner, + to_subaccount: None, + ledger_fee_e8s: request.expected_fee_e8s, + ledger_created_at_time_ns, + }, + now, + }) + })?; + let operation_id = deposit_start.operation_id; + match ledger_transfer_from_with_memo( + ledger, + IcrcAccount { + owner: caller_principal(), + subaccount: None, + }, + IcrcAccount { + owner: canister_principal(), + subaccount: None, + }, + request.amount_e8s, + request.expected_fee_e8s, + kinic_deposit_memo(operation_id), + ledger_created_at_time_ns, + ) + .await + { + LedgerTransferFromOutcome::Completed(block_index) => { + if let Err(error) = with_service(|service| { + service.complete_kinic_deposit_ledger_transfer( + operation_id, + &caller, + request.amount_e8s, + block_index, + ) + }) { + return Err(kinic_deposit_local_apply_error( + operation_id, + block_index, + error, + )); + } + #[cfg(test)] + if TEST_KINIC_DEPOSIT_APPLY_FAIL_ONCE.with(|flag| flag.replace(false)) { + return Err(kinic_deposit_local_apply_error( + operation_id, + block_index, + "test KINIC deposit apply failure".to_string(), + )); + } + let balance = with_service(|service| { + service.apply_kinic_deposit(operation_id, &caller, request.amount_e8s, now) + }) + .map_err(|error| kinic_deposit_local_apply_error(operation_id, block_index, error))?; + Ok(KinicDepositResult { + block_index, + amount_e8s: deposit_start.amount_e8s, + balance_e8s: balance.balance_e8s, + }) + } + LedgerTransferFromOutcome::BadFee { expected_fee_e8s } => { + let _ = with_service(|service| { + service.cancel_kinic_deposit_after_ledger_error( + operation_id, + &caller, + request.amount_e8s, + ) + }); + Err(format!( + "icrc2_transfer_from failed: BadFee expected fee {expected_fee_e8s}; re-approve with the current ledger fee and retry" + )) + } + LedgerTransferFromOutcome::LedgerErr(error) => { + let _ = with_service(|service| { + service.cancel_kinic_deposit_after_ledger_error( + operation_id, + &caller, + request.amount_e8s, + ) + }); + Err(error) + } + LedgerTransferFromOutcome::Ambiguous(error) => { + if let Err(mark_error) = with_service(|service| { + service.mark_kinic_deposit_ambiguous(operation_id, &caller, request.amount_e8s) + }) { + return Err(format!( + "icrc2_transfer_from result ambiguous for KINIC deposit operation_id {operation_id}; failed to mark operation ambiguous; billing authority review required: {mark_error}; original ledger ambiguity: {error}" + )); + } + Err(format!( + "icrc2_transfer_from result ambiguous for KINIC deposit operation_id {operation_id}; billing authority review required: {error}" + )) + } + } +} + +fn kinic_deposit_local_apply_error(operation_id: u64, block_index: u64, cause: String) -> String { + format!( + "KINIC deposit completed at ledger block {block_index} but local balance application failed; pending operation {operation_id} remains completed for billing authority review: {cause}" + ) +} + +#[query] +fn kinic_list_pending_operations( + request: KinicPendingOperationsPageRequest, +) -> Result { + with_service(|service| service.kinic_list_pending_operations(&caller_text(), request)) +} + +#[update] +fn market_create_listing(request: MarketCreateListingRequest) -> Result { + require_authenticated_caller()?; + with_unmetered_update( + "market_create_listing", + Some(request.database_id.clone()), + |service, caller, now| service.market_create_listing(caller, request, now), + ) +} + +#[update] +fn market_update_listing(request: MarketUpdateListingRequest) -> Result { + require_authenticated_caller()?; + with_unmetered_update("market_update_listing", None, |service, caller, now| { + service.market_update_listing(caller, request, now) + }) +} + +#[update] +fn market_publish_listing(listing_id: String) -> Result { + require_authenticated_caller()?; + with_unmetered_update("market_publish_listing", None, |service, caller, now| { + service.market_publish_listing(caller, &listing_id, now) + }) +} + +#[update] +fn market_pause_listing(listing_id: String) -> Result { + require_authenticated_caller()?; + with_unmetered_update("market_pause_listing", None, |service, caller, now| { + service.market_pause_listing(caller, &listing_id, now) + }) +} + +#[query] +fn market_list_listings(cursor: Option, limit: u32) -> Result { + with_service(|service| service.market_list_listings(cursor, limit)) +} + +#[query] +fn market_list_database_listings(database_id: String) -> Result, String> { + with_service(|service| service.market_list_database_listings(&caller_text(), &database_id)) +} + +#[query] +fn market_get_listing(listing_id: String) -> Result { + with_service(|service| service.market_get_listing(&caller_text(), &listing_id)) +} + +#[query] +fn market_preview_purchase(listing_id: String) -> Result { + with_service(|service| service.market_preview_purchase(&caller_text(), &listing_id)) +} + +#[update] +fn market_purchase_access(request: MarketPurchaseRequest) -> Result { + require_authenticated_caller()?; + with_unmetered_update("market_purchase_access", None, |service, caller, now| { + service.market_purchase_access(caller, request, now) + }) +} + +#[query] +fn market_list_entitlements( + cursor: Option, + limit: u32, +) -> Result { + with_service(|service| service.market_list_entitlements(&caller_text(), cursor, limit)) +} + +#[query] +fn market_list_orders(cursor: Option, limit: u32) -> Result { + with_service(|service| service.market_list_orders(&caller_text(), cursor, limit)) +} + +#[query] +fn market_count_active_entitlements(database_id: String) -> Result { + with_service(|service| service.market_count_active_entitlements(&caller_text(), &database_id)) +} + #[update] fn update_cycles_billing_config(update: CyclesBillingConfigUpdate) -> Result<(), String> { require_authenticated_caller()?; @@ -1246,6 +1504,7 @@ thread_local! { static TEST_LAST_LEDGER_FROM: RefCell> = const { RefCell::new(None) }; static TEST_CALLER_PRINCIPAL: RefCell> = const { RefCell::new(None) }; static TEST_DATABASE_CYCLES_PURCHASE_APPLY_FAIL_ONCE: RefCell = const { RefCell::new(false) }; + static TEST_KINIC_DEPOSIT_APPLY_FAIL_ONCE: RefCell = const { RefCell::new(false) }; static TEST_UPDATE_CHARGE_UNITS: RefCell> = const { RefCell::new(Vec::new()) }; } @@ -1259,6 +1518,11 @@ fn fail_next_apply_database_cycles_purchase_apply_for_test() { TEST_DATABASE_CYCLES_PURCHASE_APPLY_FAIL_ONCE.with(|flag| flag.replace(true)); } +#[cfg(test)] +fn fail_next_apply_kinic_deposit_for_test() { + TEST_KINIC_DEPOSIT_APPLY_FAIL_ONCE.with(|flag| flag.replace(true)); +} + #[cfg(test)] fn set_next_ledger_transfer_from_outcome_for_test(outcome: LedgerTransferFromOutcome) { TEST_LEDGER_TRANSFER_FROM_OUTCOMES.with(|slot| { @@ -1495,6 +1759,27 @@ async fn ledger_transfer_from( created_at_time_ns: u64, ) -> LedgerTransferFromOutcome { let memo = cycles_purchase_memo(operation_id); + ledger_transfer_from_with_memo( + ledger, + from, + to, + amount_e8s, + ledger_fee_e8s, + memo, + created_at_time_ns, + ) + .await +} + +async fn ledger_transfer_from_with_memo( + ledger: Principal, + from: IcrcAccount, + to: IcrcAccount, + amount_e8s: u64, + ledger_fee_e8s: u64, + memo: Vec, + created_at_time_ns: u64, +) -> LedgerTransferFromOutcome { #[cfg(test)] { record_test_ledger_from(&from); @@ -1581,6 +1866,10 @@ fn cycles_purchase_memo(operation_id: u64) -> Vec { format!("kvfs:cp:{operation_id}").into_bytes() } +fn kinic_deposit_memo(operation_id: u64) -> Vec { + format!("kvfs:md:{operation_id}").into_bytes() +} + fn with_unmetered_update(method: &str, database_id: Option, f: F) -> Result where F: FnOnce(&VfsService, &str, i64) -> Result, @@ -1811,7 +2100,7 @@ fn certified_static_responses() -> Vec<( vec![ certified_static_response_entry( II_ALTERNATIVE_ORIGINS_PATH, - II_ALTERNATIVE_ORIGINS_BODY.as_bytes().to_vec(), + ii_alternative_origins_body().as_bytes().to_vec(), "application/json; charset=utf-8", true, ), @@ -1830,6 +2119,13 @@ fn certified_static_responses() -> Vec<( ] } +fn ii_alternative_origins_body() -> &'static str { + match option_env!("KINIC_VFS_LOCAL_II_ORIGINS") { + Some("1") => II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY, + _ => II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY, + } +} + fn certified_static_response_entry( path: &'static str, body: Vec, diff --git a/crates/vfs_canister/src/tests.rs b/crates/vfs_canister/src/tests.rs index 5421bd90..a01ac46a 100644 --- a/crates/vfs_canister/src/tests.rs +++ b/crates/vfs_canister/src/tests.rs @@ -13,12 +13,13 @@ use vfs_types::{ DatabaseCyclesPurchaseRequest, DatabaseRestoreChunkRequest, DatabaseRole, DatabaseStatus, DeleteDatabaseRequest, DeleteNodeRequest, EditNodeRequest, ExportSnapshotRequest, FetchUpdatesRequest, GlobNodeType, GlobNodesRequest, GraphLinksRequest, - GraphNeighborhoodRequest, IncomingLinksRequest, KINIC_LEDGER_FEE_E8S, ListChildrenRequest, - ListNodesRequest, MkdirNodeRequest, MoveNodeRequest, MultiEdit, MultiEditNodeRequest, - NodeContextRequest, NodeEntryKind, NodeKind, OutgoingLinksRequest, QueryContextRequest, - RenameDatabaseRequest, SearchNodePathsRequest, SearchNodesRequest, SearchPreviewMode, - SourceEvidenceRequest, StorageBillingBatchRequest, WriteNodeItem, WriteNodeRequest, - WriteNodesRequest, + GraphNeighborhoodRequest, IncomingLinksRequest, KINIC_LEDGER_FEE_E8S, KinicDepositRequest, + KinicFundDatabaseCyclesRequest, KinicPendingOperationsPageRequest, ListChildrenRequest, + ListNodesRequest, MarketCreateListingRequest, MarketPurchaseRequest, MkdirNodeRequest, + MoveNodeRequest, MultiEdit, MultiEditNodeRequest, NodeContextRequest, NodeEntryKind, NodeKind, + OutgoingLinksRequest, QueryContextRequest, RenameDatabaseRequest, SearchNodePathsRequest, + SearchNodesRequest, SearchPreviewMode, SourceEvidenceRequest, StorageBillingBatchRequest, + WriteNodeItem, WriteNodeRequest, WriteNodesRequest, }; use super::{ @@ -28,20 +29,22 @@ use super::{ cancel_database_archive, check_database_write_cycles, clear_last_ledger_memo_for_test, clear_ledger_transactions_for_test, create_database, delete_node, edit_node, export_snapshot, fail_next_apply_database_cycles_purchase_apply_for_test, - fail_next_mount_database_file_for_test, fetch_updates, finalize_database_archive, - finalize_database_restore, get_cycles_billing_config, glob_nodes, grant_database_access, - graph_links, graph_neighborhood, icrc21_canister_call_consent_message, incoming_links, - last_ledger_from_for_test, last_ledger_memo_for_test, ledger_transfer_fees_for_test, - list_children, list_database_cycle_entries, list_database_cycles_pending_purchases, - list_database_members, list_databases, list_nodes, memory_manifest, mkdir_node, move_node, - multi_edit_node, outgoing_links, parse_upgrade_cycles_billing_config_arg, - purchase_database_cycles, query_context, query_index_sql_json, read_database_archive_chunk, - read_node, read_node_context, rename_database, revoke_database_access, search_node_paths, - search_nodes, set_next_ledger_transfer_from_outcome_for_test, - set_test_caller_principal_for_test, set_update_charge_units_for_test, - settle_database_storage_charges_batch, source_evidence, status, transfer_from_error_outcome, - update_charge_cycles, update_cycles_billing_config, write_database_restore_chunk, write_node, - write_nodes, + fail_next_apply_kinic_deposit_for_test, fail_next_mount_database_file_for_test, fetch_updates, + finalize_database_archive, finalize_database_restore, get_cycles_billing_config, glob_nodes, + grant_database_access, graph_links, graph_neighborhood, icrc21_canister_call_consent_message, + incoming_links, kinic_deposit_balance, kinic_fund_database_cycles, kinic_get_balance, + kinic_list_pending_operations, last_ledger_from_for_test, last_ledger_memo_for_test, + ledger_transfer_fees_for_test, list_children, list_database_cycle_entries, + list_database_cycles_pending_purchases, list_database_members, list_databases, list_nodes, + market_create_listing, market_publish_listing, market_purchase_access, memory_manifest, + mkdir_node, move_node, multi_edit_node, outgoing_links, + parse_upgrade_cycles_billing_config_arg, purchase_database_cycles, query_context, + query_index_sql_json, read_database_archive_chunk, read_node, read_node_context, + rename_database, revoke_database_access, search_node_paths, search_nodes, + set_next_ledger_transfer_from_outcome_for_test, set_test_caller_principal_for_test, + set_update_charge_units_for_test, settle_database_storage_charges_batch, source_evidence, + status, transfer_from_error_outcome, update_charge_cycles, update_cycles_billing_config, + write_database_restore_chunk, write_node, write_nodes, }; fn install_test_service() { @@ -131,6 +134,63 @@ fn pending_cycle_purchase_state(database_id: &str) -> String { }) } +fn pending_kinic_deposit_state() -> String { + SERVICE.with(|slot| { + slot.borrow() + .as_ref() + .expect("service should be installed") + .query_index_sql_json( + "SELECT json_object('status', operation_status, 'block', external_block_index) FROM kinic_pending_operations WHERE kind = 'deposit' LIMIT 1", + 1, + ) + .expect("pending operation should query") + .rows + .into_iter() + .next() + .expect("pending operation should exist") + }) +} + +fn kinic_deposit_ledger_count(principal: &str) -> String { + SERVICE.with(|slot| { + slot.borrow() + .as_ref() + .expect("service should be installed") + .query_index_sql_json( + &format!( + "SELECT json_object('count', COUNT(*)) FROM kinic_ledger WHERE principal = '{}' AND kind = 'deposit' LIMIT 1", + principal + ), + 1, + ) + .expect("KINIC ledger should query") + .rows + .into_iter() + .next() + .expect("KINIC ledger count should exist") + }) +} + +fn database_cycles_balance_json(database_id: &str) -> String { + SERVICE.with(|slot| { + slot.borrow() + .as_ref() + .expect("service should be installed") + .query_index_sql_json( + &format!( + "SELECT json_object('balance', balance_cycles) FROM database_cycle_accounts WHERE database_id = '{}' LIMIT 1", + database_id + ), + 1, + ) + .expect("database cycles account should query") + .rows + .into_iter() + .next() + .expect("database cycles account should exist") + }) +} + fn cycles_for_test_payment(service: &VfsService, payment_amount_e8s: u64) -> u64 { super::cycles_for_payment_amount_e8s( payment_amount_e8s, @@ -173,6 +233,37 @@ fn cycles_purchase_request( } } +fn kinic_deposit_request(amount_e8s: u64) -> KinicDepositRequest { + KinicDepositRequest { + amount_e8s, + expected_fee_e8s: KINIC_LEDGER_FEE_E8S, + } +} + +fn kinic_fund_request( + database_id: &str, + payment_amount_e8s: u64, +) -> KinicFundDatabaseCyclesRequest { + KinicFundDatabaseCyclesRequest { + database_id: database_id.to_string(), + payment_amount_e8s, + min_expected_cycles: 1, + } +} + +fn market_listing_request(database_id: &str, price_e8s: u64) -> MarketCreateListingRequest { + MarketCreateListingRequest { + database_id: database_id.to_string(), + title: "Private market DB".to_string(), + description: "Paid reader access".to_string(), + llm_summary: None, + summary_snapshot_revision: None, + sample_excerpts_json: "[]".to_string(), + tags_json: "[]".to_string(), + price_e8s, + } +} + fn consent_request(method: &str, arg: Vec) -> Icrc21ConsentMessageRequest { Icrc21ConsentMessageRequest { arg, @@ -392,6 +483,16 @@ fn test_billing_authority_principal() -> Principal { .expect("billing authority principal should parse") } +fn pending_operations_request( + cursor_operation_id: Option, + limit: u32, +) -> KinicPendingOperationsPageRequest { + KinicPendingOperationsPageRequest { + cursor_operation_id, + limit, + } +} + struct AuthenticatedCallerGuard; impl AuthenticatedCallerGuard { @@ -576,6 +677,251 @@ fn purchase_database_cycles_treats_duplicate_as_completed_transfer() { assert_eq!(entries[0].ledger_block_index, Some(77)); } +#[test] +fn kinic_fund_database_cycles_uses_internal_balance_without_ledger_call() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + let database_id = "internal-funding-active"; + SERVICE.with(|slot| { + slot.borrow() + .as_ref() + .expect("service should be installed") + .create_database(database_id, &Principal::management_canister().to_text(), 1) + .expect("active database should create"); + }); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(90)); + block_on_ready(kinic_deposit_balance(kinic_deposit_request(1_000))) + .expect("deposit should credit internal KINIC balance"); + clear_last_ledger_memo_for_test(); + + let result = kinic_fund_database_cycles(kinic_fund_request(database_id, 500)) + .expect("internal balance should fund database cycles"); + + assert_eq!(result.payment_amount_e8s, 500); + assert_eq!(result.amount_cycles, 1_172_500); + assert_eq!(result.database_balance_cycles, 1_172_500); + assert_eq!(result.kinic_balance_e8s, 500); + assert_eq!(last_ledger_memo_for_test(), None); + let entries = list_database_cycle_entries(database_id.to_string(), None, 10) + .expect("database ledger should load") + .entries; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].kind, "cycles_purchase"); + assert_eq!( + entries[0].method.as_deref(), + Some("kinic_fund_database_cycles") + ); + assert_eq!(entries[0].ledger_block_index, None); +} + +#[test] +fn kinic_fund_database_cycles_rejects_unallocated_pending_without_state_change() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + let database = create_database(CreateDatabaseRequest { + name: "Pending internal funding".to_string(), + }) + .expect("pending database should create"); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(91)); + block_on_ready(kinic_deposit_balance(kinic_deposit_request(1_000))) + .expect("deposit should credit internal KINIC balance"); + clear_last_ledger_memo_for_test(); + assert_eq!( + database_status_and_mount(&database.database_id), + (DatabaseStatus::Pending, None) + ); + assert_eq!( + database_cycles_balance_json(&database.database_id), + "{\"balance\":0}" + ); + let balance_before = kinic_get_balance() + .expect("KINIC balance should load") + .balance_e8s; + + let error = kinic_fund_database_cycles(kinic_fund_request(&database.database_id, 500)) + .expect_err("unallocated pending database should reject"); + + assert!(error.contains("pending database has no activation mount")); + assert_eq!( + database_status_and_mount(&database.database_id), + (DatabaseStatus::Pending, None) + ); + assert_eq!( + database_cycles_balance_json(&database.database_id), + "{\"balance\":0}" + ); + assert_eq!( + kinic_get_balance() + .expect("KINIC balance should load") + .balance_e8s, + balance_before + ); + assert_eq!(last_ledger_memo_for_test(), None); + assert!( + list_database_cycle_entries(database.database_id, None, 10) + .expect("database ledger should load") + .entries + .is_empty() + ); +} + +#[test] +fn kinic_deposit_balance_credits_wallet_after_completed_transfer() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(42)); + + let result = block_on_ready(kinic_deposit_balance(kinic_deposit_request(500))) + .expect("completed transfer-from should credit KINIC balance"); + + assert_eq!(result.block_index, 42); + assert_eq!(result.amount_e8s, 500); + assert_eq!(result.balance_e8s, 500); + assert_eq!( + kinic_get_balance() + .expect("KINIC balance should load") + .balance_e8s, + 500 + ); + assert!( + kinic_list_pending_operations(pending_operations_request(None, 100)) + .expect("pending operations should load") + .operations + .is_empty() + ); +} + +#[test] +fn kinic_deposit_duplicate_transfer_is_success_without_double_credit() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + set_next_ledger_transfer_from_outcome_for_test(transfer_from_error_outcome( + TransferFromError::Duplicate { + duplicate_of: Nat::from(77_u64), + }, + )); + + let result = block_on_ready(kinic_deposit_balance(kinic_deposit_request(700))) + .expect("duplicate transfer-from should credit KINIC balance"); + + assert_eq!(result.block_index, 77); + assert_eq!(result.balance_e8s, 700); + assert_eq!( + kinic_deposit_ledger_count(&Principal::management_canister().to_text()), + r#"{"count":1}"# + ); + assert!( + kinic_list_pending_operations(pending_operations_request(None, 100)) + .expect("pending operations should load") + .operations + .is_empty() + ); +} + +#[test] +fn kinic_deposit_completed_local_apply_failure_keeps_pending_for_review() { + install_empty_test_service(); + let caller = Principal::management_canister(); + let _caller = AuthenticatedCallerGuard::install_principal(caller); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(44)); + fail_next_apply_kinic_deposit_for_test(); + + let error = block_on_ready(kinic_deposit_balance(kinic_deposit_request(600))) + .expect_err("local apply failure should keep completed pending deposit"); + + assert!(error.contains("test KINIC deposit apply failure")); + assert!(error.contains("remains completed for billing authority review")); + assert_eq!( + pending_kinic_deposit_state(), + r#"{"status":"completed","block":44}"# + ); + let caller_view = kinic_list_pending_operations(pending_operations_request(None, 100)) + .expect("caller should view own pending"); + assert_eq!(caller_view.operations.len(), 1); + assert_eq!(caller_view.operations[0].status, "completed"); + assert_eq!( + caller_view.operations[0].required_action, + "billing_authority_review" + ); + drop(_caller); + + let _authority = + AuthenticatedCallerGuard::install_principal(test_billing_authority_principal()); + let authority_view = kinic_list_pending_operations(pending_operations_request(None, 100)) + .expect("billing authority should view all pending"); + assert_eq!(authority_view.operations.len(), 1); + assert_eq!(authority_view.operations[0].caller, caller.to_text()); +} + +#[test] +fn kinic_deposit_ambiguous_transfer_keeps_pending_for_review() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Ambiguous( + "icrc2_transfer_from timeout".to_string(), + )); + + let error = block_on_ready(kinic_deposit_balance(kinic_deposit_request(800))) + .expect_err("ambiguous transfer-from should keep pending deposit"); + + assert!(error.contains("result ambiguous")); + assert!(error.contains("billing authority review required")); + assert_eq!( + pending_kinic_deposit_state(), + r#"{"status":"ambiguous","block":null}"# + ); + let pending = kinic_list_pending_operations(pending_operations_request(None, 100)) + .expect("caller should view own pending"); + assert_eq!(pending.operations.len(), 1); + assert_eq!(pending.operations[0].status, "ambiguous"); + assert_eq!( + pending.operations[0].required_action, + "billing_authority_review" + ); +} + +#[test] +fn kinic_list_pending_operations_paginates_authority_view() { + install_empty_test_service(); + for index in 0..3 { + let caller = Principal::self_authenticating([index as u8]); + let _caller = AuthenticatedCallerGuard::install_principal(caller); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Ambiguous( + format!("ambiguous deposit {index}"), + )); + block_on_ready(kinic_deposit_balance(kinic_deposit_request(800 + index))) + .expect_err("ambiguous transfer should keep pending operation"); + } + + let _authority = + AuthenticatedCallerGuard::install_principal(test_billing_authority_principal()); + let first = kinic_list_pending_operations(pending_operations_request(None, 2)) + .expect("first pending page should load"); + assert_eq!(first.operations.len(), 2); + assert_eq!(first.next_cursor_operation_id, Some(2)); + + let second = kinic_list_pending_operations(pending_operations_request( + first.next_cursor_operation_id, + 2, + )) + .expect("second pending page should load"); + assert_eq!(second.operations.len(), 1); + assert_eq!(second.operations[0].operation_id, 3); + assert_eq!(second.next_cursor_operation_id, None); +} + +#[test] +fn kinic_balance_queries_reject_anonymous_caller() { + install_empty_test_service(); + set_test_caller_principal_for_test(Principal::anonymous()); + + let balance_error = kinic_get_balance().expect_err("anonymous balance query should reject"); + assert!(balance_error.contains("anonymous caller not allowed")); + let pending_error = kinic_list_pending_operations(pending_operations_request(None, 100)) + .expect_err("anonymous pending query should reject"); + assert!(pending_error.contains("anonymous caller not allowed")); +} + #[test] fn list_database_cycle_entries_paginates_with_clamped_limits() { install_empty_test_service(); @@ -1123,6 +1469,62 @@ fn icrc21_purchase_database_cycles_rejects_missing_database() { } } +#[test] +fn icrc21_kinic_deposit_balance_returns_consent_message() { + install_empty_test_service(); + let request = kinic_deposit_request(150_000_000); + let arg = Encode!(&request).expect("arg should encode"); + + let response = + icrc21_canister_call_consent_message(consent_request("kinic_deposit_balance", arg)); + + let message = match response { + Icrc21ConsentMessageResponse::Ok(info) => match info.consent_message { + Icrc21ConsentMessage::GenericDisplayMessage(message) => message, + }, + Icrc21ConsentMessageResponse::Err(error) => { + panic!("KINIC deposit consent message should succeed: {error:?}"); + } + }; + assert!(message.contains("Deposit KINIC to KINIC balance")); + assert!(message.contains("Amount: `1.5` KINIC")); + assert!(message.contains("Ledger transfer fee in allowance: `0.001` KINIC")); + assert!(message.contains("Spender canister:")); + assert!(message.contains("canister-held KINIC balance")); +} + +#[test] +fn icrc21_kinic_deposit_balance_rejects_zero_amount() { + install_empty_test_service(); + let arg = Encode!(&kinic_deposit_request(0)).expect("arg should encode"); + + let response = + icrc21_canister_call_consent_message(consent_request("kinic_deposit_balance", arg)); + + match response { + Icrc21ConsentMessageResponse::Err(super::Icrc21Error::UnsupportedCanisterCall(info)) => { + assert!(info.description.contains("amount must be positive")); + } + other => panic!("zero deposit consent should reject: {other:?}"), + } +} + +#[test] +fn icrc21_kinic_deposit_balance_rejects_overflow_amount() { + install_empty_test_service(); + let arg = Encode!(&kinic_deposit_request(u64::MAX)).expect("arg should encode"); + + let response = + icrc21_canister_call_consent_message(consent_request("kinic_deposit_balance", arg)); + + match response { + Icrc21ConsentMessageResponse::Err(super::Icrc21Error::UnsupportedCanisterCall(info)) => { + assert!(info.description.contains("amount exceeds i64")); + } + other => panic!("overflow deposit consent should reject: {other:?}"), + } +} + #[test] fn icrc21_rejects_unsupported_cycle_consent_method() { install_empty_test_service(); @@ -1148,6 +1550,18 @@ fn icrc21_rejects_malformed_cycle_consent_arg() { )); } +#[test] +fn icrc21_rejects_malformed_kinic_deposit_consent_arg() { + install_empty_test_service(); + let response = + icrc21_canister_call_consent_message(consent_request("kinic_deposit_balance", Vec::new())); + + assert!(matches!( + response, + Icrc21ConsentMessageResponse::Err(super::Icrc21Error::ConsentMessageUnavailable(_)) + )); +} + #[test] fn purchase_database_cycles_sends_operation_memo_to_ledger() { install_empty_test_service(); @@ -1371,6 +1785,97 @@ fn canister_list_databases_hides_deleted_databases() { assert!(summaries.is_empty()); } +#[test] +fn canister_list_databases_includes_market_entitlements_as_reader_access() { + install_empty_test_service(); + let owner = Principal::management_canister(); + let buyer = Principal::self_authenticating(b"market buyer"); + let database_id; + let listing_id; + + { + let _caller = AuthenticatedCallerGuard::install_principal(owner); + let database = create_database(CreateDatabaseRequest { + name: "Private market".to_string(), + }) + .expect("market database should create"); + database_id = database.database_id; + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(201)); + block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database_id, + 1_000_000, + ))) + .expect("market database should activate"); + write_node(WriteNodeRequest { + database_id: database_id.clone(), + path: "/Wiki/paid.md".to_string(), + kind: NodeKind::File, + content: "# Paid\n\nPrivate body".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .expect("market database content should write"); + let listing = market_create_listing(market_listing_request(&database_id, 500)) + .expect("listing should create"); + assert!(!listing.listing_id.starts_with("listing_")); + let active = market_publish_listing(listing.listing_id).expect("listing should publish"); + listing_id = active.listing_id; + } + + { + let _caller = AuthenticatedCallerGuard::install_principal(buyer); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(202)); + block_on_ready(kinic_deposit_balance(kinic_deposit_request(1_000))) + .expect("buyer should deposit KINIC"); + market_purchase_access(MarketPurchaseRequest { + listing_id: listing_id.clone(), + price_e8s: 500, + }) + .expect("buyer should purchase access"); + + let summaries = list_databases().expect("buyer database summaries should load"); + let summary = summaries + .iter() + .find(|summary| summary.database_id == database_id) + .expect("entitled database should appear in authenticated list"); + assert_eq!(summary.role, DatabaseRole::Reader); + + let node = read_node(database_id.clone(), "/Wiki/paid.md".to_string()) + .expect("entitled buyer read should succeed") + .expect("paid node should exist"); + assert_eq!(node.content, "# Paid\n\nPrivate body"); + let children = list_children(ListChildrenRequest { + database_id: database_id.clone(), + path: "/Wiki".to_string(), + }) + .expect("entitled buyer should list children"); + assert!(children.iter().any(|child| child.path == "/Wiki/paid.md")); + } + + let anonymous_summaries = list_databases().expect("anonymous summaries should load"); + assert!( + anonymous_summaries + .iter() + .all(|summary| summary.database_id != database_id) + ); + + { + let _caller = AuthenticatedCallerGuard::install_principal(owner); + grant_database_access(database_id.clone(), buyer.to_text(), DatabaseRole::Writer) + .expect("owner should grant writer access"); + } + { + let _caller = AuthenticatedCallerGuard::install_principal(buyer); + let summaries = list_databases().expect("buyer database summaries should load"); + let matching = summaries + .iter() + .filter(|summary| summary.database_id == database_id) + .collect::>(); + assert_eq!(matching.len(), 1); + assert_eq!(matching[0].role, DatabaseRole::Writer); + } +} + #[test] fn update_charge_cycles_checks_counter_order_and_overflow() { assert_eq!( diff --git a/crates/vfs_canister/src/tests_sync_contract.rs b/crates/vfs_canister/src/tests_sync_contract.rs index 03065195..8df52e57 100644 --- a/crates/vfs_canister/src/tests_sync_contract.rs +++ b/crates/vfs_canister/src/tests_sync_contract.rs @@ -10,8 +10,9 @@ use vfs_types::{ use super::{ HttpRequest, ICP_CLI_LOGIN_DISCOVERY_PATH, ICP_CLI_LOGIN_PATH, II_ALTERNATIVE_ORIGINS_PATH, - SERVICE, delete_node, export_snapshot, fetch_updates, http_request, mkdir_node, - search_node_paths, search_nodes, write_node, + II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY, II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY, SERVICE, + delete_node, export_snapshot, fetch_updates, http_request, mkdir_node, search_node_paths, + search_nodes, write_node, }; use ic_http_certification::CERTIFICATE_EXPRESSION_HEADER_NAME; @@ -97,6 +98,31 @@ fn http_request_serves_certified_ii_alternative_origins() { ); } +#[test] +fn production_ii_alternative_origins_do_not_include_localhost() { + assert!(!II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY.contains("http://127.0.0.1:3000")); + assert!(!II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY.contains("http://localhost:3000")); + assert!(!II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY.contains("http://127.0.0.1:3010")); + assert!(!II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY.contains("http://localhost:3010")); + assert!(!II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY.contains("http://127.0.0.1:3100")); + assert!(!II_PRODUCTION_ALTERNATIVE_ORIGINS_BODY.contains("http://localhost:3100")); +} + +#[test] +fn local_dev_ii_alternative_origins_include_fixed_dev_server_origins() { + assert!(!II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY.contains("http://127.0.0.1:3000")); + assert!(II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY.contains("http://localhost:3000")); + assert!(II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY.contains("http://127.0.0.1:3010")); + assert!(II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY.contains("http://localhost:3010")); + assert!(II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY.contains("http://127.0.0.1:3100")); + assert!(II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY.contains("http://localhost:3100")); + assert_eq!( + II_LOCAL_DEV_ALTERNATIVE_ORIGINS_BODY.matches("://").count(), + 10, + "Internet Identity rejects ii-alternative-origins with more than 10 entries" + ); +} + #[test] fn http_request_serves_icp_cli_login_discovery() { let response = http_request(test_http_get(ICP_CLI_LOGIN_DISCOVERY_PATH)); @@ -133,9 +159,10 @@ fn http_request_serves_icp_cli_login_page() { assert!(body.contains("Callback host/port")); assert!(body.contains("Derivation origin")); assert!(body.contains("Delegation TTL")); - assert!(body.contains("http://id.ai.localhost:")); + assert!(body.contains("https://id.ai")); + assert!(!body.contains("http://id.ai.localhost:")); assert!(body.contains("https://xis3j-paaaa-aaaai-axumq-cai.icp0.io")); - assert!(compact_body.contains("endsWith(\".localhost\")?")); + assert!(compact_body.contains("endsWith(\".localhost\")")); assert!(compact_body.contains("derivationOrigin:")); assert!(compact_body.contains(r#"method:"POST""#)); assert!(compact_body.contains(r#""content-type":"application/json""#)); diff --git a/crates/vfs_canister/vfs.did b/crates/vfs_canister/vfs.did index dc4433a3..4a72fc44 100644 --- a/crates/vfs_canister/vfs.did +++ b/crates/vfs_canister/vfs.did @@ -233,6 +233,45 @@ type IndexSqlJsonQueryResult = record { row_count : nat32; limit : nat32; }; +type KinicBalance = record { balance_e8s : nat64 }; +type KinicDepositRequest = record { + amount_e8s : nat64; + expected_fee_e8s : nat64; +}; +type KinicDepositResult = record { + block_index : nat64; + amount_e8s : nat64; + balance_e8s : nat64; +}; +type KinicFundDatabaseCyclesRequest = record { + payment_amount_e8s : nat64; + database_id : text; + min_expected_cycles : nat64; +}; +type KinicFundDatabaseCyclesResult = record { + database_balance_cycles : nat64; + payment_amount_e8s : nat64; + kinic_balance_e8s : nat64; + amount_cycles : nat64; +}; +type KinicPendingOperation = record { + status : text; + kind : text; + operation_id : nat64; + created_at_ms : int64; + amount_e8s : nat64; + required_action : text; + ledger_block_index : opt nat64; + caller : text; +}; +type KinicPendingOperationsPage = record { + operations : vec KinicPendingOperation; + next_cursor_operation_id : opt nat64; +}; +type KinicPendingOperationsPageRequest = record { + limit : nat32; + cursor_operation_id : opt nat64; +}; type OutgoingLinksRequest = record { path : text; limit : nat32; database_id : text }; type LinkEdge = record { updated_at : int64; @@ -248,6 +287,117 @@ type ListNodesRequest = record { database_id : text; prefix : text; }; +type MarketCategoryGraph = record { + edges : vec MarketCategoryGraphEdge; + nodes : vec MarketCategoryGraphNode; +}; +type MarketCategoryGraphEdge = record { + source_category : text; + target_category : text; + link_count : nat64; +}; +type MarketCategoryGraphNode = record { node_count : nat64; category : text }; +type MarketCreateListingRequest = record { + llm_summary : opt text; + title : text; + summary_snapshot_revision : opt text; + description : text; + database_id : text; + price_e8s : nat64; + sample_excerpts_json : text; + tags_json : text; +}; +type MarketEntitlement = record { + status : text; + purchased_at_ms : int64; + database_id : text; + buyer_principal : text; + order_id : text; + listing_id : text; +}; +type MarketEntitlementPage = record { + next_cursor : opt text; + entitlements : vec MarketEntitlement; +}; +type MarketListing = record { + status : MarketListingStatus; + llm_summary : opt text; + title : text; + summary_snapshot_revision : opt text; + report_count : nat64; + description : text; + updated_at_ms : int64; + created_at_ms : int64; + seller_principal : text; + purchase_count : nat64; + database_id : text; + listing_id : text; + price_e8s : nat64; + revision : nat64; + sample_excerpts_json : text; + tags_json : text; +}; +type MarketListingDetail = record { + listing : MarketListing; + preview : MarketListingPreview; + verified_stats : MarketListingVerifiedStats; +}; +type MarketListingPage = record { + listings : vec MarketListing; + next_cursor : opt text; +}; +type MarketListingPreview = record { + excerpts : vec MarketPreviewExcerpt; + top_level_paths : vec text; + preview_stale : bool; + category_graph : MarketCategoryGraph; +}; +type MarketListingStatus = variant { Paused; Active; Draft }; +type MarketListingVerifiedStats = record { + source_chars : nat64; + last_content_updated_at_ms : opt int64; + total_nodes : nat64; + folder_nodes : nat64; + logical_size_bytes : nat64; + source_nodes : nat64; + wiki_nodes : nat64; + markdown_chars : nat64; + link_edges : nat64; +}; +type MarketOrder = record { + created_at_ms : int64; + seller_principal : text; + database_id : text; + buyer_principal : text; + order_id : text; + listing_id : text; + price_e8s : nat64; + listing_revision : nat64; +}; +type MarketOrderPage = record { + orders : vec MarketOrder; + next_cursor : opt text; +}; +type MarketPreviewExcerpt = record { etag : text; path : text; excerpt : text }; +type MarketPurchasePreview = record { + already_entitled : bool; + buyer_balance_e8s : nat64; + database_id : text; + listing_id : text; + price_e8s : nat64; +}; +type MarketPurchaseRequest = record { listing_id : text; price_e8s : nat64 }; +type MarketUpdateListingRequest = record { + llm_summary : opt text; + title : text; + summary_snapshot_revision : opt text; + description : text; + listing_id : text; + price_e8s : nat64; + expected_revision : nat64; + sample_excerpts_json : text; + tags_json : text; +}; type MemoryCapability = record { name : text; description : text }; type MemoryManifest = record { recommended_entrypoint : text; @@ -344,28 +494,41 @@ type Result = variant { Ok : WriteNodeResult; Err : text }; type Result_1 = variant { Ok; Err : text }; type Result_10 = variant { Ok : vec GlobNodeHit; Err : text }; type Result_11 = variant { Ok : vec LinkEdge; Err : text }; -type Result_12 = variant { Ok : vec ChildNode; Err : text }; -type Result_13 = variant { Ok : DatabaseCycleEntryPage; Err : text }; -type Result_14 = variant { Ok : vec DatabaseCyclesPendingPurchase; Err : text }; -type Result_15 = variant { Ok : vec DatabaseMember; Err : text }; -type Result_16 = variant { Ok : vec DatabaseSummary; Err : text }; -type Result_17 = variant { Ok : vec NodeEntry; Err : text }; -type Result_18 = variant { Ok : MkdirNodeResult; Err : text }; -type Result_19 = variant { Ok : MoveNodeResult; Err : text }; +type Result_12 = variant { Ok : KinicDepositResult; Err : text }; +type Result_13 = variant { Ok : KinicFundDatabaseCyclesResult; Err : text }; +type Result_14 = variant { Ok : KinicBalance; Err : text }; +type Result_15 = variant { Ok : KinicPendingOperationsPage; Err : text }; +type Result_16 = variant { Ok : vec ChildNode; Err : text }; +type Result_17 = variant { Ok : DatabaseCycleEntryPage; Err : text }; +type Result_18 = variant { Ok : vec DatabaseCyclesPendingPurchase; Err : text }; +type Result_19 = variant { Ok : vec DatabaseMember; Err : text }; type Result_2 = variant { Ok : DatabaseArchiveInfo; Err : text }; -type Result_20 = variant { Ok : CyclesPurchaseResult; Err : text }; -type Result_21 = variant { Ok : QueryContext; Err : text }; -type Result_22 = variant { Ok : IndexSqlJsonQueryResult; Err : text }; -type Result_23 = variant { Ok : DatabaseArchiveChunk; Err : text }; -type Result_24 = variant { Ok : opt Node; Err : text }; -type Result_25 = variant { Ok : opt NodeContext; Err : text }; -type Result_26 = variant { Ok : vec SearchNodeHit; Err : text }; -type Result_27 = variant { Ok : StorageBillingBatchResult; Err : text }; -type Result_28 = variant { Ok : SourceEvidence; Err : text }; -type Result_29 = variant { Ok : vec WriteNodeResult; Err : text }; +type Result_20 = variant { Ok : vec DatabaseSummary; Err : text }; +type Result_21 = variant { Ok : vec NodeEntry; Err : text }; +type Result_22 = variant { Ok : nat64; Err : text }; +type Result_23 = variant { Ok : MarketListing; Err : text }; +type Result_24 = variant { Ok : MarketListingDetail; Err : text }; +type Result_25 = variant { Ok : vec MarketListing; Err : text }; +type Result_26 = variant { Ok : MarketEntitlementPage; Err : text }; +type Result_27 = variant { Ok : MarketListingPage; Err : text }; +type Result_28 = variant { Ok : MarketOrderPage; Err : text }; +type Result_29 = variant { Ok : MarketPurchasePreview; Err : text }; type Result_3 = variant { Ok : OpsAnswerSessionCheckResult; Err : text }; -type Result_30 = variant { Ok : WriteSourceForGenerationResult; Err : text }; +type Result_30 = variant { Ok : MarketOrder; Err : text }; +type Result_31 = variant { Ok : MkdirNodeResult; Err : text }; +type Result_32 = variant { Ok : MoveNodeResult; Err : text }; +type Result_33 = variant { Ok : CyclesPurchaseResult; Err : text }; +type Result_34 = variant { Ok : QueryContext; Err : text }; +type Result_35 = variant { Ok : IndexSqlJsonQueryResult; Err : text }; +type Result_36 = variant { Ok : DatabaseArchiveChunk; Err : text }; +type Result_37 = variant { Ok : opt Node; Err : text }; +type Result_38 = variant { Ok : opt NodeContext; Err : text }; +type Result_39 = variant { Ok : vec SearchNodeHit; Err : text }; type Result_4 = variant { Ok : CreateDatabaseResult; Err : text }; +type Result_40 = variant { Ok : StorageBillingBatchResult; Err : text }; +type Result_41 = variant { Ok : SourceEvidence; Err : text }; +type Result_42 = variant { Ok : vec WriteNodeResult; Err : text }; +type Result_43 = variant { Ok : WriteSourceForGenerationResult; Err : text }; type Result_5 = variant { Ok : DeleteNodeResult; Err : text }; type Result_6 = variant { Ok : EditNodeResult; Err : text }; type Result_7 = variant { Ok : ExportSnapshotResponse; Err : text }; @@ -504,37 +667,55 @@ service : (CyclesBillingConfig) -> { Icrc21ConsentMessageResponse, ); incoming_links : (IncomingLinksRequest) -> (Result_11) query; - list_children : (ListChildrenRequest) -> (Result_12) query; - list_database_cycle_entries : (text, opt nat64, nat32) -> (Result_13) query; - list_database_cycles_pending_purchases : (text) -> (Result_14) query; - list_database_members : (text) -> (Result_15) query; - list_databases : () -> (Result_16) query; - list_nodes : (ListNodesRequest) -> (Result_17) query; + kinic_deposit_balance : (KinicDepositRequest) -> (Result_12); + kinic_fund_database_cycles : (KinicFundDatabaseCyclesRequest) -> (Result_13); + kinic_get_balance : () -> (Result_14) query; + kinic_list_pending_operations : (KinicPendingOperationsPageRequest) -> ( + Result_15, + ) query; + list_children : (ListChildrenRequest) -> (Result_16) query; + list_database_cycle_entries : (text, opt nat64, nat32) -> (Result_17) query; + list_database_cycles_pending_purchases : (text) -> (Result_18) query; + list_database_members : (text) -> (Result_19) query; + list_databases : () -> (Result_20) query; + list_nodes : (ListNodesRequest) -> (Result_21) query; + market_count_active_entitlements : (text) -> (Result_22) query; + market_create_listing : (MarketCreateListingRequest) -> (Result_23); + market_get_listing : (text) -> (Result_24) query; + market_list_database_listings : (text) -> (Result_25) query; + market_list_entitlements : (opt text, nat32) -> (Result_26) query; + market_list_listings : (opt text, nat32) -> (Result_27) query; + market_list_orders : (opt text, nat32) -> (Result_28) query; + market_pause_listing : (text) -> (Result_23); + market_preview_purchase : (text) -> (Result_29) query; + market_publish_listing : (text) -> (Result_23); + market_purchase_access : (MarketPurchaseRequest) -> (Result_30); + market_update_listing : (MarketUpdateListingRequest) -> (Result_23); memory_manifest : () -> (MemoryManifest) query; - mkdir_node : (MkdirNodeRequest) -> (Result_18); - move_node : (MoveNodeRequest) -> (Result_19); + mkdir_node : (MkdirNodeRequest) -> (Result_31); + move_node : (MoveNodeRequest) -> (Result_32); multi_edit_node : (MultiEditNodeRequest) -> (Result_6); outgoing_links : (OutgoingLinksRequest) -> (Result_11) query; - purchase_database_cycles : (DatabaseCyclesPurchaseRequest) -> (Result_20); - query_context : (QueryContextRequest) -> (Result_21) query; - query_index_sql_json : (text, nat32) -> (Result_22) query; - read_database_archive_chunk : (text, nat64, nat32) -> (Result_23) query; - read_node : (text, text) -> (Result_24) query; - read_node_context : (NodeContextRequest) -> (Result_25) query; + purchase_database_cycles : (DatabaseCyclesPurchaseRequest) -> (Result_33); + query_context : (QueryContextRequest) -> (Result_34) query; + query_index_sql_json : (text, nat32) -> (Result_35) query; + read_database_archive_chunk : (text, nat64, nat32) -> (Result_36) query; + read_node : (text, text) -> (Result_37) query; + read_node_context : (NodeContextRequest) -> (Result_38) query; rename_database : (RenameDatabaseRequest) -> (Result_1); revoke_database_access : (text, text) -> (Result_1); - search_node_paths : (SearchNodePathsRequest) -> (Result_26) query; - search_nodes : (SearchNodesRequest) -> (Result_26) query; + search_node_paths : (SearchNodePathsRequest) -> (Result_39) query; + search_nodes : (SearchNodesRequest) -> (Result_39) query; settle_database_storage_charges_batch : (StorageBillingBatchRequest) -> ( - Result_27, + Result_40, ); - source_evidence : (SourceEvidenceRequest) -> (Result_28) query; + source_evidence : (SourceEvidenceRequest) -> (Result_41) query; status : (text) -> (Status) query; update_cycles_billing_config : (CyclesBillingConfigUpdate) -> (Result_1); write_database_restore_chunk : (DatabaseRestoreChunkRequest) -> (Result_1); write_node : (WriteNodeRequest) -> (Result); - write_nodes : (WriteNodesRequest) -> (Result_29); + write_nodes : (WriteNodesRequest) -> (Result_42); write_source_for_generation : (WriteSourceForGenerationRequest) -> ( - Result_30, + Result_43, ); } diff --git a/crates/vfs_client/src/lib.rs b/crates/vfs_client/src/lib.rs index db26029a..011c619f 100644 --- a/crates/vfs_client/src/lib.rs +++ b/crates/vfs_client/src/lib.rs @@ -18,12 +18,17 @@ use vfs_types::{ DeleteDatabaseRequest, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, GlobNodeHit, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, - IncomingLinksRequest, LinkEdge, ListChildrenRequest, ListNodesRequest, MemoryManifest, - MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, - MultiEditNodeResult, Node, NodeContext, NodeContextRequest, NodeEntry, OutgoingLinksRequest, - QueryContext, QueryContextRequest, RenameDatabaseRequest, SearchNodeHit, - SearchNodePathsRequest, SearchNodesRequest, SourceEvidence, SourceEvidenceRequest, Status, - WriteNodeRequest, WriteNodeResult, WriteNodesRequest, + IncomingLinksRequest, KinicBalance, KinicDepositRequest, KinicDepositResult, + KinicFundDatabaseCyclesRequest, KinicFundDatabaseCyclesResult, KinicPendingOperationsPage, + KinicPendingOperationsPageRequest, LinkEdge, ListChildrenRequest, ListNodesRequest, + MarketCreateListingRequest, MarketEntitlementPage, MarketListing, MarketListingPage, + MarketOrder, MarketOrderPage, MarketPurchasePreview, MarketPurchaseRequest, + MarketUpdateListingRequest, MemoryManifest, MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, + MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeContext, + NodeContextRequest, NodeEntry, OutgoingLinksRequest, QueryContext, QueryContextRequest, + RenameDatabaseRequest, SearchNodeHit, SearchNodePathsRequest, SearchNodesRequest, + SourceEvidence, SourceEvidenceRequest, Status, WriteNodeRequest, WriteNodeResult, + WriteNodesRequest, }; #[async_trait] @@ -76,6 +81,116 @@ pub trait VfsApi: Sync { "list_database_cycles_pending_purchases is not implemented by this client" )) } + async fn kinic_get_balance(&self) -> Result { + Err(anyhow!( + "kinic_get_balance is not implemented by this client" + )) + } + async fn kinic_deposit_balance( + &self, + _request: KinicDepositRequest, + ) -> Result { + Err(anyhow!( + "kinic_deposit_balance is not implemented by this client" + )) + } + async fn kinic_fund_database_cycles( + &self, + _request: KinicFundDatabaseCyclesRequest, + ) -> Result { + Err(anyhow!( + "kinic_fund_database_cycles is not implemented by this client" + )) + } + async fn kinic_list_pending_operations( + &self, + _request: KinicPendingOperationsPageRequest, + ) -> Result { + Err(anyhow!( + "kinic_list_pending_operations is not implemented by this client" + )) + } + async fn market_create_listing( + &self, + _request: MarketCreateListingRequest, + ) -> Result { + Err(anyhow!( + "market_create_listing is not implemented by this client" + )) + } + async fn market_update_listing( + &self, + _request: MarketUpdateListingRequest, + ) -> Result { + Err(anyhow!( + "market_update_listing is not implemented by this client" + )) + } + async fn market_publish_listing(&self, _listing_id: &str) -> Result { + Err(anyhow!( + "market_publish_listing is not implemented by this client" + )) + } + async fn market_pause_listing(&self, _listing_id: &str) -> Result { + Err(anyhow!( + "market_pause_listing is not implemented by this client" + )) + } + async fn market_list_listings( + &self, + _cursor: Option, + _limit: u32, + ) -> Result { + Err(anyhow!( + "market_list_listings is not implemented by this client" + )) + } + async fn market_list_database_listings( + &self, + _database_id: &str, + ) -> Result> { + Err(anyhow!( + "market_list_database_listings is not implemented by this client" + )) + } + async fn market_get_listing(&self, _listing_id: &str) -> Result { + Err(anyhow!( + "market_get_listing is not implemented by this client" + )) + } + async fn market_preview_purchase(&self, _listing_id: &str) -> Result { + Err(anyhow!( + "market_preview_purchase is not implemented by this client" + )) + } + async fn market_purchase_access(&self, _request: MarketPurchaseRequest) -> Result { + Err(anyhow!( + "market_purchase_access is not implemented by this client" + )) + } + async fn market_list_entitlements( + &self, + _cursor: Option, + _limit: u32, + ) -> Result { + Err(anyhow!( + "market_list_entitlements is not implemented by this client" + )) + } + async fn market_list_orders( + &self, + _cursor: Option, + _limit: u32, + ) -> Result { + Err(anyhow!( + "market_list_orders is not implemented by this client" + )) + } + async fn market_count_active_entitlements(&self, _database_id: &str) -> Result { + Err(anyhow!( + "market_count_active_entitlements is not implemented by this client" + )) + } async fn get_cycles_billing_config(&self) -> Result { Err(anyhow!( "get_cycles_billing_config is not implemented by this client" @@ -493,6 +608,136 @@ impl VfsApi for CanisterVfsClient { result.map_err(|error| anyhow!(error)) } + async fn kinic_get_balance(&self) -> Result { + let result: Result = self.query("kinic_get_balance", &()).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn kinic_deposit_balance( + &self, + request: KinicDepositRequest, + ) -> Result { + let result: Result = + self.update("kinic_deposit_balance", &request).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn kinic_fund_database_cycles( + &self, + request: KinicFundDatabaseCyclesRequest, + ) -> Result { + let result: Result = + self.update("kinic_fund_database_cycles", &request).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn kinic_list_pending_operations( + &self, + request: KinicPendingOperationsPageRequest, + ) -> Result { + let result: Result = self + .query("kinic_list_pending_operations", &request) + .await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_create_listing( + &self, + request: MarketCreateListingRequest, + ) -> Result { + let result: Result = + self.update("market_create_listing", &request).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_update_listing( + &self, + request: MarketUpdateListingRequest, + ) -> Result { + let result: Result = + self.update("market_update_listing", &request).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_publish_listing(&self, listing_id: &str) -> Result { + let result: Result = self + .update("market_publish_listing", &listing_id.to_string()) + .await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_pause_listing(&self, listing_id: &str) -> Result { + let result: Result = self + .update("market_pause_listing", &listing_id.to_string()) + .await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_list_listings( + &self, + cursor: Option, + limit: u32, + ) -> Result { + let result: Result = + self.query2("market_list_listings", &cursor, &limit).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_list_database_listings(&self, database_id: &str) -> Result> { + let result: Result, String> = self + .query("market_list_database_listings", &database_id.to_string()) + .await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_get_listing(&self, listing_id: &str) -> Result { + let result: Result = self + .query("market_get_listing", &listing_id.to_string()) + .await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_preview_purchase(&self, listing_id: &str) -> Result { + let result: Result = self + .query("market_preview_purchase", &listing_id.to_string()) + .await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_purchase_access(&self, request: MarketPurchaseRequest) -> Result { + let result: Result = + self.update("market_purchase_access", &request).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_list_entitlements( + &self, + cursor: Option, + limit: u32, + ) -> Result { + let result: Result = self + .query2("market_list_entitlements", &cursor, &limit) + .await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_list_orders( + &self, + cursor: Option, + limit: u32, + ) -> Result { + let result: Result = + self.query2("market_list_orders", &cursor, &limit).await?; + result.map_err(|error| anyhow!(error)) + } + + async fn market_count_active_entitlements(&self, database_id: &str) -> Result { + let result: Result = self + .query("market_count_active_entitlements", &database_id.to_string()) + .await?; + result.map_err(|error| anyhow!(error)) + } + async fn grant_database_access( &self, database_id: &str, diff --git a/crates/vfs_runtime/Cargo.toml b/crates/vfs_runtime/Cargo.toml index 97ee1f1d..1e78c078 100644 --- a/crates/vfs_runtime/Cargo.toml +++ b/crates/vfs_runtime/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] candid = "0.10.26" +serde_json = "1.0.145" sha2 = "0.10.9" vfs_types = { package = "kinic-vfs-types", version = "0.1.0", path = "../vfs_types" } vfs-store = { path = "../vfs_store" } diff --git a/crates/vfs_runtime/migrations/index_db/011_to_latest.sql b/crates/vfs_runtime/migrations/index_db/011_to_latest.sql index 40810db2..ff991819 100644 --- a/crates/vfs_runtime/migrations/index_db/011_to_latest.sql +++ b/crates/vfs_runtime/migrations/index_db/011_to_latest.sql @@ -61,6 +61,115 @@ CREATE TABLE storage_billing_state ( CHECK (key = 'timer') ); +CREATE TABLE kinic_accounts ( + principal TEXT PRIMARY KEY, + balance_e8s INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE kinic_ledger ( + entry_id INTEGER PRIMARY KEY AUTOINCREMENT, + principal TEXT NOT NULL, + source TEXT NOT NULL, + kind TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + balance_after_e8s INTEGER NOT NULL, + counterparty TEXT, + listing_id TEXT, + order_id TEXT, + external_block_index INTEGER, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX kinic_ledger_principal_idx + ON kinic_ledger(principal, entry_id); + +CREATE UNIQUE INDEX kinic_ledger_external_block_idx + ON kinic_ledger(external_block_index) + WHERE external_block_index IS NOT NULL; + +CREATE TABLE kinic_pending_operations ( + operation_id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + caller TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + from_owner TEXT, + from_subaccount BLOB, + to_owner TEXT, + to_subaccount BLOB, + ledger_fee_e8s INTEGER, + operation_status TEXT NOT NULL, + external_block_index INTEGER, + ledger_created_at_time_ns INTEGER, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX kinic_pending_operations_caller_idx + ON kinic_pending_operations(caller, operation_id); + +CREATE UNIQUE INDEX kinic_pending_operations_external_block_kind_idx + ON kinic_pending_operations(external_block_index, kind) + WHERE external_block_index IS NOT NULL; + +CREATE TABLE market_listings ( + listing_id TEXT PRIMARY KEY, + seller_principal TEXT NOT NULL, + database_id TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + llm_summary TEXT, + summary_snapshot_revision TEXT, + sample_excerpts_json TEXT NOT NULL, + tags_json TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + status TEXT NOT NULL, + revision INTEGER NOT NULL, + purchase_count INTEGER NOT NULL, + report_count INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + FOREIGN KEY (database_id) REFERENCES databases(database_id) +); + +CREATE INDEX market_listings_status_idx + ON market_listings(status, listing_id); + +CREATE INDEX market_listings_database_idx + ON market_listings(database_id); + +CREATE TABLE market_orders ( + order_id TEXT PRIMARY KEY, + listing_id TEXT NOT NULL, + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + seller_principal TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + listing_revision INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX market_orders_buyer_idx + ON market_orders(buyer_principal, order_id); + +CREATE TABLE market_entitlements ( + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + listing_id TEXT NOT NULL, + order_id TEXT NOT NULL, + purchased_at_ms INTEGER NOT NULL, + status TEXT NOT NULL, + PRIMARY KEY (database_id, buyer_principal, listing_id), + FOREIGN KEY (database_id) REFERENCES databases(database_id) +); + +CREATE UNIQUE INDEX market_entitlements_database_buyer_active_idx + ON market_entitlements(database_id, buyer_principal) + WHERE status = 'active'; + +CREATE INDEX market_entitlements_buyer_idx + ON market_entitlements(buyer_principal, database_id); + INSERT INTO database_cycle_accounts (database_id, balance_cycles, suspended_at_ms, storage_charged_at_ms, created_at_ms, updated_at_ms) diff --git a/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql b/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql index fc65cc66..ebab806e 100644 --- a/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql +++ b/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql @@ -166,3 +166,112 @@ CREATE TABLE storage_billing_state ( updated_at_ms INTEGER NOT NULL, CHECK (key = 'timer') ); + +CREATE TABLE kinic_accounts ( + principal TEXT PRIMARY KEY, + balance_e8s INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +); + +CREATE TABLE kinic_ledger ( + entry_id INTEGER PRIMARY KEY AUTOINCREMENT, + principal TEXT NOT NULL, + source TEXT NOT NULL, + kind TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + balance_after_e8s INTEGER NOT NULL, + counterparty TEXT, + listing_id TEXT, + order_id TEXT, + external_block_index INTEGER, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX kinic_ledger_principal_idx + ON kinic_ledger(principal, entry_id); + +CREATE UNIQUE INDEX kinic_ledger_external_block_idx + ON kinic_ledger(external_block_index) + WHERE external_block_index IS NOT NULL; + +CREATE TABLE kinic_pending_operations ( + operation_id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + caller TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + from_owner TEXT, + from_subaccount BLOB, + to_owner TEXT, + to_subaccount BLOB, + ledger_fee_e8s INTEGER, + operation_status TEXT NOT NULL, + external_block_index INTEGER, + ledger_created_at_time_ns INTEGER, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX kinic_pending_operations_caller_idx + ON kinic_pending_operations(caller, operation_id); + +CREATE UNIQUE INDEX kinic_pending_operations_external_block_kind_idx + ON kinic_pending_operations(external_block_index, kind) + WHERE external_block_index IS NOT NULL; + +CREATE TABLE market_listings ( + listing_id TEXT PRIMARY KEY, + seller_principal TEXT NOT NULL, + database_id TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + llm_summary TEXT, + summary_snapshot_revision TEXT, + sample_excerpts_json TEXT NOT NULL, + tags_json TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + status TEXT NOT NULL, + revision INTEGER NOT NULL, + purchase_count INTEGER NOT NULL, + report_count INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + FOREIGN KEY (database_id) REFERENCES databases(database_id) +); + +CREATE INDEX market_listings_status_idx + ON market_listings(status, listing_id); + +CREATE INDEX market_listings_database_idx + ON market_listings(database_id); + +CREATE TABLE market_orders ( + order_id TEXT PRIMARY KEY, + listing_id TEXT NOT NULL, + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + seller_principal TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + listing_revision INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX market_orders_buyer_idx + ON market_orders(buyer_principal, order_id); + +CREATE TABLE market_entitlements ( + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + listing_id TEXT NOT NULL, + order_id TEXT NOT NULL, + purchased_at_ms INTEGER NOT NULL, + status TEXT NOT NULL, + PRIMARY KEY (database_id, buyer_principal, listing_id), + FOREIGN KEY (database_id) REFERENCES databases(database_id) +); + +CREATE UNIQUE INDEX market_entitlements_database_buyer_active_idx + ON market_entitlements(database_id, buyer_principal) + WHERE status = 'active'; + +CREATE INDEX market_entitlements_buyer_idx + ON market_entitlements(buyer_principal, database_id); diff --git a/crates/vfs_runtime/src/lib.rs b/crates/vfs_runtime/src/lib.rs index e7d10d63..592a7797 100644 --- a/crates/vfs_runtime/src/lib.rs +++ b/crates/vfs_runtime/src/lib.rs @@ -15,6 +15,7 @@ use crate::sqlite::{Connection, OptionalExtension, Transaction, params}; use candid::Principal; #[cfg(target_arch = "wasm32")] use ic_sqlite_vfs::{Db, DbError, DbHandle}; +use serde_json::Value; use sha2::{Digest, Sha256}; use vfs_store::FsStore; use vfs_types::{ @@ -24,7 +25,13 @@ use vfs_types::{ DeleteDatabaseRequest, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, GlobNodeHit, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, - IncomingLinksRequest, IndexSqlJsonQueryResult, LinkEdge, ListChildrenRequest, ListNodesRequest, + IncomingLinksRequest, IndexSqlJsonQueryResult, KinicBalance, KinicFundDatabaseCyclesRequest, + KinicFundDatabaseCyclesResult, KinicPendingOperation, KinicPendingOperationsPage, + KinicPendingOperationsPageRequest, LinkEdge, ListChildrenRequest, ListNodesRequest, + MarketCategoryGraph, MarketCreateListingRequest, MarketEntitlement, MarketEntitlementPage, + MarketListing, MarketListingDetail, MarketListingPage, MarketListingPreview, + MarketListingStatus, MarketListingVerifiedStats, MarketOrder, MarketOrderPage, + MarketPreviewExcerpt, MarketPurchasePreview, MarketPurchaseRequest, MarketUpdateListingRequest, MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeContext, NodeContextRequest, NodeEntry, NodeKind, OpsAnswerSessionCheckRequest, OpsAnswerSessionCheckResult, OpsAnswerSessionRequest, @@ -71,6 +78,10 @@ const INDEX_SCHEMA_VERSION_DIRECT_CYCLES: &str = concat!("database_index:024_", const INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX: &str = "database_index:025_cycles_pending_ledger_block_index"; const INDEX_SCHEMA_VERSION_STORAGE_BILLING_BATCH: &str = "database_index:026_storage_billing_batch"; +const INDEX_SCHEMA_VERSION_MARKETPLACE_CORE: &str = "database_index:027_marketplace_core"; +const INDEX_SCHEMA_VERSION_KINIC_EXTERNAL_BLOCK_INDEXES: &str = + "database_index:028_kinic_external_block_indexes"; +const INDEX_SCHEMA_VERSION_MARKETPLACE_PREVIEW: &str = "database_index:029_marketplace_preview"; const PENDING_DATABASE_MOUNT_ID: u16 = 0; const DATABASE_SCHEMA_VERSION: &str = "vfs_store:current"; const MIN_DATABASE_MOUNT_ID: u16 = 11; @@ -96,6 +107,7 @@ pub const STORAGE_CYCLES_PER_GIB_SECOND: u128 = 127_000; const DEFAULT_STORAGE_BILLING_BATCH_LIMIT: u32 = 100; const MAX_STORAGE_BILLING_BATCH_LIMIT: u32 = 1_000; const TIMER_STORAGE_BILLING_BATCH_LIMIT: u32 = 1_000; +const MAX_KINIC_PENDING_OPERATIONS_PAGE_LIMIT: u32 = 100; const STORAGE_BILLING_BULK_MIN_BATCH_LEN: usize = 50; const GIB_BYTES: u128 = 1024 * 1024 * 1024; const MAX_DATABASE_NAME_CHARS: usize = 80; @@ -107,6 +119,23 @@ const ANONYMOUS_PRINCIPAL: &str = "2vxsx-fae"; const CYCLES_OPERATION_STATUS_IN_FLIGHT: &str = "in_flight"; const CYCLES_OPERATION_STATUS_COMPLETED: &str = "completed"; const CYCLES_OPERATION_STATUS_AMBIGUOUS: &str = "ambiguous"; +const KINIC_OPERATION_STATUS_IN_FLIGHT: &str = "in_flight"; +const KINIC_OPERATION_STATUS_COMPLETED: &str = "completed"; +const KINIC_OPERATION_STATUS_AMBIGUOUS: &str = "ambiguous"; +const KINIC_LEDGER_SOURCE_INTERNAL: &str = "canister_internal"; +const KINIC_PENDING_KIND_DEPOSIT: &str = "deposit"; +const MARKET_LISTING_STATUS_DRAFT: &str = "draft"; +const MARKET_LISTING_STATUS_ACTIVE: &str = "active"; +const MARKET_LISTING_STATUS_PAUSED: &str = "paused"; +const MARKET_ENTITLEMENT_STATUS_ACTIVE: &str = "active"; +const GENERATED_LISTING_ID_PREFIX: &str = ""; +const GENERATED_ORDER_ID_PREFIX: &str = "order_"; +const GENERATED_MARKET_ID_HASH_CHARS: usize = 16; +const MAX_MARKET_TITLE_CHARS: usize = 160; +const MAX_MARKET_DESCRIPTION_CHARS: usize = 4_000; +const MAX_MARKET_JSON_CHARS: usize = 20_000; +const MAX_MARKET_PREVIEW_EXCERPTS: usize = 5; +const MAX_MARKET_PREVIEW_EXCERPT_CHARS: usize = 400; #[derive(Clone, Debug, PartialEq, Eq)] pub struct DatabaseMeta { @@ -165,6 +194,19 @@ pub struct DatabaseCyclesPurchaseStart { pub amount_cycles: u64, } +pub struct KinicDepositWithLedgerDetails<'a> { + pub caller: &'a str, + pub amount_e8s: u64, + pub ledger: CyclesPendingLedgerDetailsInput<'a>, + pub now: i64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct KinicDepositStart { + pub operation_id: u64, + pub amount_e8s: u64, +} + #[derive(Clone, Debug, PartialEq, Eq)] struct RestoreChunk { offset: u64, @@ -601,6 +643,16 @@ impl VfsService { params![database_id], ) .map_err(|error| error.to_string())?; + tx.execute( + "DELETE FROM market_entitlements WHERE database_id = ?1", + params![database_id], + ) + .map_err(|error| error.to_string())?; + tx.execute( + "DELETE FROM market_listings WHERE database_id = ?1", + params![database_id], + ) + .map_err(|error| error.to_string())?; tx.execute( "DELETE FROM database_members WHERE database_id = ?1", params![database_id], @@ -1050,1244 +1102,2196 @@ impl VfsService { }) } - pub fn require_database_write_cycles_available(&self, database_id: &str) -> Result<(), String> { + pub fn kinic_get_balance(&self, caller: &str) -> Result { + require_authenticated_principal(caller)?; self.read_index(|conn| { - let config = load_cycles_billing_config(conn)?; - require_database_write_cycles_available_for_conn(conn, database_id, &config) + Ok(KinicBalance { + balance_e8s: u64::try_from(load_kinic_balance(conn, caller)?) + .map_err(|error| error.to_string())?, + }) }) } - pub fn prepare_metered_update( + pub fn validate_kinic_fund_database_cycles( &self, - database_id: &str, caller: &str, - required_role: RequiredRole, - ) -> Result { + request: &KinicFundDatabaseCyclesRequest, + ) -> Result<(), String> { + require_authenticated_principal(caller)?; + let payment_amount = amount_to_i64(request.payment_amount_e8s)?; self.read_index(|conn| { - let role = load_database_status(conn, database_id).and_then(|_| { - load_member_role(conn, database_id, caller)? - .ok_or_else(|| format!("principal has no access to database: {database_id}")) - })?; - if !role_allows(role, required_role) { - return Err(format!( - "principal lacks required database role: {database_id}" - )); - } let config = load_cycles_billing_config(conn)?; - require_database_write_cycles_available_for_conn(conn, database_id, &config)?; - Ok(config) + let amount_cycles = cycles_for_payment_amount_e8s(request.payment_amount_e8s, &config)?; + validate_cycles_purchase_minimum(amount_cycles, request.min_expected_cycles)?; + validate_database_cycles_purchase_for_conn( + conn, + &request.database_id, + cycles_to_i64(amount_cycles)?, + )?; + let kinic_balance = load_kinic_balance(conn, caller)?; + if kinic_balance < payment_amount { + return Err("insufficient KINIC balance".to_string()); + } + Ok(()) }) } - pub fn check_database_write_cycles( - &self, - database_id: &str, - caller: &str, - ) -> Result<(), String> { - if caller == ANONYMOUS_PRINCIPAL { - return Err("anonymous caller not allowed".to_string()); - } - self.require_role(database_id, caller, RequiredRole::Writer)?; - self.require_database_write_cycles_available(database_id) - } - - pub fn charge_database_update( + pub fn kinic_fund_database_cycles( &self, - config: &CyclesBillingConfig, - database_id: &str, caller: &str, - method: &str, - cycles_delta: u128, + request: KinicFundDatabaseCyclesRequest, now: i64, - ) -> Result<(), String> { - let computed_charge = compute_update_charge(cycles_delta)?; - if computed_charge == 0 { - return Ok(()); - } + ) -> Result { + require_authenticated_principal(caller)?; + let payment_amount = amount_to_i64(request.payment_amount_e8s)?; self.write_index(|tx| { - charge_database_update_in_tx( + let config = load_cycles_billing_config(tx)?; + let amount_cycles = cycles_for_payment_amount_e8s(request.payment_amount_e8s, &config)?; + validate_cycles_purchase_minimum(amount_cycles, request.min_expected_cycles)?; + let amount_cycles_i64 = cycles_to_i64(amount_cycles)?; + validate_database_cycles_purchase_for_conn( tx, - DatabaseCharge { - database_id, + &request.database_id, + amount_cycles_i64, + )?; + complete_pending_database_activation(tx, &request.database_id, now)?; + + let kinic_balance = load_kinic_balance(tx, caller)?; + if kinic_balance < payment_amount { + return Err("insufficient KINIC balance".to_string()); + } + let next_kinic_balance = checked_balance_add(kinic_balance, -payment_amount)?; + let database_balance = database_balance_for_update(tx, &request.database_id)?; + let next_database_balance = checked_balance_add(database_balance, amount_cycles_i64)?; + + upsert_kinic_balance(tx, caller, next_kinic_balance, now)?; + update_database_cycles_balance( + tx, + &request.database_id, + next_database_balance, + &config, + now, + )?; + insert_kinic_ledger( + tx, + KinicLedgerInsert { + principal: caller, + source: KINIC_LEDGER_SOURCE_INTERNAL, + kind: "fund_database_cycles", + amount_e8s: -payment_amount, + balance_after_e8s: next_kinic_balance, + counterparty: Some(&request.database_id), + listing_id: None, + order_id: None, + external_block_index: None, + now, + }, + )?; + insert_database_ledger( + tx, + DatabaseLedgerInsert { + database_id: &request.database_id, + kind: "cycles_purchase", + amount_cycles: amount_cycles_i64, + balance_after_cycles: next_database_balance, + payment_amount_e8s: Some(payment_amount), caller, - method, - cycles_delta, + method: Some("kinic_fund_database_cycles"), + cycles_delta: None, + config: Some(&config), + ledger_block_index: None, now, - config, - computed_charge, }, - ) + )?; + Ok(KinicFundDatabaseCyclesResult { + payment_amount_e8s: request.payment_amount_e8s, + amount_cycles, + database_balance_cycles: u64::try_from(next_database_balance) + .map_err(|error| error.to_string())?, + kinic_balance_e8s: u64::try_from(next_kinic_balance) + .map_err(|error| error.to_string())?, + }) }) } - pub fn run_database_migrations(&self, database_id: &str) -> Result<(), String> { - let meta = self.database_meta(database_id)?; - self.run_database_migrations_for_meta(database_id, &meta) - } - - pub fn run_pending_database_migrations(&self, database_id: &str) -> Result<(), String> { - let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Pending])?; - self.run_database_migrations_for_meta(database_id, &meta) - } - - fn run_database_migrations_for_meta( + pub fn begin_kinic_deposit_with_ledger_details( &self, - database_id: &str, - meta: &DatabaseMeta, - ) -> Result<(), String> { - #[cfg(not(target_arch = "wasm32"))] - if let Some(parent) = Path::new(&meta.db_file_name).parent() { - create_dir_all(parent).map_err(|error| error.to_string())?; - } - let result = self.database_store(meta)?.run_fs_migrations(); - if result.is_ok() { - let _ = self.refresh_logical_size(database_id); + request: KinicDepositWithLedgerDetails<'_>, + ) -> Result { + require_authenticated_principal(request.caller)?; + if request.amount_e8s == 0 { + return Err("KINIC deposit amount must be positive".to_string()); } - result + let amount_e8s = amount_to_i64(request.amount_e8s)?; + let ledger_fee = amount_to_i64(request.ledger.ledger_fee_e8s)?; + let ledger_created_at_time = i64::try_from(request.ledger.ledger_created_at_time_ns) + .map_err(|_| "ledger created_at_time exceeds i64".to_string())?; + self.write_index(|tx| { + ensure_no_pending_kinic_deposit_for_caller(tx, request.caller)?; + let operation_id = insert_pending_kinic_operation( + tx, + PendingKinicOperationInsert { + kind: KINIC_PENDING_KIND_DEPOSIT, + caller: request.caller, + amount_e8s, + ledger: PendingCyclesLedgerDetails { + from_owner: request.ledger.from_owner, + from_subaccount: request.ledger.from_subaccount, + to_owner: request.ledger.to_owner, + to_subaccount: request.ledger.to_subaccount, + ledger_fee_e8s: ledger_fee, + ledger_created_at_time_ns: ledger_created_at_time, + }, + operation_status: KINIC_OPERATION_STATUS_IN_FLIGHT, + now: request.now, + }, + )?; + Ok(KinicDepositStart { + operation_id, + amount_e8s: request.amount_e8s, + }) + }) } - pub fn delete_database( + pub fn complete_kinic_deposit_ledger_transfer( &self, - request: DeleteDatabaseRequest, + operation_id: u64, caller: &str, - _now: i64, + amount_e8s: u64, + ledger_block_index: u64, ) -> Result<(), String> { - let database_id = request.database_id.as_str(); - self.require_role(database_id, caller, RequiredRole::Owner)?; - self.require_no_pending_cycles_operations(database_id)?; - let status = self.read_index(|conn| load_database_status(conn, database_id))?; - if !matches!(status, DatabaseStatus::Pending | DatabaseStatus::Active) { - return Err(format!( - "database is {}: {database_id}", - status_to_db(status) - )); - } - let meta = self.database_meta(database_id).ok(); - #[cfg(target_arch = "wasm32")] - let _ = &meta; - #[cfg(not(target_arch = "wasm32"))] - if let Some(meta) = &meta - && let Err(error) = remove_file(&meta.db_file_name) - && error.kind() != std::io::ErrorKind::NotFound - { - return Err(error.to_string()); - } - self.write_index(|conn| { - delete_database_index_rows(conn, database_id)?; + let amount_e8s = amount_to_i64(amount_e8s)?; + let ledger_block_index = i64::try_from(ledger_block_index) + .map_err(|_| "ledger block index exceeds i64".to_string())?; + self.write_index(|tx| { + let operation = load_required_pending_kinic_operation( + tx, + PendingKinicOperationMatch { + operation_id, + kind: KINIC_PENDING_KIND_DEPOSIT, + caller, + amount_e8s, + }, + )?; + require_pending_kinic_operation_status( + &operation, + &[KINIC_OPERATION_STATUS_IN_FLIGHT], + "complete KINIC deposit ledger transfer", + )?; + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + tx.execute( + "UPDATE kinic_pending_operations + SET operation_status = ?2, + external_block_index = ?3 + WHERE operation_id = ?1", + params![ + operation_id, + KINIC_OPERATION_STATUS_COMPLETED, + ledger_block_index + ], + ) + .map_err(|error| error.to_string())?; Ok(()) }) } - fn require_no_pending_cycles_operations(&self, database_id: &str) -> Result<(), String> { - self.read_index(|conn| { - let pending = first_database_cycles_pending_purchase_status(conn, database_id)?; - if let Some(pending) = pending { - return Err(format!( - "database has pending cycle operation: {database_id}; operation_id={}; status={}; required_action={}", - pending.operation_id, - pending.status, - pending.required_action - )); - } - Ok(()) + pub fn apply_kinic_deposit( + &self, + operation_id: u64, + caller: &str, + amount_e8s: u64, + now: i64, + ) -> Result { + let amount_i64 = amount_to_i64(amount_e8s)?; + self.write_index(|tx| { + let operation = load_required_pending_kinic_operation( + tx, + PendingKinicOperationMatch { + operation_id, + kind: KINIC_PENDING_KIND_DEPOSIT, + caller, + amount_e8s: amount_i64, + }, + )?; + require_pending_kinic_operation_status( + &operation, + &[KINIC_OPERATION_STATUS_COMPLETED], + "apply KINIC deposit", + )?; + let block_index = operation + .external_block_index + .ok_or_else(|| "completed KINIC deposit missing ledger block index".to_string())?; + let balance = load_kinic_balance(tx, caller)?; + let next_balance = checked_balance_add(balance, amount_i64)?; + upsert_kinic_balance(tx, caller, next_balance, now)?; + insert_kinic_ledger( + tx, + KinicLedgerInsert { + principal: caller, + source: KINIC_LEDGER_SOURCE_INTERNAL, + kind: "deposit", + amount_e8s: amount_i64, + balance_after_e8s: next_balance, + counterparty: None, + listing_id: None, + order_id: None, + external_block_index: Some(block_index), + now, + }, + )?; + delete_pending_kinic_operation(tx, operation_id)?; + Ok(KinicBalance { + balance_e8s: u64::try_from(next_balance).map_err(|error| error.to_string())?, + }) }) } - pub fn begin_database_archive( + pub fn mark_kinic_deposit_ambiguous( &self, - database_id: &str, + operation_id: u64, caller: &str, - now: i64, - ) -> Result { - self.require_role(database_id, caller, RequiredRole::Owner)?; - self.require_no_pending_cycles_operations(database_id)?; - let meta = self.database_meta(database_id)?; - let size_bytes = self.database_size(&meta)?; - self.write_index(|conn| { - conn.execute( - "UPDATE databases - SET status = 'archiving', - updated_at_ms = ?2, - logical_size_bytes = ?3 - WHERE database_id = ?1", - params![ - database_id, - now, - i64::try_from(size_bytes).map_err(|error| error.to_string())? - ], + amount_e8s: u64, + ) -> Result<(), String> { + let amount_i64 = amount_to_i64(amount_e8s)?; + self.write_index(|tx| { + let operation = load_required_pending_kinic_operation( + tx, + PendingKinicOperationMatch { + operation_id, + kind: KINIC_PENDING_KIND_DEPOSIT, + caller, + amount_e8s: amount_i64, + }, + )?; + require_pending_kinic_operation_status( + &operation, + &[KINIC_OPERATION_STATUS_IN_FLIGHT], + "mark KINIC deposit ambiguous", + )?; + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + tx.execute( + "UPDATE kinic_pending_operations + SET operation_status = ?2 + WHERE operation_id = ?1", + params![operation_id, KINIC_OPERATION_STATUS_AMBIGUOUS], ) .map_err(|error| error.to_string())?; Ok(()) - })?; - Ok(DatabaseArchiveInfo { - database_id: database_id.to_string(), - size_bytes, }) } - pub fn read_database_archive_chunk( + pub fn cancel_kinic_deposit_after_ledger_error( &self, - database_id: &str, + operation_id: u64, caller: &str, - offset: u64, - max_bytes: u32, - ) -> Result, String> { - self.require_role(database_id, caller, RequiredRole::Owner)?; - let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Archiving])?; - if max_bytes == 0 { - return Ok(Vec::new()); - } - if max_bytes > MAX_ARCHIVE_CHUNK_BYTES { - return Err(format!( - "archive chunk size exceeds limit: {max_bytes} > {MAX_ARCHIVE_CHUNK_BYTES}" - )); - } - let size = meta.logical_size_bytes; - if offset >= size { - return Ok(Vec::new()); - } - let remaining = size.saturating_sub(offset); - let chunk_len = remaining.min(u64::from(max_bytes)); - self.database_export_chunk(&meta, offset, chunk_len) + amount_e8s: u64, + ) -> Result<(), String> { + let amount_i64 = amount_to_i64(amount_e8s)?; + self.write_index(|tx| { + let operation = load_required_pending_kinic_operation( + tx, + PendingKinicOperationMatch { + operation_id, + kind: KINIC_PENDING_KIND_DEPOSIT, + caller, + amount_e8s: amount_i64, + }, + )?; + require_pending_kinic_operation_status( + &operation, + &[KINIC_OPERATION_STATUS_IN_FLIGHT], + "cancel KINIC deposit", + )?; + delete_pending_kinic_operation(tx, operation_id) + }) } - pub fn finalize_database_archive( + pub fn kinic_list_pending_operations( &self, - database_id: &str, caller: &str, - snapshot_hash: Vec, + request: KinicPendingOperationsPageRequest, + ) -> Result { + require_authenticated_principal(caller)?; + let limit = clamp_kinic_pending_operations_page_limit(request.limit); + let config = self.cycles_billing_config()?; + self.read_index(|conn| { + let show_all = caller == config.billing_authority_id; + let raw = load_kinic_pending_operations( + conn, + caller, + show_all, + request.cursor_operation_id, + limit, + )?; + let has_next = raw.len() > limit as usize; + let operations = raw + .into_iter() + .take(limit as usize) + .map(PendingKinicOperationRaw::into_public) + .collect::, _>>()?; + let next_cursor_operation_id = if has_next { + operations.last().map(|operation| operation.operation_id) + } else { + None + }; + Ok(KinicPendingOperationsPage { + operations, + next_cursor_operation_id, + }) + }) + } + + pub fn market_create_listing( + &self, + caller: &str, + request: MarketCreateListingRequest, now: i64, - ) -> Result { - self.require_role(database_id, caller, RequiredRole::Owner)?; - let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Archiving])?; - validate_snapshot_hash(&snapshot_hash)?; - let actual_hash = self.database_sha256(&meta, meta.logical_size_bytes)?; - if actual_hash != snapshot_hash { - return Err("snapshot_hash does not match archived database bytes".to_string()); - } - self.write_index(|conn| { - conn.execute( - "UPDATE databases - SET status = 'archived', - snapshot_hash = ?2, - restore_size_bytes = NULL, - archived_at_ms = ?3, - updated_at_ms = ?3 - WHERE database_id = ?1", - params![database_id, snapshot_hash, now], + ) -> Result { + require_authenticated_principal(caller)?; + validate_market_create_listing_request(&request)?; + self.validate_market_listing_excerpts(&request.database_id, &request.sample_excerpts_json)?; + self.write_index(|tx| { + require_market_seller_can_list(tx, caller, &request.database_id)?; + let listing_id = unique_market_id( + tx, + "market_listings", + "listing_id", + GENERATED_LISTING_ID_PREFIX, + caller, + &request.database_id, + now, + )?; + tx.execute( + "INSERT INTO market_listings + (listing_id, seller_principal, database_id, title, description, + llm_summary, summary_snapshot_revision, sample_excerpts_json, + tags_json, price_e8s, status, revision, + purchase_count, report_count, created_at_ms, updated_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 1, 0, 0, ?12, ?12)", + params![ + listing_id, + caller, + request.database_id, + request.title, + request.description, + crate::sqlite::nullable_text_value(request.llm_summary), + crate::sqlite::nullable_text_value(request.summary_snapshot_revision), + request.sample_excerpts_json, + request.tags_json, + i64::try_from(request.price_e8s).map_err(|error| error.to_string())?, + MARKET_LISTING_STATUS_DRAFT, + now + ], ) .map_err(|error| error.to_string())?; - Ok(()) - })?; - Ok(meta) + load_market_listing_by_id(tx, &listing_id)? + .ok_or_else(|| "market listing insert failed".to_string()) + }) } - pub fn cancel_database_archive( + pub fn market_update_listing( &self, - database_id: &str, caller: &str, + request: MarketUpdateListingRequest, now: i64, - ) -> Result { - self.require_role(database_id, caller, RequiredRole::Owner)?; - let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Archiving])?; - self.write_index(|conn| { - conn.execute( - "UPDATE databases - SET status = 'active', - updated_at_ms = ?2 - WHERE database_id = ?1", - params![database_id, now], + ) -> Result { + require_authenticated_principal(caller)?; + validate_market_update_listing_request(&request)?; + let listing_database_id = self.read_index(|tx| { + let listing = load_market_listing_by_id(tx, &request.listing_id)? + .ok_or_else(|| "market listing not found".to_string())?; + require_market_listing_seller_or_admin(tx, caller, &listing)?; + if listing.revision != request.expected_revision { + return Err("market listing revision mismatch".to_string()); + } + Ok(listing.database_id) + })?; + self.validate_market_listing_excerpts(&listing_database_id, &request.sample_excerpts_json)?; + self.write_index(|tx| { + let listing = load_market_listing_by_id(tx, &request.listing_id)? + .ok_or_else(|| "market listing not found".to_string())?; + require_market_listing_seller_or_admin(tx, caller, &listing)?; + if listing.revision != request.expected_revision { + return Err("market listing revision mismatch".to_string()); + } + if listing.status == MarketListingStatus::Active { + require_market_seller_can_list( + tx, + &listing.seller_principal, + &listing.database_id, + )?; + } + tx.execute( + "UPDATE market_listings + SET title = ?2, + description = ?3, + llm_summary = ?4, + summary_snapshot_revision = ?5, + sample_excerpts_json = ?6, + tags_json = ?7, + price_e8s = ?8, + revision = revision + 1, + updated_at_ms = ?9 + WHERE listing_id = ?1", + params![ + request.listing_id, + request.title, + request.description, + crate::sqlite::nullable_text_value(request.llm_summary), + crate::sqlite::nullable_text_value(request.summary_snapshot_revision), + request.sample_excerpts_json, + request.tags_json, + i64::try_from(request.price_e8s).map_err(|error| error.to_string())?, + now + ], ) .map_err(|error| error.to_string())?; - Ok(()) - })?; - Ok(meta) + load_market_listing_by_id(tx, &listing.listing_id)? + .ok_or_else(|| "market listing update failed".to_string()) + }) } - pub fn begin_database_restore( + pub fn market_publish_listing( &self, - database_id: &str, caller: &str, - snapshot_hash: Vec, - size_bytes: u64, + listing_id: &str, now: i64, - ) -> Result { - self.begin_database_restore_session(database_id, caller, snapshot_hash, size_bytes, now) - .map(|restore| restore.meta) + ) -> Result { + self.market_set_listing_status(caller, listing_id, MARKET_LISTING_STATUS_ACTIVE, now) } - pub fn begin_database_restore_session( + pub fn market_pause_listing( &self, - database_id: &str, caller: &str, - snapshot_hash: Vec, - size_bytes: u64, + listing_id: &str, now: i64, - ) -> Result { - self.require_role(database_id, caller, RequiredRole::Owner)?; - validate_snapshot_hash(&snapshot_hash)?; - if size_bytes > MAX_DATABASE_SIZE_BYTES { - return Err(format!( - "database size exceeds limit: {size_bytes} > {MAX_DATABASE_SIZE_BYTES}" - )); - } - self.require_no_pending_cycles_operations(database_id)?; - let rollback = self.database_restore_rollback(database_id)?; - if rollback.status != DatabaseStatus::Archived { - return Err("database restore can only begin from archived status".to_string()); - } - let mount_id = rollback - .active_mount_id - .ok_or_else(|| format!("archived database has no mount: {database_id}"))?; + ) -> Result { + self.market_set_listing_status(caller, listing_id, MARKET_LISTING_STATUS_PAUSED, now) + } + + fn market_set_listing_status( + &self, + caller: &str, + listing_id: &str, + status: &str, + now: i64, + ) -> Result { + require_authenticated_principal(caller)?; self.write_index(|tx| { - record_database_restore_session(tx, &rollback, now)?; + let listing = load_market_listing_by_id(tx, listing_id)? + .ok_or_else(|| "market listing not found".to_string())?; + require_market_listing_seller_or_admin(tx, caller, &listing)?; + if status == MARKET_LISTING_STATUS_ACTIVE { + require_market_seller_can_list( + tx, + &listing.seller_principal, + &listing.database_id, + )?; + } tx.execute( - "DELETE FROM database_restore_chunks WHERE database_id = ?1", - params![database_id], + "UPDATE market_listings + SET status = ?2, + revision = revision + 1, + updated_at_ms = ?3 + WHERE listing_id = ?1", + params![listing_id, status, now], ) .map_err(|error| error.to_string())?; - tx.execute( - "UPDATE databases - SET status = 'restoring', - active_mount_id = ?2, - snapshot_hash = ?3, - archived_at_ms = NULL, - restore_size_bytes = ?4, - updated_at_ms = ?5 - WHERE database_id = ?1", + load_market_listing_by_id(tx, listing_id)? + .ok_or_else(|| "market listing status update failed".to_string()) + }) + } + + pub fn market_list_listings( + &self, + cursor: Option, + limit: u32, + ) -> Result { + let limit = page_limit(limit); + let after = cursor.unwrap_or_default(); + self.read_index(|conn| { + let mut stmt = conn + .prepare( + "SELECT l.listing_id, l.seller_principal, l.database_id, l.title, l.description, + l.llm_summary, l.summary_snapshot_revision, l.sample_excerpts_json, + l.tags_json, l.price_e8s, l.status, l.revision, + l.purchase_count, l.report_count, l.created_at_ms, l.updated_at_ms + FROM market_listings l + JOIN databases d ON d.database_id = l.database_id + JOIN database_members m + ON m.database_id = l.database_id + AND m.principal = l.seller_principal + AND m.role = 'owner' + WHERE l.status = ?1 + AND d.status = ?2 + AND l.listing_id > ?3 + ORDER BY l.listing_id ASC + LIMIT ?4", + ) + .map_err(|error| error.to_string())?; + let mut listings = crate::sqlite::query_map( + &mut stmt, params![ - database_id, - i64::from(mount_id), - snapshot_hash, - i64::try_from(size_bytes).map_err(|error| error.to_string())?, - now + MARKET_LISTING_STATUS_ACTIVE, + status_to_db(DatabaseStatus::Active), + after, + i64::from(limit) + 1 ], + map_market_listing, ) .map_err(|error| error.to_string())?; - Ok(()) - })?; - let meta = self.database_meta_allowing_restoring(database_id)?; - #[cfg(not(target_arch = "wasm32"))] - let _ = remove_file(&meta.db_file_name); - Ok(DatabaseRestoreBegin { meta, rollback }) + let next_cursor = if listings.len() > limit as usize { + listings.pop(); + listings.last().map(|listing| listing.listing_id.clone()) + } else { + None + }; + Ok(MarketListingPage { + listings, + next_cursor, + }) + }) } - pub fn rollback_database_restore_begin( + pub fn market_list_database_listings( &self, - rollback: DatabaseRestoreRollback, - now: i64, - ) -> Result<(), String> { - self.write_index(|tx| { - let current_status = load_database_status(tx, &rollback.database_id)?; - if current_status != DatabaseStatus::Restoring { + caller: &str, + database_id: &str, + ) -> Result, String> { + require_authenticated_principal(caller)?; + validate_database_id(database_id)?; + self.read_index(|conn| { + let role = load_member_role(conn, database_id, caller)? + .ok_or_else(|| format!("principal has no access to database: {database_id}"))?; + if role != DatabaseRole::Owner { return Err(format!( - "database restore rollback requires restoring status: {}", - rollback.database_id + "principal lacks required database role: {database_id}" )); } - tx.execute( - "DELETE FROM database_restore_chunks WHERE database_id = ?1", - params![rollback.database_id], - ) - .map_err(|error| error.to_string())?; - restore_database_state(tx, &rollback, now)?; - Ok(()) + let mut stmt = conn + .prepare( + "SELECT listing_id, seller_principal, database_id, title, description, + llm_summary, summary_snapshot_revision, sample_excerpts_json, + tags_json, price_e8s, status, revision, + purchase_count, report_count, created_at_ms, updated_at_ms + FROM market_listings + WHERE database_id = ?1 + ORDER BY updated_at_ms DESC, listing_id ASC", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map(&mut stmt, params![database_id], map_market_listing) + .map_err(|error| error.to_string()) }) } - pub fn cancel_database_restore( + pub fn market_get_listing( &self, - database_id: &str, caller: &str, - now: i64, - ) -> Result { - self.require_role(database_id, caller, RequiredRole::Owner)?; - let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Restoring])?; - let rollback = self.database_restore_session(database_id)?; - #[cfg(not(target_arch = "wasm32"))] - if let Err(error) = remove_file(&meta.db_file_name) - && error.kind() != std::io::ErrorKind::NotFound - { - return Err(error.to_string()); - } - self.write_index(|tx| { - tx.execute( - "DELETE FROM database_restore_chunks WHERE database_id = ?1", - params![database_id], - ) - .map_err(|error| error.to_string())?; - restore_database_state(tx, &rollback, now)?; - Ok(()) + listing_id: &str, + ) -> Result { + let listing = self.read_index(|conn| { + let listing = load_market_listing_by_id(conn, listing_id)? + .ok_or_else(|| "market listing not found".to_string())?; + if require_market_listing_purchasable(conn, &listing).is_ok() { + return Ok(listing); + } + require_market_listing_seller_or_admin(conn, caller, &listing)?; + Ok(listing) })?; - Ok(meta) + self.market_listing_detail(listing) } - pub fn write_database_restore_chunk( + fn market_listing_detail(&self, listing: MarketListing) -> Result { + let Ok(meta) = self.database_meta_with_statuses( + &listing.database_id, + &[ + DatabaseStatus::Active, + DatabaseStatus::Archiving, + DatabaseStatus::Restoring, + ], + ) else { + return Ok(empty_market_listing_detail(listing)); + }; + let store = self.database_store(&meta)?; + let (verified_stats, mut preview) = store.marketplace_preview()?; + let (excerpts, stale) = self.load_verified_market_preview_excerpts( + &listing.database_id, + &listing.sample_excerpts_json, + )?; + preview.excerpts = excerpts; + preview.preview_stale = stale; + Ok(MarketListingDetail { + listing, + verified_stats, + preview, + }) + } + + fn validate_market_listing_excerpts( &self, database_id: &str, - caller: &str, - offset: u64, - bytes: &[u8], + sample_excerpts_json: &str, ) -> Result<(), String> { - self.require_role(database_id, caller, RequiredRole::Owner)?; - if bytes.len() > MAX_RESTORE_CHUNK_BYTES { - return Err(format!( - "restore chunk size exceeds limit: {} > {MAX_RESTORE_CHUNK_BYTES}", - bytes.len() - )); + let (excerpts, stale) = + self.load_verified_market_preview_excerpts(database_id, sample_excerpts_json)?; + if stale || excerpts.len() != parse_market_preview_excerpts(sample_excerpts_json)?.len() { + return Err( + "market listing sample excerpts must match existing wiki nodes".to_string(), + ); } - let _meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Restoring])?; - let expected_size = self.restore_size_bytes(database_id)?; - let end = offset - .checked_add(bytes.len() as u64) - .ok_or_else(|| "restore chunk range overflows u64".to_string())?; - if end > expected_size { - return Err(format!( - "restore chunk exceeds expected size: end {end} > {expected_size}" - )); + Ok(()) + } + + fn load_verified_market_preview_excerpts( + &self, + database_id: &str, + sample_excerpts_json: &str, + ) -> Result<(Vec, bool), String> { + let excerpts = parse_market_preview_excerpts(sample_excerpts_json)?; + if excerpts.is_empty() { + return Ok((Vec::new(), false)); } - self.write_index(|conn| { - conn.execute( - "INSERT OR REPLACE INTO database_restore_chunks - (database_id, offset_bytes, end_bytes, bytes) - VALUES (?1, ?2, ?3, ?4)", - params![ - database_id, - i64::try_from(offset).map_err(|error| error.to_string())?, - i64::try_from(end).map_err(|error| error.to_string())?, - bytes.to_vec() - ], - ) - .map_err(|error| error.to_string())?; - Ok(()) + let meta = self.database_meta(database_id)?; + let store = self.database_store(&meta)?; + let mut verified = Vec::new(); + let mut stale = false; + for excerpt in excerpts { + match store.read_node(&excerpt.path)? { + Some(node) + if node.kind == NodeKind::File + && node.etag == excerpt.etag + && node.content.contains(&excerpt.excerpt) => + { + verified.push(excerpt); + } + _ => stale = true, + } + } + Ok((verified, stale)) + } + + pub fn market_preview_purchase( + &self, + caller: &str, + listing_id: &str, + ) -> Result { + require_authenticated_principal(caller)?; + self.read_index(|conn| { + let listing = load_market_listing_by_id(conn, listing_id)? + .ok_or_else(|| "market listing not found".to_string())?; + require_market_listing_purchasable(conn, &listing)?; + Ok(MarketPurchasePreview { + listing_id: listing.listing_id.clone(), + database_id: listing.database_id.clone(), + price_e8s: listing.price_e8s, + buyer_balance_e8s: u64::try_from(load_kinic_balance(conn, caller)?) + .map_err(|error| error.to_string())?, + already_entitled: has_active_market_entitlement( + conn, + &listing.database_id, + caller, + )?, + }) }) } - pub fn finalize_database_restore( + pub fn market_purchase_access( &self, - database_id: &str, caller: &str, + request: MarketPurchaseRequest, now: i64, - ) -> Result { - self.require_role(database_id, caller, RequiredRole::Owner)?; - let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Restoring])?; - let expected_size = self.restore_size_bytes(database_id)?; - let chunks = self.read_index(|conn| load_restore_chunks(conn, database_id))?; - if !restore_chunks_cover_expected_size(&chunks, expected_size)? { - return Err(format!( - "restore chunks are incomplete for expected size {expected_size} bytes" - )); - } - let expected_hash = self.restore_snapshot_hash(database_id)?; - let mut hasher = Sha256::new(); - let mut checksum = FNV1A64_OFFSET; - for chunk in &chunks { - hasher.update(&chunk.bytes); - checksum = fnv1a64_update(checksum, &chunk.bytes); - } - let actual_hash = hasher.finalize().to_vec(); - if actual_hash != expected_hash { - return Err("snapshot_hash does not match restored database bytes".to_string()); - } - self.import_database_bytes(&meta, expected_size, checksum, &chunks)?; - self.database_store(&meta)?.run_fs_migrations()?; + ) -> Result { + require_authenticated_principal(caller)?; self.write_index(|tx| { + let listing = load_market_listing_by_id(tx, &request.listing_id)? + .ok_or_else(|| "market listing not found".to_string())?; + require_market_listing_purchasable(tx, &listing)?; + if listing.price_e8s != request.price_e8s { + return Err("market listing price mismatch".to_string()); + } + if caller == listing.seller_principal { + return Err("market seller cannot purchase own listing".to_string()); + } + if has_active_market_entitlement(tx, &listing.database_id, caller)? { + return Err("active entitlement already exists".to_string()); + } + let price_i64 = amount_to_i64(request.price_e8s)?; + let buyer_balance = load_kinic_balance(tx, caller)?; + if buyer_balance < price_i64 { + return Err("insufficient KINIC balance".to_string()); + } + let seller_balance = load_kinic_balance(tx, &listing.seller_principal)?; + let buyer_next = checked_balance_add(buyer_balance, -price_i64)?; + let seller_next = checked_balance_add(seller_balance, price_i64)?; + let order_id = unique_market_id( + tx, + "market_orders", + "order_id", + GENERATED_ORDER_ID_PREFIX, + caller, + &listing.listing_id, + now, + )?; + upsert_kinic_balance(tx, caller, buyer_next, now)?; + upsert_kinic_balance(tx, &listing.seller_principal, seller_next, now)?; + insert_kinic_ledger( + tx, + KinicLedgerInsert { + principal: caller, + source: KINIC_LEDGER_SOURCE_INTERNAL, + kind: "purchase", + amount_e8s: -price_i64, + balance_after_e8s: buyer_next, + counterparty: Some(&listing.seller_principal), + listing_id: Some(&listing.listing_id), + order_id: Some(&order_id), + external_block_index: None, + now, + }, + )?; + insert_kinic_ledger( + tx, + KinicLedgerInsert { + principal: &listing.seller_principal, + source: KINIC_LEDGER_SOURCE_INTERNAL, + kind: "sale", + amount_e8s: price_i64, + balance_after_e8s: seller_next, + counterparty: Some(caller), + listing_id: Some(&listing.listing_id), + order_id: Some(&order_id), + external_block_index: None, + now, + }, + )?; tx.execute( - "DELETE FROM database_restore_chunks WHERE database_id = ?1", - params![database_id], - ) - .map_err(|error| error.to_string())?; - tx.execute( - "DELETE FROM database_restore_sessions WHERE database_id = ?1", - params![database_id], + "INSERT INTO market_orders + (order_id, listing_id, database_id, buyer_principal, seller_principal, + price_e8s, listing_revision, created_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + order_id, + listing.listing_id, + listing.database_id, + caller, + listing.seller_principal, + price_i64, + i64::try_from(listing.revision).map_err(|error| error.to_string())?, + now + ], ) .map_err(|error| error.to_string())?; tx.execute( - "UPDATE databases - SET status = 'active', - logical_size_bytes = ?2, - restore_size_bytes = NULL, - updated_at_ms = ?3 - WHERE database_id = ?1", + "INSERT INTO market_entitlements + (database_id, buyer_principal, listing_id, order_id, purchased_at_ms, status) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", params![ - database_id, - i64::try_from(expected_size).map_err(|error| error.to_string())?, - now + listing.database_id, + caller, + listing.listing_id, + order_id, + now, + MARKET_ENTITLEMENT_STATUS_ACTIVE ], ) .map_err(|error| error.to_string())?; - Ok(()) - })?; - self.database_meta(database_id) - } - - pub fn grant_database_access( - &self, - database_id: &str, - caller: &str, - principal: &str, - role: DatabaseRole, - now: i64, - ) -> Result<(), String> { - self.require_role(database_id, caller, RequiredRole::Owner)?; - if caller == principal && role != DatabaseRole::Owner { - return Err("owner cannot downgrade own access".to_string()); - } - self.write_index(|conn| { - if !database_member_exists(conn, database_id, principal)? { - let member_count = database_member_count_for_conn(conn, database_id)?; - if member_count >= MAX_DATABASE_MEMBERS_PER_DATABASE { - return Err("too many database members".to_string()); - } - } - conn.execute( - "INSERT INTO database_members (database_id, principal, role, created_at_ms) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(database_id, principal) - DO UPDATE SET role = excluded.role", - params![database_id, principal, role_to_db(role), now], + tx.execute( + "UPDATE market_listings + SET purchase_count = purchase_count + 1, + updated_at_ms = ?2 + WHERE listing_id = ?1", + params![listing.listing_id, now], ) .map_err(|error| error.to_string())?; - Ok(()) + load_market_order_by_id(tx, &order_id)? + .ok_or_else(|| "market order insert failed".to_string()) }) } - pub fn rename_database( + pub fn market_list_entitlements( &self, - database_id: &str, caller: &str, - name: &str, - now: i64, - ) -> Result<(), String> { - self.require_role(database_id, caller, RequiredRole::Owner)?; - self.database_meta(database_id)?; - let name = normalize_database_name(name)?; - self.write_index(|conn| { - conn.execute( - "UPDATE databases - SET name = ?2, - updated_at_ms = ?3 - WHERE database_id = ?1", - params![database_id, name, now], + cursor: Option, + limit: u32, + ) -> Result { + require_authenticated_principal(caller)?; + let limit = page_limit(limit); + let after = cursor.unwrap_or_default(); + self.read_index(|conn| { + let mut stmt = conn + .prepare( + "SELECT database_id, buyer_principal, listing_id, order_id, + purchased_at_ms, status + FROM market_entitlements + WHERE buyer_principal = ?1 + AND database_id > ?2 + AND status = ?3 + ORDER BY database_id ASC + LIMIT ?4", + ) + .map_err(|error| error.to_string())?; + let mut entitlements = crate::sqlite::query_map( + &mut stmt, + params![ + caller, + after, + MARKET_ENTITLEMENT_STATUS_ACTIVE, + i64::from(limit) + 1 + ], + map_market_entitlement, ) .map_err(|error| error.to_string())?; - Ok(()) + let next_cursor = if entitlements.len() > limit as usize { + entitlements.pop(); + entitlements + .last() + .map(|entitlement| entitlement.database_id.clone()) + } else { + None + }; + Ok(MarketEntitlementPage { + entitlements, + next_cursor, + }) }) } - pub fn revoke_database_access( + pub fn market_list_orders( &self, - database_id: &str, caller: &str, - principal: &str, - ) -> Result<(), String> { - self.require_role(database_id, caller, RequiredRole::Owner)?; - self.database_meta(database_id)?; - if caller == principal { - return Err("owner cannot revoke own access".to_string()); - } - self.write_index(|conn| { - conn.execute( - "DELETE FROM database_members WHERE database_id = ?1 AND principal = ?2", - params![database_id, principal], + cursor: Option, + limit: u32, + ) -> Result { + require_authenticated_principal(caller)?; + let limit = page_limit(limit); + let after = cursor.unwrap_or_default(); + self.read_index(|conn| { + let mut stmt = conn + .prepare( + "SELECT order_id, listing_id, database_id, buyer_principal, seller_principal, + price_e8s, listing_revision, created_at_ms + FROM market_orders + WHERE buyer_principal = ?1 AND order_id > ?2 + ORDER BY order_id ASC + LIMIT ?3", + ) + .map_err(|error| error.to_string())?; + let mut orders = crate::sqlite::query_map( + &mut stmt, + params![caller, after, i64::from(limit) + 1], + map_market_order, ) .map_err(|error| error.to_string())?; - Ok(()) + let next_cursor = if orders.len() > limit as usize { + orders.pop(); + orders.last().map(|order| order.order_id.clone()) + } else { + None + }; + Ok(MarketOrderPage { + orders, + next_cursor, + }) }) } - pub fn list_database_members( + pub fn market_count_active_entitlements( &self, - database_id: &str, caller: &str, - ) -> Result, String> { - self.database_meta(database_id)?; + database_id: &str, + ) -> Result { + require_authenticated_principal(caller)?; self.read_index(|conn| { - let caller_role = load_member_role(conn, database_id, caller)? - .ok_or_else(|| format!("principal has no access to database: {database_id}"))?; - if caller_role != DatabaseRole::Owner - && !(caller == ANONYMOUS_PRINCIPAL - && role_allows(caller_role, RequiredRole::Reader)) - { - return Err(format!( - "principal lacks required database role: {database_id}" - )); + load_database_status(conn, database_id)?; + let config = load_cycles_billing_config(conn)?; + if caller != config.billing_authority_id { + let role = load_member_role(conn, database_id, caller)? + .ok_or_else(|| format!("principal has no access to database: {database_id}"))?; + if role != DatabaseRole::Owner { + return Err(format!( + "principal lacks required database role: {database_id}" + )); + } } - let mut stmt = conn - .prepare( - "SELECT database_id, principal, role, created_at_ms - FROM database_members - WHERE database_id = ?1 - ORDER BY principal ASC", + let count: i64 = conn + .query_row( + "SELECT COUNT(*) + FROM market_entitlements + WHERE database_id = ?1 + AND status = ?2", + params![database_id, MARKET_ENTITLEMENT_STATUS_ACTIVE], + |row| crate::sqlite::row_get(row, 0), ) .map_err(|error| error.to_string())?; - crate::sqlite::query_map(&mut stmt, params![database_id], |row| { - Ok(DatabaseMember { - database_id: crate::sqlite::row_get(row, 0)?, - principal: crate::sqlite::row_get(row, 1)?, - role: role_from_db(&crate::sqlite::row_get::(row, 2)?)?, - created_at_ms: crate::sqlite::row_get(row, 3)?, - }) - }) - .map_err(|error| error.to_string()) + u64::try_from(count).map_err(|error| error.to_string()) }) } - pub fn status(&self, database_id: &str, caller: &str) -> Result { - self.with_database_store(database_id, caller, RequiredRole::Reader, |store| { - store.status() + pub fn require_database_write_cycles_available(&self, database_id: &str) -> Result<(), String> { + self.read_index(|conn| { + let config = load_cycles_billing_config(conn)?; + require_database_write_cycles_available_for_conn(conn, database_id, &config) }) } - pub fn read_node( + pub fn prepare_metered_update( &self, database_id: &str, caller: &str, - path: &str, - ) -> Result, String> { - self.with_database_store(database_id, caller, RequiredRole::Reader, |store| { - store.read_node(path) + required_role: RequiredRole, + ) -> Result { + self.read_index(|conn| { + let role = load_database_status(conn, database_id).and_then(|_| { + load_member_role(conn, database_id, caller)? + .ok_or_else(|| format!("principal has no access to database: {database_id}")) + })?; + if !role_allows(role, required_role) { + return Err(format!( + "principal lacks required database role: {database_id}" + )); + } + let config = load_cycles_billing_config(conn)?; + require_database_write_cycles_available_for_conn(conn, database_id, &config)?; + Ok(config) }) } - pub fn authorize_url_ingest_trigger_session( + pub fn check_database_write_cycles( &self, + database_id: &str, caller: &str, - request: UrlIngestTriggerSessionRequest, - now: i64, ) -> Result<(), String> { - validate_url_ingest_trigger_session_request(&request)?; - if caller == "2vxsx-fae" { + if caller == ANONYMOUS_PRINCIPAL { return Err("anonymous caller not allowed".to_string()); } - self.require_role(&request.database_id, caller, RequiredRole::Writer)?; - self.require_role( - &request.database_id, - DEFAULT_LLM_WRITER_PRINCIPAL, - RequiredRole::Writer, - ) - .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; - self.write_index(|conn| { - purge_expired_url_ingest_trigger_sessions(conn, now)?; - conn.execute( - "INSERT INTO url_ingest_trigger_sessions - (database_id, session_nonce, principal, expires_at_ms, created_at_ms, - refreshed_at_ms) - VALUES (?1, ?2, ?3, ?4, ?5, ?5) - ON CONFLICT(database_id, session_nonce) DO UPDATE SET - principal = excluded.principal, - expires_at_ms = excluded.expires_at_ms, - refreshed_at_ms = excluded.refreshed_at_ms", - params![ - request.database_id, - request.session_nonce, - caller, - now + URL_INGEST_TRIGGER_SESSION_TTL_MS, - now - ], - ) - .map_err(|error| error.to_string())?; - Ok(()) - }) - } - - pub fn check_url_ingest_trigger_session( - &self, - request: UrlIngestTriggerSessionCheckRequest, - now: i64, - ) -> Result<(), String> { - validate_url_ingest_trigger_session_check_request(&request)?; - self.require_role( - &request.database_id, - DEFAULT_LLM_WRITER_PRINCIPAL, - RequiredRole::Writer, - ) - .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; - let principal: String = self.read_index(|conn| { - conn.query_row( - "SELECT principal FROM url_ingest_trigger_sessions - WHERE database_id = ?1 - AND session_nonce = ?2 - AND expires_at_ms >= ?3", - params![request.database_id, request.session_nonce, now], - |row| crate::sqlite::row_get(row, 0), - ) - .optional() - .map_err(|error| error.to_string())? - .ok_or_else(|| "url ingest trigger session is missing or expired".to_string()) - })?; - let node = self - .read_node(&request.database_id, &principal, &request.request_path)? - .ok_or_else(|| format!("url ingest request not found: {}", request.request_path))?; - validate_url_ingest_request_node(&node, &principal)?; - self.require_database_write_cycles_available(&request.database_id) + self.require_role(database_id, caller, RequiredRole::Writer)?; + self.require_database_write_cycles_available(database_id) } - pub fn authorize_ops_answer_session( + pub fn charge_database_update( &self, + config: &CyclesBillingConfig, + database_id: &str, caller: &str, - request: OpsAnswerSessionRequest, + method: &str, + cycles_delta: u128, now: i64, ) -> Result<(), String> { - validate_ops_answer_session_request(&request)?; - if caller == "2vxsx-fae" { - return Err("anonymous caller not allowed".to_string()); + let computed_charge = compute_update_charge(cycles_delta)?; + if computed_charge == 0 { + return Ok(()); } - self.require_role(&request.database_id, caller, RequiredRole::Reader)?; - self.write_index(|conn| { - purge_expired_ops_answer_sessions(conn, now)?; - conn.execute( - "INSERT INTO ops_answer_sessions - (database_id, session_nonce, principal, expires_at_ms, created_at_ms, - refreshed_at_ms) - VALUES (?1, ?2, ?3, ?4, ?5, ?5) - ON CONFLICT(database_id, session_nonce) DO UPDATE SET - principal = excluded.principal, - expires_at_ms = excluded.expires_at_ms, - refreshed_at_ms = excluded.refreshed_at_ms", - params![ - request.database_id, - request.session_nonce, + self.write_index(|tx| { + charge_database_update_in_tx( + tx, + DatabaseCharge { + database_id, caller, - now + OPS_ANSWER_SESSION_TTL_MS, - now - ], + method, + cycles_delta, + now, + config, + computed_charge, + }, ) - .map_err(|error| error.to_string())?; - Ok(()) }) } - pub fn check_ops_answer_session( - &self, - request: OpsAnswerSessionCheckRequest, - now: i64, - ) -> Result { - validate_ops_answer_session_check_request(&request)?; - let principal: String = self.read_index(|conn| { - conn.query_row( - "SELECT principal FROM ops_answer_sessions - WHERE database_id = ?1 - AND session_nonce = ?2 - AND expires_at_ms >= ?3", - params![request.database_id, request.session_nonce, now], - |row| crate::sqlite::row_get(row, 0), - ) - .optional() - .map_err(|error| error.to_string())? - .ok_or_else(|| "ops answer session is missing or expired".to_string()) - })?; - self.require_role(&request.database_id, &principal, RequiredRole::Reader)?; - self.require_database_write_cycles_available(&request.database_id)?; - Ok(OpsAnswerSessionCheckResult { principal }) + pub fn run_database_migrations(&self, database_id: &str) -> Result<(), String> { + let meta = self.database_meta(database_id)?; + self.run_database_migrations_for_meta(database_id, &meta) } - pub fn check_source_run_session( + pub fn run_pending_database_migrations(&self, database_id: &str) -> Result<(), String> { + let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Pending])?; + self.run_database_migrations_for_meta(database_id, &meta) + } + + fn run_database_migrations_for_meta( &self, - request: SourceRunSessionCheckRequest, - now: i64, + database_id: &str, + meta: &DatabaseMeta, ) -> Result<(), String> { - validate_source_run_session_check_request(&request)?; - self.require_role( - &request.database_id, - DEFAULT_LLM_WRITER_PRINCIPAL, - RequiredRole::Writer, - ) - .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; - let principal: String = self.read_index(|conn| { - conn.query_row( - "SELECT principal FROM source_run_sessions - WHERE database_id = ?1 - AND source_path = ?2 - AND source_etag = ?3 - AND session_nonce = ?4 - AND expires_at_ms >= ?5", - params![ - request.database_id, - request.source_path, - request.source_etag, - request.session_nonce, - now - ], - |row| crate::sqlite::row_get(row, 0), - ) - .optional() - .map_err(|error| error.to_string())? - .ok_or_else(|| "source run session is missing or expired".to_string()) - })?; - self.require_role(&request.database_id, &principal, RequiredRole::Writer)?; - let source = self - .read_node(&request.database_id, &principal, &request.source_path)? - .ok_or_else(|| format!("source node not found: {}", request.source_path))?; - if source.kind != NodeKind::Source { - return Err("source run session target is not a source node".to_string()); + #[cfg(not(target_arch = "wasm32"))] + if let Some(parent) = Path::new(&meta.db_file_name).parent() { + create_dir_all(parent).map_err(|error| error.to_string())?; } - if source.etag != request.source_etag { - return Err("source run session source etag is stale".to_string()); + let result = self.database_store(meta)?.run_fs_migrations(); + if result.is_ok() { + let _ = self.refresh_logical_size(database_id); } - self.require_database_write_cycles_available(&request.database_id)?; - Ok(()) + result } - pub fn list_nodes( + pub fn delete_database( &self, + request: DeleteDatabaseRequest, caller: &str, - request: ListNodesRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.list_nodes(request) + _now: i64, + ) -> Result<(), String> { + let database_id = request.database_id.as_str(); + self.require_role(database_id, caller, RequiredRole::Owner)?; + self.require_no_pending_cycles_operations(database_id)?; + let status = self.read_index(|conn| load_database_status(conn, database_id))?; + if !matches!(status, DatabaseStatus::Pending | DatabaseStatus::Active) { + return Err(format!( + "database is {}: {database_id}", + status_to_db(status) + )); + } + let meta = self.database_meta(database_id).ok(); + #[cfg(target_arch = "wasm32")] + let _ = &meta; + #[cfg(not(target_arch = "wasm32"))] + if let Some(meta) = &meta + && let Err(error) = remove_file(&meta.db_file_name) + && error.kind() != std::io::ErrorKind::NotFound + { + return Err(error.to_string()); + } + self.write_index(|conn| { + delete_database_index_rows(conn, database_id)?; + Ok(()) }) } - pub fn list_children( - &self, - caller: &str, - request: ListChildrenRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.list_children(request) + fn require_no_pending_cycles_operations(&self, database_id: &str) -> Result<(), String> { + self.read_index(|conn| { + let pending = first_database_cycles_pending_purchase_status(conn, database_id)?; + if let Some(pending) = pending { + return Err(format!( + "database has pending cycle operation: {database_id}; operation_id={}; status={}; required_action={}", + pending.operation_id, + pending.status, + pending.required_action + )); + } + Ok(()) }) } - pub fn write_node( + pub fn begin_database_archive( &self, + database_id: &str, caller: &str, - request: WriteNodeRequest, now: i64, - ) -> Result { - validate_source_path_for_kind(&request.path, &request.kind)?; - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - store.write_node(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); - } - result + ) -> Result { + self.require_role(database_id, caller, RequiredRole::Owner)?; + self.require_no_pending_cycles_operations(database_id)?; + let meta = self.database_meta(database_id)?; + let size_bytes = self.database_size(&meta)?; + self.write_index(|conn| { + conn.execute( + "UPDATE databases + SET status = 'archiving', + updated_at_ms = ?2, + logical_size_bytes = ?3 + WHERE database_id = ?1", + params![ + database_id, + now, + i64::try_from(size_bytes).map_err(|error| error.to_string())? + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) + })?; + Ok(DatabaseArchiveInfo { + database_id: database_id.to_string(), + size_bytes, + }) } - pub fn write_source_for_generation( + pub fn read_database_archive_chunk( &self, + database_id: &str, caller: &str, - request: WriteSourceForGenerationRequest, - now: i64, - ) -> Result { - if caller == ANONYMOUS_PRINCIPAL { - return Err("anonymous caller not allowed".to_string()); + offset: u64, + max_bytes: u32, + ) -> Result, String> { + self.require_role(database_id, caller, RequiredRole::Owner)?; + let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Archiving])?; + if max_bytes == 0 { + return Ok(Vec::new()); } - validate_source_for_generation_request(&request)?; - self.require_role(&request.database_id, caller, RequiredRole::Writer)?; - self.require_role( - &request.database_id, - DEFAULT_LLM_WRITER_PRINCIPAL, - RequiredRole::Writer, - ) - .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; - - let database_id = request.database_id.clone(); - let session_nonce = request.session_nonce.clone(); - let path = request.path.clone(); - let write_request = WriteNodeRequest { - database_id: request.database_id, - path: request.path, - kind: NodeKind::Source, - content: request.content, - metadata_json: request.metadata_json, - expected_etag: request.expected_etag, - }; - let write = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - store.write_node(write_request, now) - })?; - let _ = self.write_source_run_session( - &database_id, - &path, - &write.node.etag, - &session_nonce, - caller, - now, - ); - let _ = self.refresh_logical_size(&database_id); - Ok(WriteSourceForGenerationResult { - write, - session_nonce, - }) + if max_bytes > MAX_ARCHIVE_CHUNK_BYTES { + return Err(format!( + "archive chunk size exceeds limit: {max_bytes} > {MAX_ARCHIVE_CHUNK_BYTES}" + )); + } + let size = meta.logical_size_bytes; + if offset >= size { + return Ok(Vec::new()); + } + let remaining = size.saturating_sub(offset); + let chunk_len = remaining.min(u64::from(max_bytes)); + self.database_export_chunk(&meta, offset, chunk_len) } - pub fn write_nodes( + pub fn finalize_database_archive( &self, + database_id: &str, caller: &str, - request: WriteNodesRequest, + snapshot_hash: Vec, now: i64, - ) -> Result, String> { - for node in &request.nodes { - validate_source_path_for_kind(&node.path, &node.kind)?; - } - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - store.write_nodes(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); + ) -> Result { + self.require_role(database_id, caller, RequiredRole::Owner)?; + let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Archiving])?; + validate_snapshot_hash(&snapshot_hash)?; + let actual_hash = self.database_sha256(&meta, meta.logical_size_bytes)?; + if actual_hash != snapshot_hash { + return Err("snapshot_hash does not match archived database bytes".to_string()); } - result + self.write_index(|conn| { + conn.execute( + "UPDATE databases + SET status = 'archived', + snapshot_hash = ?2, + restore_size_bytes = NULL, + archived_at_ms = ?3, + updated_at_ms = ?3 + WHERE database_id = ?1", + params![database_id, snapshot_hash, now], + ) + .map_err(|error| error.to_string())?; + Ok(()) + })?; + Ok(meta) } - pub fn delete_node( + pub fn cancel_database_archive( &self, + database_id: &str, caller: &str, - request: DeleteNodeRequest, now: i64, - ) -> Result { - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - store.delete_node(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); - } - result + ) -> Result { + self.require_role(database_id, caller, RequiredRole::Owner)?; + let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Archiving])?; + self.write_index(|conn| { + conn.execute( + "UPDATE databases + SET status = 'active', + updated_at_ms = ?2 + WHERE database_id = ?1", + params![database_id, now], + ) + .map_err(|error| error.to_string())?; + Ok(()) + })?; + Ok(meta) } - pub fn append_node( + pub fn begin_database_restore( &self, + database_id: &str, caller: &str, - request: AppendNodeRequest, + snapshot_hash: Vec, + size_bytes: u64, now: i64, - ) -> Result { - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - let kind = store - .read_node(&request.path)? - .map(|node| node.kind) - .or_else(|| request.kind.clone()) - .unwrap_or(NodeKind::File); - validate_source_path_for_kind(&request.path, &kind)?; - store.append_node(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); - } - result + ) -> Result { + self.begin_database_restore_session(database_id, caller, snapshot_hash, size_bytes, now) + .map(|restore| restore.meta) } - pub fn edit_node( + pub fn begin_database_restore_session( &self, + database_id: &str, caller: &str, - request: EditNodeRequest, + snapshot_hash: Vec, + size_bytes: u64, now: i64, - ) -> Result { - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - store.edit_node(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); + ) -> Result { + self.require_role(database_id, caller, RequiredRole::Owner)?; + validate_snapshot_hash(&snapshot_hash)?; + if size_bytes > MAX_DATABASE_SIZE_BYTES { + return Err(format!( + "database size exceeds limit: {size_bytes} > {MAX_DATABASE_SIZE_BYTES}" + )); } - result + self.require_no_pending_cycles_operations(database_id)?; + let rollback = self.database_restore_rollback(database_id)?; + if rollback.status != DatabaseStatus::Archived { + return Err("database restore can only begin from archived status".to_string()); + } + let mount_id = rollback + .active_mount_id + .ok_or_else(|| format!("archived database has no mount: {database_id}"))?; + self.write_index(|tx| { + record_database_restore_session(tx, &rollback, now)?; + tx.execute( + "DELETE FROM database_restore_chunks WHERE database_id = ?1", + params![database_id], + ) + .map_err(|error| error.to_string())?; + tx.execute( + "UPDATE databases + SET status = 'restoring', + active_mount_id = ?2, + snapshot_hash = ?3, + archived_at_ms = NULL, + restore_size_bytes = ?4, + updated_at_ms = ?5 + WHERE database_id = ?1", + params![ + database_id, + i64::from(mount_id), + snapshot_hash, + i64::try_from(size_bytes).map_err(|error| error.to_string())?, + now + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) + })?; + let meta = self.database_meta_allowing_restoring(database_id)?; + #[cfg(not(target_arch = "wasm32"))] + let _ = remove_file(&meta.db_file_name); + Ok(DatabaseRestoreBegin { meta, rollback }) } - pub fn mkdir_node( + pub fn rollback_database_restore_begin( &self, - caller: &str, - request: MkdirNodeRequest, + rollback: DatabaseRestoreRollback, now: i64, - ) -> Result { - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - store.mkdir_node(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); - } - result + ) -> Result<(), String> { + self.write_index(|tx| { + let current_status = load_database_status(tx, &rollback.database_id)?; + if current_status != DatabaseStatus::Restoring { + return Err(format!( + "database restore rollback requires restoring status: {}", + rollback.database_id + )); + } + tx.execute( + "DELETE FROM database_restore_chunks WHERE database_id = ?1", + params![rollback.database_id], + ) + .map_err(|error| error.to_string())?; + restore_database_state(tx, &rollback, now)?; + Ok(()) + }) } - pub fn move_node( + pub fn cancel_database_restore( &self, + database_id: &str, caller: &str, - request: MoveNodeRequest, now: i64, - ) -> Result { - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - if let Some(node) = store.read_node(&request.from_path)? { - validate_source_path_for_kind(&request.to_path, &node.kind)?; - } - store.move_node(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); + ) -> Result { + self.require_role(database_id, caller, RequiredRole::Owner)?; + let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Restoring])?; + let rollback = self.database_restore_session(database_id)?; + #[cfg(not(target_arch = "wasm32"))] + if let Err(error) = remove_file(&meta.db_file_name) + && error.kind() != std::io::ErrorKind::NotFound + { + return Err(error.to_string()); } - result - } - - pub fn glob_nodes( - &self, - caller: &str, - request: GlobNodesRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.glob_nodes(request) - }) + self.write_index(|tx| { + tx.execute( + "DELETE FROM database_restore_chunks WHERE database_id = ?1", + params![database_id], + ) + .map_err(|error| error.to_string())?; + restore_database_state(tx, &rollback, now)?; + Ok(()) + })?; + Ok(meta) } - pub fn incoming_links( + pub fn write_database_restore_chunk( &self, + database_id: &str, caller: &str, - request: IncomingLinksRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.incoming_links(request) + offset: u64, + bytes: &[u8], + ) -> Result<(), String> { + self.require_role(database_id, caller, RequiredRole::Owner)?; + if bytes.len() > MAX_RESTORE_CHUNK_BYTES { + return Err(format!( + "restore chunk size exceeds limit: {} > {MAX_RESTORE_CHUNK_BYTES}", + bytes.len() + )); + } + let _meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Restoring])?; + let expected_size = self.restore_size_bytes(database_id)?; + let end = offset + .checked_add(bytes.len() as u64) + .ok_or_else(|| "restore chunk range overflows u64".to_string())?; + if end > expected_size { + return Err(format!( + "restore chunk exceeds expected size: end {end} > {expected_size}" + )); + } + self.write_index(|conn| { + conn.execute( + "INSERT OR REPLACE INTO database_restore_chunks + (database_id, offset_bytes, end_bytes, bytes) + VALUES (?1, ?2, ?3, ?4)", + params![ + database_id, + i64::try_from(offset).map_err(|error| error.to_string())?, + i64::try_from(end).map_err(|error| error.to_string())?, + bytes.to_vec() + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) }) } - pub fn outgoing_links( + pub fn finalize_database_restore( &self, + database_id: &str, caller: &str, - request: OutgoingLinksRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.outgoing_links(request) - }) + now: i64, + ) -> Result { + self.require_role(database_id, caller, RequiredRole::Owner)?; + let meta = self.database_meta_with_statuses(database_id, &[DatabaseStatus::Restoring])?; + let expected_size = self.restore_size_bytes(database_id)?; + let chunks = self.read_index(|conn| load_restore_chunks(conn, database_id))?; + if !restore_chunks_cover_expected_size(&chunks, expected_size)? { + return Err(format!( + "restore chunks are incomplete for expected size {expected_size} bytes" + )); + } + let expected_hash = self.restore_snapshot_hash(database_id)?; + let mut hasher = Sha256::new(); + let mut checksum = FNV1A64_OFFSET; + for chunk in &chunks { + hasher.update(&chunk.bytes); + checksum = fnv1a64_update(checksum, &chunk.bytes); + } + let actual_hash = hasher.finalize().to_vec(); + if actual_hash != expected_hash { + return Err("snapshot_hash does not match restored database bytes".to_string()); + } + self.import_database_bytes(&meta, expected_size, checksum, &chunks)?; + self.database_store(&meta)?.run_fs_migrations()?; + self.write_index(|tx| { + tx.execute( + "DELETE FROM database_restore_chunks WHERE database_id = ?1", + params![database_id], + ) + .map_err(|error| error.to_string())?; + tx.execute( + "DELETE FROM database_restore_sessions WHERE database_id = ?1", + params![database_id], + ) + .map_err(|error| error.to_string())?; + tx.execute( + "UPDATE databases + SET status = 'active', + logical_size_bytes = ?2, + restore_size_bytes = NULL, + updated_at_ms = ?3 + WHERE database_id = ?1", + params![ + database_id, + i64::try_from(expected_size).map_err(|error| error.to_string())?, + now + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) + })?; + self.database_meta(database_id) } - pub fn graph_links( + pub fn grant_database_access( &self, + database_id: &str, caller: &str, - request: GraphLinksRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.graph_links(request) + principal: &str, + role: DatabaseRole, + now: i64, + ) -> Result<(), String> { + self.require_role(database_id, caller, RequiredRole::Owner)?; + if caller == principal && role != DatabaseRole::Owner { + return Err("owner cannot downgrade own access".to_string()); + } + self.write_index(|conn| { + if !database_member_exists(conn, database_id, principal)? { + let member_count = database_member_count_for_conn(conn, database_id)?; + if member_count >= MAX_DATABASE_MEMBERS_PER_DATABASE { + return Err("too many database members".to_string()); + } + } + conn.execute( + "INSERT INTO database_members (database_id, principal, role, created_at_ms) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(database_id, principal) + DO UPDATE SET role = excluded.role", + params![database_id, principal, role_to_db(role), now], + ) + .map_err(|error| error.to_string())?; + Ok(()) }) } - pub fn graph_neighborhood( + pub fn rename_database( &self, + database_id: &str, caller: &str, - request: GraphNeighborhoodRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.graph_neighborhood(request) + name: &str, + now: i64, + ) -> Result<(), String> { + self.require_role(database_id, caller, RequiredRole::Owner)?; + self.database_meta(database_id)?; + let name = normalize_database_name(name)?; + self.write_index(|conn| { + conn.execute( + "UPDATE databases + SET name = ?2, + updated_at_ms = ?3 + WHERE database_id = ?1", + params![database_id, name, now], + ) + .map_err(|error| error.to_string())?; + Ok(()) }) } - pub fn read_node_context( + pub fn revoke_database_access( &self, + database_id: &str, caller: &str, - request: NodeContextRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.read_node_context(request) + principal: &str, + ) -> Result<(), String> { + self.require_role(database_id, caller, RequiredRole::Owner)?; + self.database_meta(database_id)?; + if caller == principal { + return Err("owner cannot revoke own access".to_string()); + } + self.write_index(|conn| { + conn.execute( + "DELETE FROM database_members WHERE database_id = ?1 AND principal = ?2", + params![database_id, principal], + ) + .map_err(|error| error.to_string())?; + Ok(()) }) } - pub fn query_context( + pub fn list_database_members( &self, + database_id: &str, caller: &str, - request: QueryContextRequest, - ) -> Result { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.query_context(request) + ) -> Result, String> { + self.database_meta(database_id)?; + self.read_index(|conn| { + let caller_role = load_member_role(conn, database_id, caller)? + .ok_or_else(|| format!("principal has no access to database: {database_id}"))?; + if caller_role != DatabaseRole::Owner + && !(caller == ANONYMOUS_PRINCIPAL + && role_allows(caller_role, RequiredRole::Reader)) + { + return Err(format!( + "principal lacks required database role: {database_id}" + )); + } + let mut stmt = conn + .prepare( + "SELECT database_id, principal, role, created_at_ms + FROM database_members + WHERE database_id = ?1 + ORDER BY principal ASC", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map(&mut stmt, params![database_id], |row| { + Ok(DatabaseMember { + database_id: crate::sqlite::row_get(row, 0)?, + principal: crate::sqlite::row_get(row, 1)?, + role: role_from_db(&crate::sqlite::row_get::(row, 2)?)?, + created_at_ms: crate::sqlite::row_get(row, 3)?, + }) + }) + .map_err(|error| error.to_string()) }) } - pub fn source_evidence( + pub fn status(&self, database_id: &str, caller: &str) -> Result { + self.with_database_store(database_id, caller, RequiredRole::Reader, |store| { + store.status() + }) + } + + pub fn read_node( &self, + database_id: &str, caller: &str, - request: SourceEvidenceRequest, - ) -> Result { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.source_evidence(request) - }) + path: &str, + ) -> Result, String> { + self.with_market_read_database_store(database_id, caller, |store| store.read_node(path)) } - pub fn multi_edit_node( + pub fn authorize_url_ingest_trigger_session( &self, caller: &str, - request: MultiEditNodeRequest, + request: UrlIngestTriggerSessionRequest, now: i64, - ) -> Result { - let database_id = request.database_id.clone(); - let result = - self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { - store.multi_edit_node(request, now) - }); - if result.is_ok() { - let _ = self.refresh_logical_size(&database_id); + ) -> Result<(), String> { + validate_url_ingest_trigger_session_request(&request)?; + if caller == "2vxsx-fae" { + return Err("anonymous caller not allowed".to_string()); } - result + self.require_role(&request.database_id, caller, RequiredRole::Writer)?; + self.require_role( + &request.database_id, + DEFAULT_LLM_WRITER_PRINCIPAL, + RequiredRole::Writer, + ) + .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; + self.write_index(|conn| { + purge_expired_url_ingest_trigger_sessions(conn, now)?; + conn.execute( + "INSERT INTO url_ingest_trigger_sessions + (database_id, session_nonce, principal, expires_at_ms, created_at_ms, + refreshed_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?5) + ON CONFLICT(database_id, session_nonce) DO UPDATE SET + principal = excluded.principal, + expires_at_ms = excluded.expires_at_ms, + refreshed_at_ms = excluded.refreshed_at_ms", + params![ + request.database_id, + request.session_nonce, + caller, + now + URL_INGEST_TRIGGER_SESSION_TTL_MS, + now + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) + }) } - pub fn search_nodes( + pub fn check_url_ingest_trigger_session( + &self, + request: UrlIngestTriggerSessionCheckRequest, + now: i64, + ) -> Result<(), String> { + validate_url_ingest_trigger_session_check_request(&request)?; + self.require_role( + &request.database_id, + DEFAULT_LLM_WRITER_PRINCIPAL, + RequiredRole::Writer, + ) + .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; + let principal: String = self.read_index(|conn| { + conn.query_row( + "SELECT principal FROM url_ingest_trigger_sessions + WHERE database_id = ?1 + AND session_nonce = ?2 + AND expires_at_ms >= ?3", + params![request.database_id, request.session_nonce, now], + |row| crate::sqlite::row_get(row, 0), + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| "url ingest trigger session is missing or expired".to_string()) + })?; + let node = self + .read_node(&request.database_id, &principal, &request.request_path)? + .ok_or_else(|| format!("url ingest request not found: {}", request.request_path))?; + validate_url_ingest_request_node(&node, &principal)?; + self.require_database_write_cycles_available(&request.database_id) + } + + pub fn authorize_ops_answer_session( &self, caller: &str, - request: SearchNodesRequest, - ) -> Result, String> { - let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.search_nodes(request) + request: OpsAnswerSessionRequest, + now: i64, + ) -> Result<(), String> { + validate_ops_answer_session_request(&request)?; + if caller == "2vxsx-fae" { + return Err("anonymous caller not allowed".to_string()); + } + self.require_role(&request.database_id, caller, RequiredRole::Reader)?; + self.write_index(|conn| { + purge_expired_ops_answer_sessions(conn, now)?; + conn.execute( + "INSERT INTO ops_answer_sessions + (database_id, session_nonce, principal, expires_at_ms, created_at_ms, + refreshed_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?5) + ON CONFLICT(database_id, session_nonce) DO UPDATE SET + principal = excluded.principal, + expires_at_ms = excluded.expires_at_ms, + refreshed_at_ms = excluded.refreshed_at_ms", + params![ + request.database_id, + request.session_nonce, + caller, + now + OPS_ANSWER_SESSION_TTL_MS, + now + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) }) } - pub fn search_node_paths( + pub fn check_ops_answer_session( + &self, + request: OpsAnswerSessionCheckRequest, + now: i64, + ) -> Result { + validate_ops_answer_session_check_request(&request)?; + let principal: String = self.read_index(|conn| { + conn.query_row( + "SELECT principal FROM ops_answer_sessions + WHERE database_id = ?1 + AND session_nonce = ?2 + AND expires_at_ms >= ?3", + params![request.database_id, request.session_nonce, now], + |row| crate::sqlite::row_get(row, 0), + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| "ops answer session is missing or expired".to_string()) + })?; + self.require_role(&request.database_id, &principal, RequiredRole::Reader)?; + self.require_database_write_cycles_available(&request.database_id)?; + Ok(OpsAnswerSessionCheckResult { principal }) + } + + pub fn check_source_run_session( + &self, + request: SourceRunSessionCheckRequest, + now: i64, + ) -> Result<(), String> { + validate_source_run_session_check_request(&request)?; + self.require_role( + &request.database_id, + DEFAULT_LLM_WRITER_PRINCIPAL, + RequiredRole::Writer, + ) + .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; + let principal: String = self.read_index(|conn| { + conn.query_row( + "SELECT principal FROM source_run_sessions + WHERE database_id = ?1 + AND source_path = ?2 + AND source_etag = ?3 + AND session_nonce = ?4 + AND expires_at_ms >= ?5", + params![ + request.database_id, + request.source_path, + request.source_etag, + request.session_nonce, + now + ], + |row| crate::sqlite::row_get(row, 0), + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| "source run session is missing or expired".to_string()) + })?; + self.require_role(&request.database_id, &principal, RequiredRole::Writer)?; + let source = self + .read_node(&request.database_id, &principal, &request.source_path)? + .ok_or_else(|| format!("source node not found: {}", request.source_path))?; + if source.kind != NodeKind::Source { + return Err("source run session target is not a source node".to_string()); + } + if source.etag != request.source_etag { + return Err("source run session source etag is stale".to_string()); + } + self.require_database_write_cycles_available(&request.database_id)?; + Ok(()) + } + + pub fn list_nodes( &self, caller: &str, - request: SearchNodePathsRequest, - ) -> Result, String> { + request: ListNodesRequest, + ) -> Result, String> { let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.search_node_paths(request) + self.with_market_read_database_store(&database_id, caller, |store| { + store.list_nodes(request) }) } - pub fn export_fs_snapshot( + pub fn list_children( &self, caller: &str, - request: ExportSnapshotRequest, - ) -> Result { + request: ListChildrenRequest, + ) -> Result, String> { let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.export_snapshot(request) + self.with_market_read_database_store(&database_id, caller, |store| { + store.list_children(request) }) } - pub fn fetch_fs_updates( + pub fn write_node( &self, caller: &str, - request: FetchUpdatesRequest, - ) -> Result { + request: WriteNodeRequest, + now: i64, + ) -> Result { + validate_source_path_for_kind(&request.path, &request.kind)?; let database_id = request.database_id.clone(); - self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { - store.fetch_updates(request) + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + store.write_node(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); + } + result + } + + pub fn write_source_for_generation( + &self, + caller: &str, + request: WriteSourceForGenerationRequest, + now: i64, + ) -> Result { + if caller == ANONYMOUS_PRINCIPAL { + return Err("anonymous caller not allowed".to_string()); + } + validate_source_for_generation_request(&request)?; + self.require_role(&request.database_id, caller, RequiredRole::Writer)?; + self.require_role( + &request.database_id, + DEFAULT_LLM_WRITER_PRINCIPAL, + RequiredRole::Writer, + ) + .map_err(|error| format!("LLM writer principal lacks writer access: {error}"))?; + + let database_id = request.database_id.clone(); + let session_nonce = request.session_nonce.clone(); + let path = request.path.clone(); + let write_request = WriteNodeRequest { + database_id: request.database_id, + path: request.path, + kind: NodeKind::Source, + content: request.content, + metadata_json: request.metadata_json, + expected_etag: request.expected_etag, + }; + let write = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + store.write_node(write_request, now) + })?; + let _ = self.write_source_run_session( + &database_id, + &path, + &write.node.etag, + &session_nonce, + caller, + now, + ); + let _ = self.refresh_logical_size(&database_id); + Ok(WriteSourceForGenerationResult { + write, + session_nonce, }) } - fn with_database_store( + pub fn write_nodes( &self, - database_id: &str, caller: &str, - required_role: RequiredRole, - f: impl FnOnce(&FsStore) -> Result, - ) -> Result { - self.require_role(database_id, caller, required_role)?; - let meta = self.database_meta(database_id)?; - let store = self.database_store(&meta)?; - f(&store) + request: WriteNodesRequest, + now: i64, + ) -> Result, String> { + for node in &request.nodes { + validate_source_path_for_kind(&node.path, &node.kind)?; + } + let database_id = request.database_id.clone(); + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + store.write_nodes(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); + } + result } - pub fn require_database_role( + pub fn delete_node( &self, - database_id: &str, caller: &str, - required_role: RequiredRole, - ) -> Result<(), String> { - self.require_role(database_id, caller, required_role) + request: DeleteNodeRequest, + now: i64, + ) -> Result { + let database_id = request.database_id.clone(); + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + store.delete_node(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); + } + result } - fn require_role( + pub fn append_node( &self, - database_id: &str, caller: &str, - required_role: RequiredRole, - ) -> Result<(), String> { - let role = self.read_index(|conn| { - load_database_status(conn, database_id)?; - load_member_role(conn, database_id, caller)? - .ok_or_else(|| format!("principal has no access to database: {database_id}")) - })?; - if role_allows(role, required_role) { - Ok(()) - } else { - Err(format!( - "principal lacks required database role: {database_id}" - )) + request: AppendNodeRequest, + now: i64, + ) -> Result { + let database_id = request.database_id.clone(); + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + let kind = store + .read_node(&request.path)? + .map(|node| node.kind) + .or_else(|| request.kind.clone()) + .unwrap_or(NodeKind::File); + validate_source_path_for_kind(&request.path, &kind)?; + store.append_node(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); } + result } - fn database_meta(&self, database_id: &str) -> Result { - self.read_index(|conn| { - load_database(conn, database_id)?.ok_or_else(|| database_meta_error(conn, database_id)) - }) + pub fn edit_node( + &self, + caller: &str, + request: EditNodeRequest, + now: i64, + ) -> Result { + let database_id = request.database_id.clone(); + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + store.edit_node(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); + } + result } - fn database_meta_allowing_restoring(&self, database_id: &str) -> Result { - self.database_meta_with_statuses( - database_id, - &[ - DatabaseStatus::Pending, - DatabaseStatus::Active, - DatabaseStatus::Restoring, - ], - ) + pub fn mkdir_node( + &self, + caller: &str, + request: MkdirNodeRequest, + now: i64, + ) -> Result { + let database_id = request.database_id.clone(); + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + store.mkdir_node(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); + } + result } - fn database_meta_with_statuses( + pub fn move_node( &self, - database_id: &str, - statuses: &[DatabaseStatus], - ) -> Result { - self.read_index(|conn| { - load_database_with_statuses(conn, database_id, statuses)? - .ok_or_else(|| database_meta_error(conn, database_id)) + caller: &str, + request: MoveNodeRequest, + now: i64, + ) -> Result { + let database_id = request.database_id.clone(); + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + if let Some(node) = store.read_node(&request.from_path)? { + validate_source_path_for_kind(&request.to_path, &node.kind)?; + } + store.move_node(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); + } + result + } + + pub fn glob_nodes( + &self, + caller: &str, + request: GlobNodesRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { + store.glob_nodes(request) }) } - fn database_restore_rollback( + pub fn incoming_links( &self, - database_id: &str, - ) -> Result { - self.read_index(|conn| { - conn.query_row( - "SELECT database_id, status, active_mount_id, snapshot_hash, archived_at_ms, - restore_size_bytes - FROM databases - WHERE database_id = ?1", - params![database_id], - |row| { - let active_mount_id: Option = crate::sqlite::row_get(row, 2)?; - let restore_size_bytes: Option = crate::sqlite::row_get(row, 5)?; - Ok(DatabaseRestoreRollback { - database_id: crate::sqlite::row_get(row, 0)?, - status: status_from_db(&crate::sqlite::row_get::(row, 1)?)?, - active_mount_id: active_mount_id.map(mount_id_from_db).transpose()?, - snapshot_hash: crate::sqlite::row_get(row, 3)?, - archived_at_ms: crate::sqlite::row_get(row, 4)?, - restore_size_bytes: restore_size_bytes.map(|size| size.max(0) as u64), - }) - }, - ) - .optional() - .map_err(|error| error.to_string())? - .ok_or_else(|| format!("database not found: {database_id}")) + caller: &str, + request: IncomingLinksRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_market_read_database_store(&database_id, caller, |store| { + store.incoming_links(request) }) } - fn database_restore_session( + pub fn outgoing_links( &self, - database_id: &str, - ) -> Result { - self.read_index(|conn| { - conn.query_row( - "SELECT database_id, status, active_mount_id, snapshot_hash, archived_at_ms, - restore_size_bytes + caller: &str, + request: OutgoingLinksRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_market_read_database_store(&database_id, caller, |store| { + store.outgoing_links(request) + }) + } + + pub fn graph_links( + &self, + caller: &str, + request: GraphLinksRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_market_read_database_store(&database_id, caller, |store| { + store.graph_links(request) + }) + } + + pub fn graph_neighborhood( + &self, + caller: &str, + request: GraphNeighborhoodRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_market_read_database_store(&database_id, caller, |store| { + store.graph_neighborhood(request) + }) + } + + pub fn read_node_context( + &self, + caller: &str, + request: NodeContextRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_market_read_database_store(&database_id, caller, |store| { + store.read_node_context(request) + }) + } + + pub fn query_context( + &self, + caller: &str, + request: QueryContextRequest, + ) -> Result { + let database_id = request.database_id.clone(); + self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { + store.query_context(request) + }) + } + + pub fn source_evidence( + &self, + caller: &str, + request: SourceEvidenceRequest, + ) -> Result { + let database_id = request.database_id.clone(); + self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { + store.source_evidence(request) + }) + } + + pub fn multi_edit_node( + &self, + caller: &str, + request: MultiEditNodeRequest, + now: i64, + ) -> Result { + let database_id = request.database_id.clone(); + let result = + self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { + store.multi_edit_node(request, now) + }); + if result.is_ok() { + let _ = self.refresh_logical_size(&database_id); + } + result + } + + pub fn search_nodes( + &self, + caller: &str, + request: SearchNodesRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_market_read_database_store(&database_id, caller, |store| { + store.search_nodes(request) + }) + } + + pub fn search_node_paths( + &self, + caller: &str, + request: SearchNodePathsRequest, + ) -> Result, String> { + let database_id = request.database_id.clone(); + self.with_market_read_database_store(&database_id, caller, |store| { + store.search_node_paths(request) + }) + } + + pub fn export_fs_snapshot( + &self, + caller: &str, + request: ExportSnapshotRequest, + ) -> Result { + let database_id = request.database_id.clone(); + self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { + store.export_snapshot(request) + }) + } + + pub fn fetch_fs_updates( + &self, + caller: &str, + request: FetchUpdatesRequest, + ) -> Result { + let database_id = request.database_id.clone(); + self.with_database_store(&database_id, caller, RequiredRole::Reader, |store| { + store.fetch_updates(request) + }) + } + + fn with_database_store( + &self, + database_id: &str, + caller: &str, + required_role: RequiredRole, + f: impl FnOnce(&FsStore) -> Result, + ) -> Result { + self.require_role(database_id, caller, required_role)?; + let meta = self.database_meta(database_id)?; + let store = self.database_store(&meta)?; + f(&store) + } + + fn with_market_read_database_store( + &self, + database_id: &str, + caller: &str, + f: impl FnOnce(&FsStore) -> Result, + ) -> Result { + self.require_market_read_access(database_id, caller)?; + let meta = self.database_meta(database_id)?; + let store = self.database_store(&meta)?; + f(&store) + } + + pub fn require_database_role( + &self, + database_id: &str, + caller: &str, + required_role: RequiredRole, + ) -> Result<(), String> { + self.require_role(database_id, caller, required_role) + } + + fn require_role( + &self, + database_id: &str, + caller: &str, + required_role: RequiredRole, + ) -> Result<(), String> { + let role = self.read_index(|conn| { + load_database_status(conn, database_id)?; + load_member_role(conn, database_id, caller)? + .ok_or_else(|| format!("principal has no access to database: {database_id}")) + })?; + if role_allows(role, required_role) { + Ok(()) + } else { + Err(format!( + "principal lacks required database role: {database_id}" + )) + } + } + + fn require_market_read_access(&self, database_id: &str, caller: &str) -> Result<(), String> { + self.read_index(|conn| { + load_database_status(conn, database_id)?; + if let Some(role) = load_member_role(conn, database_id, caller)? + && role_allows(role, RequiredRole::Reader) + { + return Ok(()); + } + if has_active_market_entitlement(conn, database_id, caller)? { + return Ok(()); + } + Err(format!( + "principal has no access to database: {database_id}" + )) + }) + } + + fn database_meta(&self, database_id: &str) -> Result { + self.read_index(|conn| { + load_database(conn, database_id)?.ok_or_else(|| database_meta_error(conn, database_id)) + }) + } + + fn database_meta_allowing_restoring(&self, database_id: &str) -> Result { + self.database_meta_with_statuses( + database_id, + &[ + DatabaseStatus::Pending, + DatabaseStatus::Active, + DatabaseStatus::Restoring, + ], + ) + } + + fn database_meta_with_statuses( + &self, + database_id: &str, + statuses: &[DatabaseStatus], + ) -> Result { + self.read_index(|conn| { + load_database_with_statuses(conn, database_id, statuses)? + .ok_or_else(|| database_meta_error(conn, database_id)) + }) + } + + fn database_restore_rollback( + &self, + database_id: &str, + ) -> Result { + self.read_index(|conn| { + conn.query_row( + "SELECT database_id, status, active_mount_id, snapshot_hash, archived_at_ms, + restore_size_bytes + FROM databases + WHERE database_id = ?1", + params![database_id], + |row| { + let active_mount_id: Option = crate::sqlite::row_get(row, 2)?; + let restore_size_bytes: Option = crate::sqlite::row_get(row, 5)?; + Ok(DatabaseRestoreRollback { + database_id: crate::sqlite::row_get(row, 0)?, + status: status_from_db(&crate::sqlite::row_get::(row, 1)?)?, + active_mount_id: active_mount_id.map(mount_id_from_db).transpose()?, + snapshot_hash: crate::sqlite::row_get(row, 3)?, + archived_at_ms: crate::sqlite::row_get(row, 4)?, + restore_size_bytes: restore_size_bytes.map(|size| size.max(0) as u64), + }) + }, + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("database not found: {database_id}")) + }) + } + + fn database_restore_session( + &self, + database_id: &str, + ) -> Result { + self.read_index(|conn| { + conn.query_row( + "SELECT database_id, status, active_mount_id, snapshot_hash, archived_at_ms, + restore_size_bytes FROM database_restore_sessions WHERE database_id = ?1", params![database_id], @@ -2634,6 +3638,9 @@ fn run_index_migrations_in_tx_for_upgrade( enum IndexSchemaState { Latest, + MarketplacePreviewUpgrade, + KinicExternalBlockIndexesUpgrade, + MarketplaceCoreUpgrade, StorageBillingBatchUpgrade, Mainnet011, } @@ -2644,8 +3651,26 @@ fn ensure_existing_index_schema_is_latest( ) -> Result<(), String> { match classify_existing_index_schema_state(conn)? { IndexSchemaState::Latest => validate_index_schema(conn), + IndexSchemaState::MarketplacePreviewUpgrade => { + apply_marketplace_preview_index_migration(conn)?; + validate_index_schema(conn) + } + IndexSchemaState::KinicExternalBlockIndexesUpgrade => { + apply_kinic_external_block_indexes_migration(conn)?; + apply_marketplace_preview_index_migration(conn)?; + validate_index_schema(conn) + } + IndexSchemaState::MarketplaceCoreUpgrade => { + apply_marketplace_core_index_migration(conn)?; + apply_kinic_external_block_indexes_migration(conn)?; + apply_marketplace_preview_index_migration(conn)?; + validate_index_schema(conn) + } IndexSchemaState::StorageBillingBatchUpgrade => { apply_storage_billing_batch_index_migration(conn)?; + apply_marketplace_core_index_migration(conn)?; + apply_kinic_external_block_indexes_migration(conn)?; + apply_marketplace_preview_index_migration(conn)?; validate_index_schema(conn) } IndexSchemaState::Mainnet011 => { @@ -2684,11 +3709,20 @@ fn classify_existing_index_schema_state( "unsupported partial billing index schema: table {table} already exists" )); } - if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_STORAGE_BILLING_BATCH)? { + if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_MARKETPLACE_PREVIEW)? { return Ok(IndexSchemaState::Latest); } - if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX)? { - return Ok(IndexSchemaState::StorageBillingBatchUpgrade); + if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_KINIC_EXTERNAL_BLOCK_INDEXES)? { + return Ok(IndexSchemaState::MarketplacePreviewUpgrade); + } + if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_MARKETPLACE_CORE)? { + return Ok(IndexSchemaState::KinicExternalBlockIndexesUpgrade); + } + if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_STORAGE_BILLING_BATCH)? { + return Ok(IndexSchemaState::MarketplaceCoreUpgrade); + } + if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX)? { + return Ok(IndexSchemaState::StorageBillingBatchUpgrade); } if !migration_applied_tx(conn, INDEX_SCHEMA_VERSION_SOURCE_RUN_SESSIONS)? { return Err(format!( @@ -2741,6 +3775,182 @@ fn apply_storage_billing_batch_index_migration(conn: &Transaction<'_>) -> Result Ok(()) } +fn apply_marketplace_core_index_migration(conn: &Transaction<'_>) -> Result<(), String> { + conn.execute_batch( + " + CREATE TABLE kinic_accounts ( + principal TEXT PRIMARY KEY, + balance_e8s INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + + CREATE TABLE kinic_ledger ( + entry_id INTEGER PRIMARY KEY AUTOINCREMENT, + principal TEXT NOT NULL, + source TEXT NOT NULL, + kind TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + balance_after_e8s INTEGER NOT NULL, + counterparty TEXT, + listing_id TEXT, + order_id TEXT, + external_block_index INTEGER, + created_at_ms INTEGER NOT NULL + ); + + CREATE INDEX kinic_ledger_principal_idx + ON kinic_ledger(principal, entry_id); + + CREATE TABLE kinic_pending_operations ( + operation_id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + caller TEXT NOT NULL, + amount_e8s INTEGER NOT NULL, + from_owner TEXT, + from_subaccount BLOB, + to_owner TEXT, + to_subaccount BLOB, + ledger_fee_e8s INTEGER, + operation_status TEXT NOT NULL, + external_block_index INTEGER, + ledger_created_at_time_ns INTEGER, + created_at_ms INTEGER NOT NULL + ); + + CREATE INDEX kinic_pending_operations_caller_idx + ON kinic_pending_operations(caller, operation_id); + + CREATE TABLE market_listings ( + listing_id TEXT PRIMARY KEY, + seller_principal TEXT NOT NULL, + database_id TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + llm_summary TEXT, + summary_snapshot_revision TEXT, + sample_excerpts_json TEXT NOT NULL, + tags_json TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + status TEXT NOT NULL, + revision INTEGER NOT NULL, + purchase_count INTEGER NOT NULL, + report_count INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + FOREIGN KEY (database_id) REFERENCES databases(database_id) + ); + + CREATE INDEX market_listings_status_idx + ON market_listings(status, listing_id); + + CREATE INDEX market_listings_database_idx + ON market_listings(database_id); + + CREATE TABLE market_orders ( + order_id TEXT PRIMARY KEY, + listing_id TEXT NOT NULL, + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + seller_principal TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + listing_revision INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL + ); + + CREATE INDEX market_orders_buyer_idx + ON market_orders(buyer_principal, order_id); + + CREATE TABLE market_entitlements ( + database_id TEXT NOT NULL, + buyer_principal TEXT NOT NULL, + listing_id TEXT NOT NULL, + order_id TEXT NOT NULL, + purchased_at_ms INTEGER NOT NULL, + status TEXT NOT NULL, + PRIMARY KEY (database_id, buyer_principal, listing_id), + FOREIGN KEY (database_id) REFERENCES databases(database_id) + ); + + CREATE UNIQUE INDEX market_entitlements_database_buyer_active_idx + ON market_entitlements(database_id, buyer_principal) + WHERE status = 'active'; + + CREATE INDEX market_entitlements_buyer_idx + ON market_entitlements(buyer_principal, database_id); + ", + ) + .map_err(|error| error.to_string())?; + insert_schema_migration_now(conn, INDEX_SCHEMA_VERSION_MARKETPLACE_CORE)?; + Ok(()) +} + +fn apply_marketplace_preview_index_migration(conn: &Transaction<'_>) -> Result<(), String> { + conn.execute_batch( + " + CREATE TABLE market_listings_next ( + listing_id TEXT PRIMARY KEY, + seller_principal TEXT NOT NULL, + database_id TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + llm_summary TEXT, + summary_snapshot_revision TEXT, + sample_excerpts_json TEXT NOT NULL, + tags_json TEXT NOT NULL, + price_e8s INTEGER NOT NULL, + status TEXT NOT NULL, + revision INTEGER NOT NULL, + purchase_count INTEGER NOT NULL, + report_count INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL, + FOREIGN KEY (database_id) REFERENCES databases(database_id) + ); + + INSERT INTO market_listings_next + (listing_id, seller_principal, database_id, title, description, + llm_summary, summary_snapshot_revision, sample_excerpts_json, + tags_json, price_e8s, status, revision, purchase_count, report_count, + created_at_ms, updated_at_ms) + SELECT listing_id, seller_principal, database_id, title, description, + llm_summary, summary_snapshot_revision, sample_excerpts_json, + tags_json, price_e8s, status, revision, purchase_count, report_count, + created_at_ms, updated_at_ms + FROM market_listings; + + DROP TABLE market_listings; + ALTER TABLE market_listings_next RENAME TO market_listings; + + CREATE INDEX market_listings_status_idx + ON market_listings(status, listing_id); + + CREATE INDEX market_listings_database_idx + ON market_listings(database_id); + ", + ) + .map_err(|error| error.to_string())?; + insert_schema_migration_now(conn, INDEX_SCHEMA_VERSION_MARKETPLACE_PREVIEW)?; + Ok(()) +} + +fn apply_kinic_external_block_indexes_migration(conn: &Transaction<'_>) -> Result<(), String> { + conn.execute_batch( + " + CREATE UNIQUE INDEX kinic_ledger_external_block_idx + ON kinic_ledger(external_block_index) + WHERE external_block_index IS NOT NULL; + + CREATE UNIQUE INDEX kinic_pending_operations_external_block_kind_idx + ON kinic_pending_operations(external_block_index, kind) + WHERE external_block_index IS NOT NULL; + ", + ) + .map_err(|error| error.to_string())?; + insert_schema_migration_now(conn, INDEX_SCHEMA_VERSION_KINIC_EXTERNAL_BLOCK_INDEXES)?; + Ok(()) +} + fn create_schema_migrations(conn: &Transaction<'_>) -> Result<(), String> { conn.execute( "CREATE TABLE schema_migrations (version TEXT PRIMARY KEY, applied_at INTEGER NOT NULL)", @@ -2867,6 +4077,9 @@ const INDEX_SCHEMA_VERSIONS: &[&str] = &[ INDEX_SCHEMA_VERSION_DIRECT_CYCLES, INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX, INDEX_SCHEMA_VERSION_STORAGE_BILLING_BATCH, + INDEX_SCHEMA_VERSION_MARKETPLACE_CORE, + INDEX_SCHEMA_VERSION_KINIC_EXTERNAL_BLOCK_INDEXES, + INDEX_SCHEMA_VERSION_MARKETPLACE_PREVIEW, ]; const INDEX_SCHEMA_TABLES_WITHOUT_MIGRATIONS: &[&str] = &[ @@ -2883,6 +4096,12 @@ const INDEX_SCHEMA_TABLES_WITHOUT_MIGRATIONS: &[&str] = &[ "database_cycle_pending_operations", "cycles_billing_config", "storage_billing_state", + "kinic_accounts", + "kinic_ledger", + "kinic_pending_operations", + "market_listings", + "market_orders", + "market_entitlements", ]; const POST_011_INDEX_SCHEMA_VERSIONS: &[&str] = &[ @@ -2901,6 +4120,9 @@ const POST_011_INDEX_SCHEMA_VERSIONS: &[&str] = &[ INDEX_SCHEMA_VERSION_DIRECT_CYCLES, INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX, INDEX_SCHEMA_VERSION_STORAGE_BILLING_BATCH, + INDEX_SCHEMA_VERSION_MARKETPLACE_CORE, + INDEX_SCHEMA_VERSION_KINIC_EXTERNAL_BLOCK_INDEXES, + INDEX_SCHEMA_VERSION_MARKETPLACE_PREVIEW, ]; const POST_011_INDEX_SCHEMA_TABLES: &[&str] = &[ @@ -2909,6 +4131,12 @@ const POST_011_INDEX_SCHEMA_TABLES: &[&str] = &[ "database_cycle_pending_operations", "cycles_billing_config", "storage_billing_state", + "kinic_accounts", + "kinic_ledger", + "kinic_pending_operations", + "market_listings", + "market_orders", + "market_entitlements", ]; fn validate_pre_billing_index_schema(conn: &Transaction<'_>) -> Result<(), String> { @@ -3028,6 +4256,20 @@ fn validate_pre_billing_index_schema(conn: &Transaction<'_>) -> Result<(), Strin return Err(format!("unsupported index schema: missing index {index}")); } } + if tx_sqlite_master_entry_exists(conn, "table", "market_seller_allowlist")? { + return Err("unsupported index schema: stale table market_seller_allowlist".to_string()); + } + if index_column_exists(conn, "market_entitlements", "expires_at_ms")? { + return Err( + "unsupported index schema: stale column market_entitlements.expires_at_ms".to_string(), + ); + } + if index_column_exists(conn, "market_listings", "sample_questions_json")? { + return Err( + "unsupported index schema: stale column market_listings.sample_questions_json" + .to_string(), + ); + } Ok(()) } @@ -3042,6 +4284,12 @@ fn validate_index_schema(conn: &Transaction<'_>) -> Result<(), String> { "database_cycle_pending_operations", "cycles_billing_config", "storage_billing_state", + "kinic_accounts", + "kinic_ledger", + "kinic_pending_operations", + "market_listings", + "market_orders", + "market_entitlements", ] { if !tx_sqlite_master_entry_exists(conn, "table", table)? { return Err(format!("unsupported index schema: missing table {table}")); @@ -3122,6 +4370,89 @@ fn validate_index_schema(conn: &Transaction<'_>) -> Result<(), String> { "storage_billing_state", &["key", "cursor_mount_id", "billing_now_ms", "updated_at_ms"][..], ), + ( + "kinic_accounts", + &["principal", "balance_e8s", "created_at_ms", "updated_at_ms"][..], + ), + ( + "kinic_ledger", + &[ + "entry_id", + "principal", + "source", + "kind", + "amount_e8s", + "balance_after_e8s", + "counterparty", + "listing_id", + "order_id", + "external_block_index", + "created_at_ms", + ][..], + ), + ( + "kinic_pending_operations", + &[ + "operation_id", + "kind", + "caller", + "amount_e8s", + "from_owner", + "from_subaccount", + "to_owner", + "to_subaccount", + "ledger_fee_e8s", + "operation_status", + "external_block_index", + "ledger_created_at_time_ns", + "created_at_ms", + ][..], + ), + ( + "market_listings", + &[ + "listing_id", + "seller_principal", + "database_id", + "title", + "description", + "llm_summary", + "summary_snapshot_revision", + "sample_excerpts_json", + "tags_json", + "price_e8s", + "status", + "revision", + "purchase_count", + "report_count", + "created_at_ms", + "updated_at_ms", + ][..], + ), + ( + "market_orders", + &[ + "order_id", + "listing_id", + "database_id", + "buyer_principal", + "seller_principal", + "price_e8s", + "listing_revision", + "created_at_ms", + ][..], + ), + ( + "market_entitlements", + &[ + "database_id", + "buyer_principal", + "listing_id", + "order_id", + "purchased_at_ms", + "status", + ][..], + ), ] { for column in columns { if !index_column_exists(conn, table, column)? { @@ -3136,6 +4467,15 @@ fn validate_index_schema(conn: &Transaction<'_>) -> Result<(), String> { "database_restore_chunks_database_id_idx", "database_cycle_ledger_database_idx", "database_cycle_pending_operations_database_idx", + "kinic_ledger_principal_idx", + "kinic_ledger_external_block_idx", + "kinic_pending_operations_caller_idx", + "kinic_pending_operations_external_block_kind_idx", + "market_listings_status_idx", + "market_listings_database_idx", + "market_orders_buyer_idx", + "market_entitlements_database_buyer_active_idx", + "market_entitlements_buyer_idx", ] { if !tx_sqlite_master_entry_exists(conn, "index", index)? { return Err(format!("unsupported index schema: missing index {index}")); @@ -3438,6 +4778,8 @@ fn delete_database_index_rows(conn: &Connection, database_id: &str) -> Result<() "database_cycle_pending_operations", "database_cycle_ledger", "database_cycle_accounts", + "market_entitlements", + "market_listings", "database_members", "database_restore_chunks", "database_restore_sessions", @@ -3577,144 +4919,417 @@ fn update_database_cycles_balance( WHERE database_id = ?1", &values, ) - .map_err(|error| error.to_string())?; - Ok(()) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn load_storage_cycle_account( + conn: &Connection, + database_id: &str, +) -> Result { + conn.query_row( + "SELECT balance_cycles, suspended_at_ms, storage_charged_at_ms + FROM database_cycle_accounts + WHERE database_id = ?1", + params![database_id], + |row| { + Ok(StorageCycleAccount { + balance_cycles: crate::sqlite::row_get(row, 0)?, + suspended_at_ms: crate::sqlite::row_get(row, 1)?, + storage_charged_at_ms: crate::sqlite::row_get(row, 2)?, + }) + }, + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("database cycles account not found: {database_id}")) +} + +fn update_database_storage_account( + conn: &Transaction<'_>, + database_id: &str, + balance_cycles: i64, + suspended_at_ms: Option, + storage_charged_at_ms: i64, + now: i64, +) -> Result<(), String> { + let values = vec![ + crate::sqlite::text_value(database_id), + crate::sqlite::integer_value(balance_cycles), + crate::sqlite::nullable_integer_value(suspended_at_ms), + crate::sqlite::integer_value(storage_charged_at_ms), + crate::sqlite::integer_value(now), + ]; + crate::sqlite::execute_values( + conn, + "UPDATE database_cycle_accounts + SET balance_cycles = ?2, + suspended_at_ms = ?3, + storage_charged_at_ms = ?4, + updated_at_ms = ?5 + WHERE database_id = ?1", + &values, + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +struct PendingCyclesOperation { + database_id: String, + kind: String, + caller: String, + cycles: i64, + payment_amount_e8s: i64, + operation_status: String, + ledger_block_index: Option, +} + +struct DatabaseCyclesPendingPurchaseRaw { + operation_id: i64, + database_id: String, + caller: String, + status: String, + amount_cycles: i64, + payment_amount_e8s: i64, + ledger_block_index: Option, + created_at_ms: i64, +} + +impl DatabaseCyclesPendingPurchaseRaw { + fn into_public(self) -> Result { + let amount_cycles = u64::try_from(self.amount_cycles).map_err(|error| error.to_string())?; + let payment_amount_e8s = + u64::try_from(self.payment_amount_e8s).map_err(|error| error.to_string())?; + let operation_id = u64::try_from(self.operation_id).map_err(|error| error.to_string())?; + let ledger_block_index = self + .ledger_block_index + .map(u64::try_from) + .transpose() + .map_err(|error| error.to_string())?; + Ok(DatabaseCyclesPendingPurchase { + operation_id, + database_id: self.database_id, + status: self.status.clone(), + amount_cycles, + payment_amount_e8s, + ledger_block_index, + created_at_ms: self.created_at_ms, + required_action: pending_cycles_required_action(&self.status).to_string(), + }) + } +} + +struct PendingCyclesLedgerDetails<'a> { + from_owner: &'a str, + from_subaccount: Option<&'a [u8]>, + to_owner: &'a str, + to_subaccount: Option<&'a [u8]>, + ledger_fee_e8s: i64, + ledger_created_at_time_ns: i64, +} + +struct PendingCyclesOperationInsert<'a> { + database_id: &'a str, + kind: &'a str, + caller: &'a str, + cycles: i64, + payment_amount_e8s: i64, + ledger: PendingCyclesLedgerDetails<'a>, + operation_status: &'a str, + now: i64, +} + +struct PendingCyclesOperationMatch<'a> { + operation_id: u64, + database_id: &'a str, + kind: &'a str, + caller: &'a str, + cycles: i64, +} + +fn insert_pending_cycles_operation( + conn: &Transaction<'_>, + operation: PendingCyclesOperationInsert<'_>, +) -> Result { + let values = vec![ + crate::sqlite::text_value(operation.database_id), + crate::sqlite::text_value(operation.kind), + crate::sqlite::text_value(operation.caller), + crate::sqlite::integer_value(operation.cycles), + crate::sqlite::integer_value(operation.payment_amount_e8s), + crate::sqlite::text_value(operation.ledger.from_owner), + crate::sqlite::nullable_blob_value(operation.ledger.from_subaccount.map(Vec::from)), + crate::sqlite::text_value(operation.ledger.to_owner), + crate::sqlite::nullable_blob_value(operation.ledger.to_subaccount.map(Vec::from)), + crate::sqlite::integer_value(operation.ledger.ledger_fee_e8s), + crate::sqlite::integer_value(operation.ledger.ledger_created_at_time_ns), + crate::sqlite::text_value(operation.operation_status), + crate::sqlite::integer_value(operation.now), + ]; + crate::sqlite::execute_values( + conn, + "INSERT INTO database_cycle_pending_operations + (database_id, kind, caller, cycles, payment_amount_e8s, from_owner, from_subaccount, + to_owner, to_subaccount, ledger_fee_e8s, ledger_created_at_time_ns, operation_status, + created_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + &values, + ) + .map_err(|error| error.to_string())?; + let operation_id = crate::sqlite::last_insert_rowid(conn).map_err(|error| error.to_string())?; + u64::try_from(operation_id).map_err(|error| error.to_string()) +} + +fn load_pending_cycles_operation( + conn: &Connection, + operation_id: u64, +) -> Result { + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + conn.query_row( + "SELECT database_id, kind, caller, cycles, payment_amount_e8s, + from_owner, from_subaccount, to_owner, to_subaccount, + ledger_fee_e8s, ledger_created_at_time_ns, operation_status, ledger_block_index + FROM database_cycle_pending_operations + WHERE operation_id = ?1", + params![operation_id], + map_pending_cycles_operation, + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| "pending cycle operation not found".to_string()) +} + +fn require_pending_operation_status( + operation: &PendingCyclesOperation, + allowed: &[&str], + action: &str, +) -> Result<(), String> { + if allowed + .iter() + .any(|status| operation.operation_status == *status) + { + return Ok(()); + } + Err(format!( + "cannot {action}; cycle purchase operation is {}", + operation.operation_status + )) +} + +fn load_required_pending_cycles_operation( + conn: &Transaction<'_>, + expected: PendingCyclesOperationMatch<'_>, +) -> Result { + let operation = load_pending_cycles_operation(conn, expected.operation_id)?; + if operation.database_id != expected.database_id + || operation.kind != expected.kind + || operation.caller != expected.caller + || operation.cycles != expected.cycles + { + return Err("pending cycle operation mismatch".to_string()); + } + Ok(operation) +} + +fn delete_pending_cycles_operation( + conn: &Transaction<'_>, + operation_id: u64, +) -> Result<(), String> { + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + conn.execute( + "DELETE FROM database_cycle_pending_operations WHERE operation_id = ?1", + params![operation_id], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn ensure_no_pending_cycles_purchase_for_caller( + conn: &Connection, + database_id: &str, + caller: &str, +) -> Result<(), String> { + let count: i64 = conn + .query_row( + "SELECT COUNT(*) + FROM database_cycle_pending_operations + WHERE database_id = ?1 + AND caller = ?2 + AND kind = 'cycles_purchase'", + params![database_id, caller], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string())?; + if count > 0 { + return Err("cycles purchase already pending for caller".to_string()); + } + Ok(()) +} + +fn load_database_cycles_pending_purchase_statuses( + conn: &Connection, + database_id: &str, +) -> Result, String> { + let mut stmt = conn + .prepare( + "SELECT operation_id, database_id, caller, operation_status, cycles, + payment_amount_e8s, ledger_block_index, created_at_ms + FROM database_cycle_pending_operations + WHERE database_id = ?1 AND kind = 'cycles_purchase' + ORDER BY operation_id ASC", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map( + &mut stmt, + params![database_id], + map_database_cycles_pending_purchase_raw, + ) + .map_err(|error| error.to_string()) } -fn load_storage_cycle_account( +fn first_database_cycles_pending_purchase_status( conn: &Connection, database_id: &str, -) -> Result { +) -> Result, String> { conn.query_row( - "SELECT balance_cycles, suspended_at_ms, storage_charged_at_ms - FROM database_cycle_accounts - WHERE database_id = ?1", + "SELECT operation_id, database_id, caller, operation_status, cycles, + payment_amount_e8s, ledger_block_index, created_at_ms + FROM database_cycle_pending_operations + WHERE database_id = ?1 AND kind = 'cycles_purchase' + ORDER BY operation_id ASC + LIMIT 1", params![database_id], - |row| { - Ok(StorageCycleAccount { - balance_cycles: crate::sqlite::row_get(row, 0)?, - suspended_at_ms: crate::sqlite::row_get(row, 1)?, - storage_charged_at_ms: crate::sqlite::row_get(row, 2)?, - }) - }, + map_database_cycles_pending_purchase_raw, ) .optional() .map_err(|error| error.to_string())? - .ok_or_else(|| format!("database cycles account not found: {database_id}")) + .map(DatabaseCyclesPendingPurchaseRaw::into_public) + .transpose() } -fn update_database_storage_account( - conn: &Transaction<'_>, - database_id: &str, - balance_cycles: i64, - suspended_at_ms: Option, - storage_charged_at_ms: i64, - now: i64, -) -> Result<(), String> { - let values = vec![ - crate::sqlite::text_value(database_id), - crate::sqlite::integer_value(balance_cycles), - crate::sqlite::nullable_integer_value(suspended_at_ms), - crate::sqlite::integer_value(storage_charged_at_ms), - crate::sqlite::integer_value(now), - ]; - crate::sqlite::execute_values( - conn, - "UPDATE database_cycle_accounts - SET balance_cycles = ?2, - suspended_at_ms = ?3, - storage_charged_at_ms = ?4, - updated_at_ms = ?5 - WHERE database_id = ?1", - &values, - ) - .map_err(|error| error.to_string())?; - Ok(()) +fn map_database_cycles_pending_purchase_raw( + row: &crate::sqlite::Row<'_>, +) -> crate::sqlite::Result { + Ok(DatabaseCyclesPendingPurchaseRaw { + operation_id: crate::sqlite::row_get(row, 0)?, + database_id: crate::sqlite::row_get(row, 1)?, + caller: crate::sqlite::row_get(row, 2)?, + status: crate::sqlite::row_get(row, 3)?, + amount_cycles: crate::sqlite::row_get(row, 4)?, + payment_amount_e8s: crate::sqlite::row_get(row, 5)?, + ledger_block_index: crate::sqlite::row_get(row, 6)?, + created_at_ms: crate::sqlite::row_get(row, 7)?, + }) } -struct PendingCyclesOperation { - database_id: String, +fn pending_cycles_required_action(status: &str) -> &'static str { + match status { + CYCLES_OPERATION_STATUS_IN_FLIGHT => "wait_for_ledger_result", + CYCLES_OPERATION_STATUS_AMBIGUOUS | CYCLES_OPERATION_STATUS_COMPLETED => { + "billing_authority_review" + } + _ => "billing_authority_review", + } +} + +fn map_pending_cycles_operation( + row: &crate::sqlite::Row<'_>, +) -> crate::sqlite::Result { + Ok(PendingCyclesOperation { + database_id: crate::sqlite::row_get(row, 0)?, + kind: crate::sqlite::row_get(row, 1)?, + caller: crate::sqlite::row_get(row, 2)?, + cycles: crate::sqlite::row_get(row, 3)?, + payment_amount_e8s: crate::sqlite::row_get(row, 4)?, + operation_status: crate::sqlite::row_get(row, 11)?, + ledger_block_index: crate::sqlite::row_get(row, 12)?, + }) +} + +struct PendingKinicOperation { kind: String, caller: String, - cycles: i64, - payment_amount_e8s: i64, + amount_e8s: i64, operation_status: String, - ledger_block_index: Option, + external_block_index: Option, } -struct DatabaseCyclesPendingPurchaseRaw { +struct PendingKinicOperationInsert<'a> { + kind: &'a str, + caller: &'a str, + amount_e8s: i64, + ledger: PendingCyclesLedgerDetails<'a>, + operation_status: &'a str, + now: i64, +} + +struct PendingKinicOperationMatch<'a> { + operation_id: u64, + kind: &'a str, + caller: &'a str, + amount_e8s: i64, +} + +struct PendingKinicOperationRaw { operation_id: i64, - database_id: String, + kind: String, caller: String, status: String, - amount_cycles: i64, - payment_amount_e8s: i64, - ledger_block_index: Option, + amount_e8s: i64, + external_block_index: Option, created_at_ms: i64, } -impl DatabaseCyclesPendingPurchaseRaw { - fn into_public(self) -> Result { - let amount_cycles = u64::try_from(self.amount_cycles).map_err(|error| error.to_string())?; - let payment_amount_e8s = - u64::try_from(self.payment_amount_e8s).map_err(|error| error.to_string())?; +impl PendingKinicOperationRaw { + fn into_public(self) -> Result { let operation_id = u64::try_from(self.operation_id).map_err(|error| error.to_string())?; + let amount_e8s = u64::try_from(self.amount_e8s).map_err(|error| error.to_string())?; let ledger_block_index = self - .ledger_block_index + .external_block_index .map(u64::try_from) .transpose() .map_err(|error| error.to_string())?; - Ok(DatabaseCyclesPendingPurchase { + Ok(KinicPendingOperation { operation_id, - database_id: self.database_id, + kind: self.kind, + caller: self.caller, status: self.status.clone(), - amount_cycles, - payment_amount_e8s, + amount_e8s, ledger_block_index, created_at_ms: self.created_at_ms, - required_action: pending_cycles_required_action(&self.status).to_string(), + required_action: pending_kinic_required_action(&self.status).to_string(), }) } } -struct PendingCyclesLedgerDetails<'a> { - from_owner: &'a str, - from_subaccount: Option<&'a [u8]>, - to_owner: &'a str, - to_subaccount: Option<&'a [u8]>, - ledger_fee_e8s: i64, - ledger_created_at_time_ns: i64, -} - -struct PendingCyclesOperationInsert<'a> { - database_id: &'a str, +struct KinicLedgerInsert<'a> { + principal: &'a str, + source: &'a str, kind: &'a str, - caller: &'a str, - cycles: i64, - payment_amount_e8s: i64, - ledger: PendingCyclesLedgerDetails<'a>, - operation_status: &'a str, + amount_e8s: i64, + balance_after_e8s: i64, + counterparty: Option<&'a str>, + listing_id: Option<&'a str>, + order_id: Option<&'a str>, + external_block_index: Option, now: i64, } -struct PendingCyclesOperationMatch<'a> { - operation_id: u64, - database_id: &'a str, - kind: &'a str, - caller: &'a str, - cycles: i64, +fn require_authenticated_principal(caller: &str) -> Result<(), String> { + if caller == ANONYMOUS_PRINCIPAL { + return Err("anonymous caller not allowed".to_string()); + } + Ok(()) } -fn insert_pending_cycles_operation( +fn insert_pending_kinic_operation( conn: &Transaction<'_>, - operation: PendingCyclesOperationInsert<'_>, + operation: PendingKinicOperationInsert<'_>, ) -> Result { let values = vec![ - crate::sqlite::text_value(operation.database_id), crate::sqlite::text_value(operation.kind), crate::sqlite::text_value(operation.caller), - crate::sqlite::integer_value(operation.cycles), - crate::sqlite::integer_value(operation.payment_amount_e8s), + crate::sqlite::integer_value(operation.amount_e8s), crate::sqlite::text_value(operation.ledger.from_owner), crate::sqlite::nullable_blob_value(operation.ledger.from_subaccount.map(Vec::from)), crate::sqlite::text_value(operation.ledger.to_owner), @@ -3726,11 +5341,11 @@ fn insert_pending_cycles_operation( ]; crate::sqlite::execute_values( conn, - "INSERT INTO database_cycle_pending_operations - (database_id, kind, caller, cycles, payment_amount_e8s, from_owner, from_subaccount, - to_owner, to_subaccount, ledger_fee_e8s, ledger_created_at_time_ns, operation_status, - created_at_ms) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + "INSERT INTO kinic_pending_operations + (kind, caller, amount_e8s, from_owner, from_subaccount, to_owner, + to_subaccount, ledger_fee_e8s, ledger_created_at_time_ns, + operation_status, created_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", &values, ) .map_err(|error| error.to_string())?; @@ -3738,172 +5353,610 @@ fn insert_pending_cycles_operation( u64::try_from(operation_id).map_err(|error| error.to_string()) } -fn load_pending_cycles_operation( - conn: &Connection, - operation_id: u64, -) -> Result { - let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; +fn load_pending_kinic_operation( + conn: &Connection, + operation_id: u64, +) -> Result { + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + conn.query_row( + "SELECT kind, caller, amount_e8s, operation_status, external_block_index + FROM kinic_pending_operations + WHERE operation_id = ?1", + params![operation_id], + |row| { + Ok(PendingKinicOperation { + kind: crate::sqlite::row_get(row, 0)?, + caller: crate::sqlite::row_get(row, 1)?, + amount_e8s: crate::sqlite::row_get(row, 2)?, + operation_status: crate::sqlite::row_get(row, 3)?, + external_block_index: crate::sqlite::row_get(row, 4)?, + }) + }, + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| "pending KINIC operation not found".to_string()) +} + +fn load_required_pending_kinic_operation( + conn: &Transaction<'_>, + expected: PendingKinicOperationMatch<'_>, +) -> Result { + let operation = load_pending_kinic_operation(conn, expected.operation_id)?; + if operation.kind != expected.kind + || operation.caller != expected.caller + || operation.amount_e8s != expected.amount_e8s + { + return Err("pending KINIC operation mismatch".to_string()); + } + Ok(operation) +} + +fn require_pending_kinic_operation_status( + operation: &PendingKinicOperation, + allowed: &[&str], + action: &str, +) -> Result<(), String> { + if allowed + .iter() + .any(|status| operation.operation_status == *status) + { + return Ok(()); + } + Err(format!( + "cannot {action}; KINIC operation is {}", + operation.operation_status + )) +} + +fn delete_pending_kinic_operation(conn: &Transaction<'_>, operation_id: u64) -> Result<(), String> { + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + conn.execute( + "DELETE FROM kinic_pending_operations WHERE operation_id = ?1", + params![operation_id], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn ensure_no_pending_kinic_deposit_for_caller( + conn: &Connection, + caller: &str, +) -> Result<(), String> { + let count: i64 = conn + .query_row( + "SELECT COUNT(*) + FROM kinic_pending_operations + WHERE caller = ?1 + AND kind = ?2 + AND operation_status IN (?3, ?4, ?5)", + params![ + caller, + KINIC_PENDING_KIND_DEPOSIT, + KINIC_OPERATION_STATUS_IN_FLIGHT, + KINIC_OPERATION_STATUS_COMPLETED, + KINIC_OPERATION_STATUS_AMBIGUOUS + ], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string())?; + if count > 0 { + return Err("KINIC deposit already pending for caller".to_string()); + } + Ok(()) +} + +fn load_kinic_pending_operations( + conn: &Connection, + caller: &str, + show_all: bool, + cursor_operation_id: Option, + limit: u32, +) -> Result, String> { + let cursor = cursor_operation_id + .map(i64::try_from) + .transpose() + .map_err(|_| "pending KINIC operation cursor is out of range".to_string())? + .unwrap_or(0); + let fetch_limit = i64::from(limit.saturating_add(1)); + if show_all { + let mut stmt = conn + .prepare( + "SELECT operation_id, kind, caller, operation_status, amount_e8s, + external_block_index, created_at_ms + FROM kinic_pending_operations + WHERE kind = ?1 + AND operation_id > ?2 + ORDER BY operation_id ASC + LIMIT ?3", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map( + &mut stmt, + params![KINIC_PENDING_KIND_DEPOSIT, cursor, fetch_limit], + map_pending_kinic_operation_raw, + ) + .map_err(|error| error.to_string()) + } else { + let mut stmt = conn + .prepare( + "SELECT operation_id, kind, caller, operation_status, amount_e8s, + external_block_index, created_at_ms + FROM kinic_pending_operations + WHERE kind = ?1 AND caller = ?2 + AND operation_id > ?3 + ORDER BY operation_id ASC + LIMIT ?4", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map( + &mut stmt, + params![KINIC_PENDING_KIND_DEPOSIT, caller, cursor, fetch_limit], + map_pending_kinic_operation_raw, + ) + .map_err(|error| error.to_string()) + } +} + +fn clamp_kinic_pending_operations_page_limit(limit: u32) -> u32 { + limit.clamp(1, MAX_KINIC_PENDING_OPERATIONS_PAGE_LIMIT) +} + +fn map_pending_kinic_operation_raw( + row: &crate::sqlite::Row<'_>, +) -> crate::sqlite::Result { + Ok(PendingKinicOperationRaw { + operation_id: crate::sqlite::row_get(row, 0)?, + kind: crate::sqlite::row_get(row, 1)?, + caller: crate::sqlite::row_get(row, 2)?, + status: crate::sqlite::row_get(row, 3)?, + amount_e8s: crate::sqlite::row_get(row, 4)?, + external_block_index: crate::sqlite::row_get(row, 5)?, + created_at_ms: crate::sqlite::row_get(row, 6)?, + }) +} + +fn pending_kinic_required_action(status: &str) -> &'static str { + match status { + KINIC_OPERATION_STATUS_IN_FLIGHT => "wait_for_ledger_result", + KINIC_OPERATION_STATUS_AMBIGUOUS | KINIC_OPERATION_STATUS_COMPLETED => { + "billing_authority_review" + } + _ => "billing_authority_review", + } +} + +fn load_kinic_balance(conn: &Connection, principal: &str) -> Result { conn.query_row( - "SELECT database_id, kind, caller, cycles, payment_amount_e8s, - from_owner, from_subaccount, to_owner, to_subaccount, - ledger_fee_e8s, ledger_created_at_time_ns, operation_status, ledger_block_index - FROM database_cycle_pending_operations - WHERE operation_id = ?1", - params![operation_id], - map_pending_cycles_operation, + "SELECT balance_e8s FROM kinic_accounts WHERE principal = ?1", + params![principal], + |row| crate::sqlite::row_get(row, 0), ) .optional() - .map_err(|error| error.to_string())? - .ok_or_else(|| "pending cycle operation not found".to_string()) -} - -fn require_pending_operation_status( - operation: &PendingCyclesOperation, - allowed: &[&str], - action: &str, -) -> Result<(), String> { - if allowed - .iter() - .any(|status| operation.operation_status == *status) - { - return Ok(()); - } - Err(format!( - "cannot {action}; cycle purchase operation is {}", - operation.operation_status - )) + .map_err(|error| error.to_string()) + .map(|balance| balance.unwrap_or(0)) } -fn load_required_pending_cycles_operation( +fn upsert_kinic_balance( conn: &Transaction<'_>, - expected: PendingCyclesOperationMatch<'_>, -) -> Result { - let operation = load_pending_cycles_operation(conn, expected.operation_id)?; - if operation.database_id != expected.database_id - || operation.kind != expected.kind - || operation.caller != expected.caller - || operation.cycles != expected.cycles - { - return Err("pending cycle operation mismatch".to_string()); - } - Ok(operation) + principal: &str, + balance_e8s: i64, + now: i64, +) -> Result<(), String> { + conn.execute( + "INSERT INTO kinic_accounts (principal, balance_e8s, created_at_ms, updated_at_ms) + VALUES (?1, ?2, ?3, ?3) + ON CONFLICT(principal) DO UPDATE SET + balance_e8s = excluded.balance_e8s, + updated_at_ms = excluded.updated_at_ms", + params![principal, balance_e8s, now], + ) + .map_err(|error| error.to_string())?; + Ok(()) } -fn delete_pending_cycles_operation( - conn: &Transaction<'_>, - operation_id: u64, -) -> Result<(), String> { - let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; +fn insert_kinic_ledger(conn: &Transaction<'_>, entry: KinicLedgerInsert<'_>) -> Result<(), String> { conn.execute( - "DELETE FROM database_cycle_pending_operations WHERE operation_id = ?1", - params![operation_id], + "INSERT INTO kinic_ledger + (principal, source, kind, amount_e8s, balance_after_e8s, counterparty, + listing_id, order_id, external_block_index, created_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + entry.principal, + entry.source, + entry.kind, + entry.amount_e8s, + entry.balance_after_e8s, + crate::sqlite::nullable_text_value(entry.counterparty.map(str::to_string)), + crate::sqlite::nullable_text_value(entry.listing_id.map(str::to_string)), + crate::sqlite::nullable_text_value(entry.order_id.map(str::to_string)), + crate::sqlite::nullable_integer_value(entry.external_block_index), + entry.now + ], ) .map_err(|error| error.to_string())?; Ok(()) } -fn ensure_no_pending_cycles_purchase_for_caller( +fn require_market_seller_can_list( conn: &Connection, + seller: &str, database_id: &str, +) -> Result<(), String> { + let status = load_database_status(conn, database_id)?; + if status != DatabaseStatus::Active { + return Err(format!( + "database is {}: {database_id}", + status_to_db(status) + )); + } + let role = load_member_role(conn, database_id, seller)? + .ok_or_else(|| format!("principal has no access to database: {database_id}"))?; + if role != DatabaseRole::Owner { + return Err("market seller must be database owner".to_string()); + } + Ok(()) +} + +fn require_market_listing_seller_or_admin( + conn: &Connection, caller: &str, + listing: &MarketListing, +) -> Result<(), String> { + let config = load_cycles_billing_config(conn)?; + if caller == listing.seller_principal || caller == config.billing_authority_id { + return Ok(()); + } + Err("market listing seller or admin required".to_string()) +} + +fn require_market_listing_purchasable( + conn: &Connection, + listing: &MarketListing, ) -> Result<(), String> { + if listing.status != MarketListingStatus::Active { + return Err("market listing is not active".to_string()); + } + require_market_seller_can_list(conn, &listing.seller_principal, &listing.database_id) +} + +fn has_active_market_entitlement( + conn: &Connection, + database_id: &str, + caller: &str, +) -> Result { + if caller == ANONYMOUS_PRINCIPAL { + return Ok(false); + } let count: i64 = conn .query_row( "SELECT COUNT(*) - FROM database_cycle_pending_operations + FROM market_entitlements WHERE database_id = ?1 - AND caller = ?2 - AND kind = 'cycles_purchase'", - params![database_id, caller], + AND buyer_principal = ?2 + AND status = ?3", + params![database_id, caller, MARKET_ENTITLEMENT_STATUS_ACTIVE], |row| crate::sqlite::row_get(row, 0), ) .map_err(|error| error.to_string())?; - if count > 0 { - return Err("cycles purchase already pending for caller".to_string()); - } - Ok(()) + Ok(count > 0) } -fn load_database_cycles_pending_purchase_statuses( +fn load_market_listing_by_id( conn: &Connection, - database_id: &str, -) -> Result, String> { - let mut stmt = conn - .prepare( - "SELECT operation_id, database_id, caller, operation_status, cycles, - payment_amount_e8s, ledger_block_index, created_at_ms - FROM database_cycle_pending_operations - WHERE database_id = ?1 AND kind = 'cycles_purchase' - ORDER BY operation_id ASC", - ) - .map_err(|error| error.to_string())?; - crate::sqlite::query_map( - &mut stmt, - params![database_id], - map_database_cycles_pending_purchase_raw, + listing_id: &str, +) -> Result, String> { + conn.query_row( + "SELECT listing_id, seller_principal, database_id, title, description, + llm_summary, summary_snapshot_revision, sample_excerpts_json, + tags_json, price_e8s, status, revision, + purchase_count, report_count, created_at_ms, updated_at_ms + FROM market_listings + WHERE listing_id = ?1", + params![listing_id], + map_market_listing, ) + .optional() .map_err(|error| error.to_string()) } -fn first_database_cycles_pending_purchase_status( +fn map_market_listing(row: &crate::sqlite::Row<'_>) -> crate::sqlite::Result { + let price_e8s: i64 = crate::sqlite::row_get(row, 9)?; + let revision: i64 = crate::sqlite::row_get(row, 11)?; + let purchase_count: i64 = crate::sqlite::row_get(row, 12)?; + let report_count: i64 = crate::sqlite::row_get(row, 13)?; + Ok(MarketListing { + listing_id: crate::sqlite::row_get(row, 0)?, + seller_principal: crate::sqlite::row_get(row, 1)?, + database_id: crate::sqlite::row_get(row, 2)?, + title: crate::sqlite::row_get(row, 3)?, + description: crate::sqlite::row_get(row, 4)?, + llm_summary: crate::sqlite::row_get(row, 5)?, + summary_snapshot_revision: crate::sqlite::row_get(row, 6)?, + sample_excerpts_json: crate::sqlite::row_get(row, 7)?, + tags_json: crate::sqlite::row_get(row, 8)?, + price_e8s: u64::try_from(price_e8s) + .map_err(|_| crate::sqlite::integral_value_out_of_range(9, price_e8s))?, + status: market_listing_status_from_db(&crate::sqlite::row_get::(row, 10)?)?, + revision: u64::try_from(revision) + .map_err(|_| crate::sqlite::integral_value_out_of_range(11, revision))?, + purchase_count: u64::try_from(purchase_count) + .map_err(|_| crate::sqlite::integral_value_out_of_range(12, purchase_count))?, + report_count: u64::try_from(report_count) + .map_err(|_| crate::sqlite::integral_value_out_of_range(13, report_count))?, + created_at_ms: crate::sqlite::row_get(row, 14)?, + updated_at_ms: crate::sqlite::row_get(row, 15)?, + }) +} + +fn load_market_order_by_id( conn: &Connection, - database_id: &str, -) -> Result, String> { + order_id: &str, +) -> Result, String> { conn.query_row( - "SELECT operation_id, database_id, caller, operation_status, cycles, - payment_amount_e8s, ledger_block_index, created_at_ms - FROM database_cycle_pending_operations - WHERE database_id = ?1 AND kind = 'cycles_purchase' - ORDER BY operation_id ASC - LIMIT 1", - params![database_id], - map_database_cycles_pending_purchase_raw, + "SELECT order_id, listing_id, database_id, buyer_principal, seller_principal, + price_e8s, listing_revision, created_at_ms + FROM market_orders + WHERE order_id = ?1", + params![order_id], + map_market_order, ) .optional() - .map_err(|error| error.to_string())? - .map(DatabaseCyclesPendingPurchaseRaw::into_public) - .transpose() + .map_err(|error| error.to_string()) } -fn map_database_cycles_pending_purchase_raw( - row: &crate::sqlite::Row<'_>, -) -> crate::sqlite::Result { - Ok(DatabaseCyclesPendingPurchaseRaw { - operation_id: crate::sqlite::row_get(row, 0)?, - database_id: crate::sqlite::row_get(row, 1)?, - caller: crate::sqlite::row_get(row, 2)?, - status: crate::sqlite::row_get(row, 3)?, - amount_cycles: crate::sqlite::row_get(row, 4)?, - payment_amount_e8s: crate::sqlite::row_get(row, 5)?, - ledger_block_index: crate::sqlite::row_get(row, 6)?, +fn map_market_order(row: &crate::sqlite::Row<'_>) -> crate::sqlite::Result { + let price_e8s: i64 = crate::sqlite::row_get(row, 5)?; + let listing_revision: i64 = crate::sqlite::row_get(row, 6)?; + Ok(MarketOrder { + order_id: crate::sqlite::row_get(row, 0)?, + listing_id: crate::sqlite::row_get(row, 1)?, + database_id: crate::sqlite::row_get(row, 2)?, + buyer_principal: crate::sqlite::row_get(row, 3)?, + seller_principal: crate::sqlite::row_get(row, 4)?, + price_e8s: u64::try_from(price_e8s) + .map_err(|_| crate::sqlite::integral_value_out_of_range(5, price_e8s))?, + listing_revision: u64::try_from(listing_revision) + .map_err(|_| crate::sqlite::integral_value_out_of_range(6, listing_revision))?, created_at_ms: crate::sqlite::row_get(row, 7)?, }) } -fn pending_cycles_required_action(status: &str) -> &'static str { - match status { - CYCLES_OPERATION_STATUS_IN_FLIGHT => "wait_for_ledger_result", - CYCLES_OPERATION_STATUS_AMBIGUOUS | CYCLES_OPERATION_STATUS_COMPLETED => { - "billing_authority_review" - } - _ => "billing_authority_review", +fn map_market_entitlement( + row: &crate::sqlite::Row<'_>, +) -> crate::sqlite::Result { + Ok(MarketEntitlement { + database_id: crate::sqlite::row_get(row, 0)?, + buyer_principal: crate::sqlite::row_get(row, 1)?, + listing_id: crate::sqlite::row_get(row, 2)?, + order_id: crate::sqlite::row_get(row, 3)?, + purchased_at_ms: crate::sqlite::row_get(row, 4)?, + status: crate::sqlite::row_get(row, 5)?, + }) +} + +fn market_listing_status_from_db(value: &str) -> crate::sqlite::Result { + match value { + MARKET_LISTING_STATUS_DRAFT => Ok(MarketListingStatus::Draft), + MARKET_LISTING_STATUS_ACTIVE => Ok(MarketListingStatus::Active), + MARKET_LISTING_STATUS_PAUSED => Ok(MarketListingStatus::Paused), + _ => Err(crate::sqlite::invalid_query()), } } -fn map_pending_cycles_operation( - row: &crate::sqlite::Row<'_>, -) -> crate::sqlite::Result { - Ok(PendingCyclesOperation { - database_id: crate::sqlite::row_get(row, 0)?, - kind: crate::sqlite::row_get(row, 1)?, - caller: crate::sqlite::row_get(row, 2)?, - cycles: crate::sqlite::row_get(row, 3)?, - payment_amount_e8s: crate::sqlite::row_get(row, 4)?, - operation_status: crate::sqlite::row_get(row, 11)?, - ledger_block_index: crate::sqlite::row_get(row, 12)?, +fn empty_market_listing_detail(listing: MarketListing) -> MarketListingDetail { + MarketListingDetail { + listing, + verified_stats: MarketListingVerifiedStats { + total_nodes: 0, + wiki_nodes: 0, + source_nodes: 0, + folder_nodes: 0, + markdown_chars: 0, + source_chars: 0, + link_edges: 0, + logical_size_bytes: 0, + last_content_updated_at_ms: None, + }, + preview: MarketListingPreview { + top_level_paths: Vec::new(), + excerpts: Vec::new(), + category_graph: MarketCategoryGraph { + nodes: Vec::new(), + edges: Vec::new(), + }, + preview_stale: true, + }, + } +} + +fn validate_market_create_listing_request( + request: &MarketCreateListingRequest, +) -> Result<(), String> { + validate_database_id(&request.database_id)?; + validate_market_listing_metadata(MarketListingMetadataValidation { + title: &request.title, + description: &request.description, + llm_summary: request.llm_summary.as_deref(), + summary_snapshot_revision: request.summary_snapshot_revision.as_deref(), + sample_excerpts_json: &request.sample_excerpts_json, + tags_json: &request.tags_json, + price_e8s: request.price_e8s, + }) +} + +fn validate_market_update_listing_request( + request: &MarketUpdateListingRequest, +) -> Result<(), String> { + validate_market_listing_metadata(MarketListingMetadataValidation { + title: &request.title, + description: &request.description, + llm_summary: request.llm_summary.as_deref(), + summary_snapshot_revision: request.summary_snapshot_revision.as_deref(), + sample_excerpts_json: &request.sample_excerpts_json, + tags_json: &request.tags_json, + price_e8s: request.price_e8s, }) } +struct MarketListingMetadataValidation<'a> { + title: &'a str, + description: &'a str, + llm_summary: Option<&'a str>, + summary_snapshot_revision: Option<&'a str>, + sample_excerpts_json: &'a str, + tags_json: &'a str, + price_e8s: u64, +} + +fn validate_market_listing_metadata( + input: MarketListingMetadataValidation<'_>, +) -> Result<(), String> { + if input.price_e8s == 0 { + return Err("market listing price must be positive".to_string()); + } + amount_to_i64(input.price_e8s)?; + validate_market_text( + "market listing title", + input.title, + 1, + MAX_MARKET_TITLE_CHARS, + )?; + validate_market_text( + "market listing description", + input.description, + 1, + MAX_MARKET_DESCRIPTION_CHARS, + )?; + if let Some(summary) = input.llm_summary { + validate_market_text( + "market listing summary", + summary, + 0, + MAX_MARKET_DESCRIPTION_CHARS, + )?; + } + if let Some(revision) = input.summary_snapshot_revision { + validate_market_text("market listing summary revision", revision, 0, 256)?; + } + validate_market_text( + "market listing sample excerpts", + input.sample_excerpts_json, + 0, + MAX_MARKET_JSON_CHARS, + )?; + parse_market_preview_excerpts(input.sample_excerpts_json)?; + validate_market_text( + "market listing tags", + input.tags_json, + 0, + MAX_MARKET_JSON_CHARS, + ) +} + +fn parse_market_preview_excerpts(value: &str) -> Result, String> { + let parsed = serde_json::from_str::(value) + .map_err(|error| format!("market listing sample excerpts JSON is invalid: {error}"))?; + let Value::Array(items) = parsed else { + return Err("market listing sample excerpts must be a JSON array".to_string()); + }; + if items.len() > MAX_MARKET_PREVIEW_EXCERPTS { + return Err(format!( + "market listing sample excerpts must have at most {MAX_MARKET_PREVIEW_EXCERPTS} items" + )); + } + let mut excerpts = Vec::with_capacity(items.len()); + for item in items { + let Value::Object(object) = item else { + return Err("market listing sample excerpt must be an object".to_string()); + }; + let path = market_preview_string_field(&object, "path")?; + let etag = market_preview_string_field(&object, "etag")?; + let excerpt = market_preview_string_field(&object, "excerpt")?; + validate_source_path_for_kind(&path, &NodeKind::File)?; + if !path.starts_with("/Wiki/") { + return Err("market listing sample excerpt path must be under /Wiki".to_string()); + } + validate_market_text("market listing sample excerpt etag", &etag, 1, 256)?; + validate_market_text( + "market listing sample excerpt", + &excerpt, + 1, + MAX_MARKET_PREVIEW_EXCERPT_CHARS, + )?; + excerpts.push(MarketPreviewExcerpt { + path, + etag, + excerpt, + }); + } + Ok(excerpts) +} + +fn market_preview_string_field( + object: &serde_json::Map, + field: &str, +) -> Result { + object + .get(field) + .and_then(Value::as_str) + .map(ToString::to_string) + .ok_or_else(|| format!("market listing sample excerpt {field} is required")) +} + +fn validate_market_text( + label: &str, + value: &str, + min_chars: usize, + max_chars: usize, +) -> Result<(), String> { + let count = value.chars().count(); + if count < min_chars || count > max_chars { + return Err(format!( + "{label} must be {min_chars}..{max_chars} characters" + )); + } + if value.chars().any(char::is_control) { + return Err(format!("{label} may not contain control characters")); + } + Ok(()) +} + +fn unique_market_id( + conn: &Connection, + table: &str, + column: &str, + prefix: &str, + caller: &str, + seed: &str, + now: i64, +) -> Result { + for attempt in 0..16_u32 { + let id = generated_market_id(prefix, caller, seed, now, attempt); + let sql = format!("SELECT COUNT(*) FROM {table} WHERE {column} = ?1"); + let count: i64 = conn + .query_row(&sql, params![id], |row| crate::sqlite::row_get(row, 0)) + .map_err(|error| error.to_string())?; + if count == 0 { + return Ok(id); + } + } + Err("failed to allocate market id".to_string()) +} + +fn generated_market_id(prefix: &str, caller: &str, seed: &str, now: i64, attempt: u32) -> String { + let mut hasher = Sha256::new(); + hasher.update(prefix.as_bytes()); + hasher.update(caller.as_bytes()); + hasher.update(seed.as_bytes()); + hasher.update(now.to_be_bytes()); + hasher.update(attempt.to_be_bytes()); + format!( + "{prefix}{}", + &base32_lower(&hasher.finalize())[..GENERATED_MARKET_ID_HASH_CHARS] + ) +} + struct DatabaseLedgerInsert<'a> { database_id: &'a str, kind: &'a str, @@ -5451,31 +7504,68 @@ fn load_database_summaries_for_caller( let mut stmt = conn .prepare( "SELECT d.database_id, d.name, d.status, m.role, d.logical_size_bytes, - COALESCE(b.balance_cycles, 0), b.suspended_at_ms, - d.archived_at_ms, d.deleted_at_ms - FROM databases d - INNER JOIN database_members m ON m.database_id = d.database_id - LEFT JOIN database_cycle_accounts b ON b.database_id = d.database_id - WHERE m.principal = ?1 - ORDER BY d.database_id ASC", + COALESCE(b.balance_cycles, 0), b.suspended_at_ms, + d.archived_at_ms, d.deleted_at_ms, + 0 AS access_source_rank, + CASE m.role + WHEN 'owner' THEN 0 + WHEN 'writer' THEN 1 + ELSE 2 + END AS role_rank + FROM databases d + INNER JOIN database_members m ON m.database_id = d.database_id + LEFT JOIN database_cycle_accounts b ON b.database_id = d.database_id + WHERE m.principal = ?1 + UNION ALL + SELECT d.database_id, d.name, d.status, 'reader' AS role, d.logical_size_bytes, + COALESCE(b.balance_cycles, 0), b.suspended_at_ms, + d.archived_at_ms, d.deleted_at_ms, + 1 AS access_source_rank, + 2 AS role_rank + FROM databases d + INNER JOIN market_entitlements e ON e.database_id = d.database_id + LEFT JOIN database_cycle_accounts b ON b.database_id = d.database_id + WHERE e.buyer_principal = ?2 + AND e.status = ?3 + AND d.status = ?4 + ORDER BY 1 ASC, 10 ASC, 11 ASC", ) .map_err(|error| error.to_string())?; - crate::sqlite::query_map(&mut stmt, params![caller], |row| { - let logical_size_bytes: i64 = crate::sqlite::row_get(row, 4)?; - let cycles_balance: i64 = crate::sqlite::row_get(row, 5)?; - Ok(DatabaseSummary { - database_id: crate::sqlite::row_get(row, 0)?, - name: crate::sqlite::row_get(row, 1)?, - status: status_from_db(&crate::sqlite::row_get::(row, 2)?)?, - role: role_from_db(&crate::sqlite::row_get::(row, 3)?)?, - logical_size_bytes: logical_size_bytes.max(0) as u64, - cycles_balance: Some(cycles_balance.max(0) as u64), - cycles_suspended_at_ms: crate::sqlite::row_get(row, 6)?, - archived_at_ms: crate::sqlite::row_get(row, 7)?, - deleted_at_ms: crate::sqlite::row_get(row, 8)?, - }) - }) - .map_err(|error| error.to_string()) + let rows = crate::sqlite::query_map( + &mut stmt, + params![ + caller, + caller, + MARKET_ENTITLEMENT_STATUS_ACTIVE, + status_to_db(DatabaseStatus::Active) + ], + |row| { + let logical_size_bytes: i64 = crate::sqlite::row_get(row, 4)?; + let cycles_balance: i64 = crate::sqlite::row_get(row, 5)?; + Ok(DatabaseSummary { + database_id: crate::sqlite::row_get(row, 0)?, + name: crate::sqlite::row_get(row, 1)?, + status: status_from_db(&crate::sqlite::row_get::(row, 2)?)?, + role: role_from_db(&crate::sqlite::row_get::(row, 3)?)?, + logical_size_bytes: logical_size_bytes.max(0) as u64, + cycles_balance: Some(cycles_balance.max(0) as u64), + cycles_suspended_at_ms: crate::sqlite::row_get(row, 6)?, + archived_at_ms: crate::sqlite::row_get(row, 7)?, + deleted_at_ms: crate::sqlite::row_get(row, 8)?, + }) + }, + ) + .map_err(|error| error.to_string())?; + let mut summaries = Vec::new(); + for row in rows { + if summaries + .last() + .is_none_or(|last: &DatabaseSummary| last.database_id != row.database_id) + { + summaries.push(row); + } + } + Ok(summaries) } fn map_database_meta_with_statuses( diff --git a/crates/vfs_runtime/src/sqlite.rs b/crates/vfs_runtime/src/sqlite.rs index bfa432f7..336443a0 100644 --- a/crates/vfs_runtime/src/sqlite.rs +++ b/crates/vfs_runtime/src/sqlite.rs @@ -222,6 +222,16 @@ pub(crate) fn text_value(value: impl Into) -> types::Value { types::Value::Text(value.into()) } +#[cfg(target_arch = "wasm32")] +pub(crate) fn nullable_text_value(value: Option) -> types::Value { + value.map(types::Value::Text).unwrap_or(types::Value::Null) +} + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn nullable_text_value(value: Option) -> types::Value { + value.map(types::Value::Text).unwrap_or(types::Value::Null) +} + #[cfg(target_arch = "wasm32")] pub(crate) fn integer_value(value: i64) -> types::Value { types::Value::Integer(value) diff --git a/crates/vfs_runtime/tests/database_service.rs b/crates/vfs_runtime/tests/database_service.rs index b9864cda..234a0314 100644 --- a/crates/vfs_runtime/tests/database_service.rs +++ b/crates/vfs_runtime/tests/database_service.rs @@ -8,16 +8,18 @@ use sha2::{Digest, Sha256}; use tempfile::tempdir; use vfs_runtime::{ CyclesPendingLedgerDetailsInput, DEFAULT_LLM_WRITER_PRINCIPAL, - DatabaseCyclesPurchaseWithLedgerDetails, MAX_ARCHIVE_CHUNK_BYTES, MAX_DATABASE_SIZE_BYTES, - MAX_RESTORE_CHUNK_BYTES, VfsService, cycles_for_payment_amount_e8s, + DatabaseCyclesPurchaseWithLedgerDetails, KinicDepositWithLedgerDetails, + MAX_ARCHIVE_CHUNK_BYTES, MAX_DATABASE_SIZE_BYTES, MAX_RESTORE_CHUNK_BYTES, VfsService, + cycles_for_payment_amount_e8s, }; use vfs_types::{ AppendNodeRequest, CyclesBillingConfigUpdate, DatabaseRole, DatabaseStatus, DeleteDatabaseRequest, DeleteNodeRequest, EditNodeRequest, KINIC_LEDGER_FEE_E8S, - MkdirNodeRequest, MoveNodeRequest, NodeKind, OpsAnswerSessionCheckRequest, - OpsAnswerSessionRequest, SearchNodesRequest, SearchPreviewMode, SourceRunSessionCheckRequest, - UrlIngestTriggerSessionCheckRequest, UrlIngestTriggerSessionRequest, WriteNodeRequest, - WriteSourceForGenerationRequest, + KinicFundDatabaseCyclesRequest, KinicPendingOperationsPageRequest, MarketCreateListingRequest, + MarketPurchaseRequest, MkdirNodeRequest, MoveNodeRequest, NodeKind, + OpsAnswerSessionCheckRequest, OpsAnswerSessionRequest, SearchNodesRequest, SearchPreviewMode, + SourceRunSessionCheckRequest, UrlIngestTriggerSessionCheckRequest, + UrlIngestTriggerSessionRequest, WriteNodeRequest, WriteSourceForGenerationRequest, }; fn service() -> VfsService { @@ -40,6 +42,128 @@ fn delete_request(database_id: &str) -> DeleteDatabaseRequest { } } +fn market_listing_request(database_id: &str, price_e8s: u64) -> MarketCreateListingRequest { + MarketCreateListingRequest { + database_id: database_id.to_string(), + title: "Team database".to_string(), + description: "Reusable team knowledge base".to_string(), + llm_summary: None, + summary_snapshot_revision: None, + sample_excerpts_json: "[]".to_string(), + tags_json: "[]".to_string(), + price_e8s, + } +} + +fn excerpt_json(path: &str, etag: &str, excerpt: &str) -> String { + format!("[{}]", excerpt_object(path, etag, excerpt)) +} + +fn excerpt_object(path: &str, etag: &str, excerpt: &str) -> String { + format!(r#"{{"path":"{path}","etag":"{etag}","excerpt":"{excerpt}"}}"#) +} + +fn credit_kinic_balance( + service: &VfsService, + caller: &str, + amount_e8s: u64, + block_index: u64, + now: i64, +) { + let start = service + .begin_kinic_deposit_with_ledger_details(KinicDepositWithLedgerDetails { + caller, + amount_e8s, + ledger: CyclesPendingLedgerDetailsInput { + from_owner: caller, + from_subaccount: None, + to_owner: "canister", + to_subaccount: None, + ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, + ledger_created_at_time_ns: u64::try_from(now).expect("now should fit u64"), + }, + now, + }) + .expect("deposit should begin"); + service + .complete_kinic_deposit_ledger_transfer(start.operation_id, caller, amount_e8s, block_index) + .expect("ledger transfer should complete"); + service + .apply_kinic_deposit(start.operation_id, caller, amount_e8s, now + 1) + .expect("deposit should credit balance"); +} + +fn kinic_fund_request( + database_id: &str, + payment_amount_e8s: u64, +) -> KinicFundDatabaseCyclesRequest { + KinicFundDatabaseCyclesRequest { + database_id: database_id.to_string(), + payment_amount_e8s, + min_expected_cycles: 0, + } +} + +fn database_status_text(root: &std::path::Path, database_id: &str) -> String { + let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); + conn.query_row( + "SELECT status FROM databases WHERE database_id = ?1", + params![database_id], + |row| row.get(0), + ) + .expect("database status should load") +} + +fn database_cycle_ledger_row( + root: &std::path::Path, + database_id: &str, +) -> (String, i64, Option, String, Option) { + let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); + conn.query_row( + "SELECT kind, amount_cycles, payment_amount_e8s, method, ledger_block_index + FROM database_cycle_ledger + WHERE database_id = ?1 + ORDER BY entry_id DESC + LIMIT 1", + params![database_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .expect("database cycle ledger row should load") +} + +fn kinic_ledger_row( + root: &std::path::Path, + principal: &str, +) -> (String, String, i64, i64, Option) { + let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); + conn.query_row( + "SELECT source, kind, amount_e8s, balance_after_e8s, counterparty + FROM kinic_ledger + WHERE principal = ?1 + ORDER BY entry_id DESC + LIMIT 1", + params![principal], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .expect("KINIC ledger row should load") +} + #[test] fn mainnet_011_index_upgrades_to_latest() { let dir = tempdir().expect("tempdir should create"); @@ -137,6 +261,15 @@ fn mainnet_011_index_upgrades_to_latest() { schema_migration_count(&root, "database_index:020_cycles_billing_config_version"), 1 ); + assert_eq!( + schema_migration_count(&root, "database_index:028_kinic_external_block_indexes"), + 1 + ); + assert!(index_exists(&root, "kinic_ledger_external_block_idx")); + assert!(index_exists( + &root, + "kinic_pending_operations_external_block_kind_idx" + )); assert_eq!(cycles_billing_config_key_count(&root, "config_version"), 0); } @@ -486,6 +619,20 @@ fn database_member_count(root: &std::path::Path, database_id: &str) -> i64 { .expect("member count should load") } +fn market_row_count(root: &std::path::Path, table: &str, database_id: &str) -> i64 { + assert!( + ["market_listings", "market_orders", "market_entitlements"].contains(&table), + "test helper must only read marketplace tables" + ); + let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); + conn.query_row( + &format!("SELECT COUNT(*) FROM {table} WHERE database_id = ?1"), + params![database_id], + |row| row.get(0), + ) + .expect("market row count should load") +} + fn database_cycles_balance(root: &std::path::Path, database_id: &str) -> i64 { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); conn.query_row( @@ -592,6 +739,32 @@ fn schema_migration_count(root: &std::path::Path, version: &str) -> i64 { .expect("migration count should load") } +fn index_exists(root: &std::path::Path, name: &str) -> bool { + let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); + conn.query_row( + "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?1", + params![name], + |row| row.get::<_, i64>(0), + ) + .optional() + .expect("index existence should load") + .is_some() +} + +#[test] +fn fresh_index_schema_contains_kinic_external_block_unique_indexes() { + let (_service, root) = service_with_root(); + assert!(index_exists(&root, "kinic_ledger_external_block_idx")); + assert!(index_exists( + &root, + "kinic_pending_operations_external_block_kind_idx" + )); + assert_eq!( + schema_migration_count(&root, "database_index:028_kinic_external_block_indexes"), + 1 + ); +} + fn mount_history_row(root: &std::path::Path, mount_id: u16) -> (String, String) { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); conn.query_row( @@ -4186,3 +4359,1034 @@ fn move_node_validates_source_target_path() { ) .expect("canonical source target should pass"); } + +#[test] +fn kinic_fund_database_cycles_moves_internal_balance_to_database_balance() { + let (service, root) = service_with_root(); + service + .create_database("fund-db", "owner", 1) + .expect("database should create"); + credit_kinic_balance(&service, "buyer", 1_000, 10, 2); + let config = service + .cycles_billing_config() + .expect("cycles billing config should load"); + let expected_cycles = + cycles_for_payment_amount_e8s(250, &config).expect("cycles quote should compute"); + + let result = service + .kinic_fund_database_cycles("buyer", kinic_fund_request("fund-db", 250), 3) + .expect("internal balance funding should succeed"); + + assert_eq!(result.payment_amount_e8s, 250); + assert_eq!(result.amount_cycles, expected_cycles); + assert_eq!(result.database_balance_cycles, expected_cycles); + assert_eq!(result.kinic_balance_e8s, 750); + assert_eq!( + service + .kinic_get_balance("buyer") + .expect("buyer balance should load") + .balance_e8s, + 750 + ); + assert_eq!( + database_cycles_balance(&root, "fund-db"), + expected_cycles as i64 + ); + assert_eq!( + kinic_ledger_row(&root, "buyer"), + ( + "canister_internal".to_string(), + "fund_database_cycles".to_string(), + -250, + 750, + Some("fund-db".to_string()) + ) + ); + assert_eq!( + database_cycle_ledger_row(&root, "fund-db"), + ( + "cycles_purchase".to_string(), + expected_cycles as i64, + Some(250), + "kinic_fund_database_cycles".to_string(), + None + ) + ); +} + +#[test] +fn kinic_deposit_external_block_indexes_reject_duplicate_pending_and_ledger_blocks() { + let service = service(); + let first = service + .begin_kinic_deposit_with_ledger_details(KinicDepositWithLedgerDetails { + caller: "alice", + amount_e8s: 100, + ledger: CyclesPendingLedgerDetailsInput { + from_owner: "alice", + from_subaccount: None, + to_owner: "canister", + to_subaccount: None, + ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, + ledger_created_at_time_ns: 1, + }, + now: 1, + }) + .expect("first deposit should begin"); + service + .complete_kinic_deposit_ledger_transfer(first.operation_id, "alice", 100, 77) + .expect("first pending transfer should complete"); + let second = service + .begin_kinic_deposit_with_ledger_details(KinicDepositWithLedgerDetails { + caller: "bob", + amount_e8s: 100, + ledger: CyclesPendingLedgerDetailsInput { + from_owner: "bob", + from_subaccount: None, + to_owner: "canister", + to_subaccount: None, + ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, + ledger_created_at_time_ns: 2, + }, + now: 2, + }) + .expect("second deposit should begin"); + let pending_duplicate = service + .complete_kinic_deposit_ledger_transfer(second.operation_id, "bob", 100, 77) + .expect_err("duplicate pending external block should reject"); + assert!(pending_duplicate.contains("UNIQUE constraint failed")); + + service + .apply_kinic_deposit(first.operation_id, "alice", 100, 3) + .expect("first deposit should apply"); + let third = service + .begin_kinic_deposit_with_ledger_details(KinicDepositWithLedgerDetails { + caller: "carol", + amount_e8s: 100, + ledger: CyclesPendingLedgerDetailsInput { + from_owner: "carol", + from_subaccount: None, + to_owner: "canister", + to_subaccount: None, + ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, + ledger_created_at_time_ns: 4, + }, + now: 4, + }) + .expect("third deposit should begin"); + service + .complete_kinic_deposit_ledger_transfer(third.operation_id, "carol", 100, 77) + .expect("ledger duplicate is checked when applying after pending is deleted"); + let ledger_duplicate = service + .apply_kinic_deposit(third.operation_id, "carol", 100, 5) + .expect_err("duplicate ledger external block should reject"); + assert!(ledger_duplicate.contains("UNIQUE constraint failed")); + assert_eq!( + service + .kinic_get_balance("carol") + .expect("failed duplicate should not credit") + .balance_e8s, + 0 + ); +} + +#[test] +fn kinic_list_pending_operations_paginates_and_clamps_limits() { + let service = service(); + for index in 0..3 { + let caller = format!("caller-{index}"); + service + .begin_kinic_deposit_with_ledger_details(KinicDepositWithLedgerDetails { + caller: &caller, + amount_e8s: 100 + index, + ledger: CyclesPendingLedgerDetailsInput { + from_owner: &caller, + from_subaccount: None, + to_owner: "canister", + to_subaccount: None, + ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, + ledger_created_at_time_ns: index, + }, + now: i64::try_from(index).expect("index should fit"), + }) + .expect("deposit should begin"); + } + + let first = service + .kinic_list_pending_operations( + "rrkah-fqaaa-aaaaa-aaaaq-cai", + KinicPendingOperationsPageRequest { + cursor_operation_id: None, + limit: 2, + }, + ) + .expect("first page should load"); + assert_eq!(first.operations.len(), 2); + assert_eq!(first.next_cursor_operation_id, Some(2)); + + let second = service + .kinic_list_pending_operations( + "rrkah-fqaaa-aaaaa-aaaaq-cai", + KinicPendingOperationsPageRequest { + cursor_operation_id: first.next_cursor_operation_id, + limit: 0, + }, + ) + .expect("zero limit should clamp to one"); + assert_eq!(second.operations.len(), 1); + assert_eq!(second.operations[0].operation_id, 3); + assert_eq!(second.next_cursor_operation_id, None); +} + +#[test] +fn kinic_fund_database_cycles_can_activate_pending_database_after_mount_allocation() { + let (service, root) = service_with_root(); + let pending = service + .reserve_pending_generated_database("Fund pending", "owner", 1) + .expect("pending database should reserve"); + service + .activate_pending_database_for_cycles_purchase(&pending.database_id, 2) + .expect("pending activation should start") + .expect("pending activation should allocate mount"); + credit_kinic_balance(&service, "owner", 1_000, 20, 3); + + let result = service + .kinic_fund_database_cycles("owner", kinic_fund_request(&pending.database_id, 400), 4) + .expect("pending database funding should succeed"); + + assert!(result.amount_cycles > 0); + assert_eq!(result.kinic_balance_e8s, 600); + assert_eq!(database_status_text(&root, &pending.database_id), "active"); +} + +#[test] +fn kinic_fund_database_cycles_rejects_invalid_payment_or_database_state() { + let (service, root) = service_with_root(); + service + .create_database("fund-active", "owner", 1) + .expect("active database should create"); + credit_kinic_balance(&service, "buyer", 100, 10, 2); + + let insufficient = service + .kinic_fund_database_cycles("buyer", kinic_fund_request("fund-active", 101), 3) + .expect_err("insufficient KINIC balance should reject"); + assert!(insufficient.contains("insufficient KINIC balance")); + + let config = service + .cycles_billing_config() + .expect("cycles billing config should load"); + let quoted = cycles_for_payment_amount_e8s(50, &config).expect("quote should compute"); + let min_mismatch = service + .kinic_fund_database_cycles( + "buyer", + KinicFundDatabaseCyclesRequest { + database_id: "fund-active".to_string(), + payment_amount_e8s: 50, + min_expected_cycles: quoted + 1, + }, + 4, + ) + .expect_err("quote mismatch should reject"); + assert!(min_mismatch.contains("cycles purchase quote changed")); + + let pending = service + .reserve_pending_generated_database("Unallocated", "owner", 5) + .expect("pending database should reserve"); + let pending_error = service + .kinic_fund_database_cycles("buyer", kinic_fund_request(&pending.database_id, 50), 6) + .expect_err("unallocated pending database should reject"); + assert!(pending_error.contains("pending database has no activation mount")); + assert_eq!(database_status_text(&root, &pending.database_id), "pending"); + assert_eq!(database_cycles_balance(&root, &pending.database_id), 0); + assert_eq!( + service + .kinic_get_balance("buyer") + .expect("buyer balance should load") + .balance_e8s, + 100 + ); + + service + .create_database("fund-archive", "owner", 7) + .expect("archive database should create"); + service + .begin_database_archive("fund-archive", "owner", 8) + .expect("archive should begin"); + let archived = service + .kinic_fund_database_cycles("buyer", kinic_fund_request("fund-archive", 50), 9) + .expect_err("archiving database should reject"); + assert!(archived.contains("database is archiving")); +} + +#[test] +fn market_purchase_moves_internal_balance_and_creates_entitlement() { + let service = service(); + service + .create_database("market-db", "seller", 1) + .expect("database should create"); + let listing = service + .market_create_listing("seller", market_listing_request("market-db", 250), 2) + .expect("listing should create"); + assert!(!listing.listing_id.starts_with("listing_")); + let listing = service + .market_publish_listing("seller", &listing.listing_id, 3) + .expect("listing should publish"); + credit_kinic_balance(&service, "buyer", 1_000, 10, 5); + + let order = service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id.clone(), + price_e8s: listing.price_e8s, + }, + 6, + ) + .expect("purchase should succeed"); + + assert_eq!(order.buyer_principal, "buyer"); + assert_eq!(order.seller_principal, "seller"); + assert_eq!( + service + .kinic_get_balance("buyer") + .expect("buyer balance should load") + .balance_e8s, + 750 + ); + assert_eq!( + service + .kinic_get_balance("seller") + .expect("seller balance should load") + .balance_e8s, + 250 + ); + assert_eq!( + service + .market_list_entitlements("buyer", None, 10) + .expect("entitlements should load") + .entitlements + .len(), + 1 + ); + let second_listing = service + .market_create_listing("seller", market_listing_request("market-db", 300), 7) + .expect("second listing should create"); + let second_listing = service + .market_publish_listing("seller", &second_listing.listing_id, 8) + .expect("second listing should publish"); + assert!( + service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: second_listing.listing_id, + price_e8s: second_listing.price_e8s, + }, + 9, + ) + .is_err(), + "same buyer cannot repurchase the same database" + ); +} + +#[test] +fn market_purchase_rejects_seller_self_purchase() { + let service = service(); + service + .create_database("self-market", "seller", 1) + .expect("database should create"); + let listing = service + .market_create_listing("seller", market_listing_request("self-market", 250), 2) + .expect("listing should create"); + let listing = service + .market_publish_listing("seller", &listing.listing_id, 3) + .expect("listing should publish"); + credit_kinic_balance(&service, "seller", 1_000, 10, 4); + + let error = service + .market_purchase_access( + "seller", + MarketPurchaseRequest { + listing_id: listing.listing_id, + price_e8s: listing.price_e8s, + }, + 5, + ) + .expect_err("seller must not buy their own listing"); + + assert!(error.contains("seller cannot purchase own listing")); + assert_eq!( + service + .kinic_get_balance("seller") + .expect("seller balance should load") + .balance_e8s, + 1_000 + ); +} + +#[test] +fn market_purchase_rejects_insufficient_balance_existing_entitlement_and_price_mismatch() { + let service = service(); + service + .create_database("reject-market", "seller", 1) + .expect("database should create"); + let listing = service + .market_create_listing("seller", market_listing_request("reject-market", 100), 2) + .expect("listing should create"); + let listing = service + .market_publish_listing("seller", &listing.listing_id, 3) + .expect("listing should publish"); + + let insufficient = service + .market_purchase_access( + "low-buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id.clone(), + price_e8s: listing.price_e8s, + }, + 4, + ) + .expect_err("insufficient balance should reject"); + assert!(insufficient.contains("insufficient KINIC balance")); + + credit_kinic_balance(&service, "buyer", 500, 10, 5); + let price_mismatch = service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id.clone(), + price_e8s: listing.price_e8s + 1, + }, + 6, + ) + .expect_err("price mismatch should reject"); + assert!(price_mismatch.contains("market listing price mismatch")); + assert_eq!( + service + .kinic_get_balance("buyer") + .expect("buyer balance should remain") + .balance_e8s, + 500 + ); + + service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id.clone(), + price_e8s: listing.price_e8s, + }, + 7, + ) + .expect("first purchase should succeed"); + let duplicate = service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id, + price_e8s: listing.price_e8s, + }, + 8, + ) + .expect_err("existing entitlement should reject"); + assert!(duplicate.contains("active entitlement already exists")); +} + +#[test] +fn market_listing_owner_policy_and_database_listing_query() { + let service = service(); + service + .create_database("owner-market", "seller", 1) + .expect("database should create"); + service + .grant_database_access("owner-market", "seller", "reader", DatabaseRole::Reader, 2) + .expect("reader should be granted"); + assert!( + service + .market_create_listing("reader", market_listing_request("owner-market", 100), 3) + .is_err(), + "non-owner must not create listing" + ); + + let listing = service + .market_create_listing("seller", market_listing_request("owner-market", 100), 4) + .expect("owner should create listing"); + assert!( + service + .market_list_database_listings("reader", "owner-market") + .is_err(), + "non-owner must not list database listings" + ); + let listings = service + .market_list_database_listings("seller", "owner-market") + .expect("owner should list database listings"); + assert_eq!(listings.len(), 1); + assert_eq!(listings[0].listing_id, listing.listing_id); +} + +#[test] +fn market_listing_detail_returns_verified_preview() { + let service = service(); + service + .create_database("preview-market", "seller", 1) + .expect("database should create"); + ensure_parent_folders(&service, "seller", "preview-market", "/Wiki/alpha/a.md", 2); + ensure_parent_folders(&service, "seller", "preview-market", "/Wiki/beta/b.md", 2); + ensure_parent_folders( + &service, + "seller", + "preview-market", + "/Sources/raw/web/source.md", + 2, + ); + let alpha = service + .write_node( + "seller", + WriteNodeRequest { + database_id: "preview-market".to_string(), + path: "/Wiki/alpha/a.md".to_string(), + kind: NodeKind::File, + content: "Alpha paid insight links to [beta](/Wiki/beta/b.md).".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 2, + ) + .expect("alpha node should write"); + service + .write_node( + "seller", + WriteNodeRequest { + database_id: "preview-market".to_string(), + path: "/Wiki/beta/b.md".to_string(), + kind: NodeKind::File, + content: "Beta paid body".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 3, + ) + .expect("beta node should write"); + service + .write_node( + "seller", + WriteNodeRequest { + database_id: "preview-market".to_string(), + path: "/Sources/raw/web/source.md".to_string(), + kind: NodeKind::Source, + content: "raw source body".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 4, + ) + .expect("source node should write"); + + let mut request = market_listing_request("preview-market", 100); + request.sample_excerpts_json = + excerpt_json("/Wiki/alpha/a.md", &alpha.node.etag, "paid insight links"); + let listing = service + .market_create_listing("seller", request, 5) + .expect("listing should create"); + let listing = service + .market_publish_listing("seller", &listing.listing_id, 6) + .expect("listing should publish"); + let detail = service + .market_get_listing("2vxsx-fae", &listing.listing_id) + .expect("anonymous should read public listing preview"); + + assert_eq!(detail.listing.listing_id, listing.listing_id); + assert!(detail.verified_stats.total_nodes >= 5); + assert!(detail.verified_stats.wiki_nodes >= 4); + assert_eq!(detail.verified_stats.source_nodes, 1); + assert!(detail.verified_stats.folder_nodes >= 3); + assert!(detail.verified_stats.markdown_chars >= 64); + assert_eq!(detail.verified_stats.source_chars, 15); + assert_eq!(detail.verified_stats.link_edges, 1); + assert_eq!(detail.preview.excerpts.len(), 1); + assert_eq!(detail.preview.excerpts[0].excerpt, "paid insight links"); + assert!(!detail.preview.preview_stale); + assert!( + detail + .preview + .top_level_paths + .contains(&"/Wiki/alpha".to_string()) + ); + assert!( + detail + .preview + .top_level_paths + .contains(&"/Wiki/beta".to_string()) + ); + assert!( + detail + .preview + .category_graph + .nodes + .iter() + .any(|node| node.category == "/Wiki/alpha") + ); + assert!( + detail + .preview + .category_graph + .edges + .iter() + .any(|edge| edge.source_category == "/Wiki/alpha" + && edge.target_category == "/Wiki/beta" + && edge.link_count == 1) + ); + + service + .write_node( + "seller", + WriteNodeRequest { + database_id: "preview-market".to_string(), + path: "/Wiki/alpha/a.md".to_string(), + kind: NodeKind::File, + content: "Alpha body changed".to_string(), + metadata_json: "{}".to_string(), + expected_etag: Some(alpha.node.etag), + }, + 7, + ) + .expect("alpha node should update"); + let stale_detail = service + .market_get_listing("2vxsx-fae", &listing.listing_id) + .expect("anonymous should still read stale public listing preview"); + assert!(stale_detail.preview.excerpts.is_empty()); + assert!(stale_detail.preview.preview_stale); +} + +#[test] +fn market_listing_sample_excerpts_must_be_verified_wiki_substrings() { + let service = service(); + service + .create_database("verified-excerpts", "seller", 1) + .expect("database should create"); + ensure_parent_folders( + &service, + "seller", + "verified-excerpts", + "/Wiki/alpha/a.md", + 2, + ); + ensure_parent_folders( + &service, + "seller", + "verified-excerpts", + "/Sources/raw/web/source.md", + 2, + ); + let node = service + .write_node( + "seller", + WriteNodeRequest { + database_id: "verified-excerpts".to_string(), + path: "/Wiki/alpha/a.md".to_string(), + kind: NodeKind::File, + content: "Verified paid excerpt body".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 2, + ) + .expect("wiki node should write"); + service + .write_node( + "seller", + WriteNodeRequest { + database_id: "verified-excerpts".to_string(), + path: "/Sources/raw/web/source.md".to_string(), + kind: NodeKind::Source, + content: "Verified paid excerpt body".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 3, + ) + .expect("source node should write"); + + for sample_excerpts_json in [ + excerpt_json("/Wiki/alpha/a.md", "wrong-etag", "paid excerpt"), + excerpt_json("/Wiki/alpha/a.md", &node.node.etag, "missing text"), + excerpt_json( + "/Sources/raw/web/source.md", + &node.node.etag, + "paid excerpt", + ), + format!( + "[{},{},{},{},{},{}]", + excerpt_object("/Wiki/alpha/a.md", &node.node.etag, "paid excerpt"), + excerpt_object("/Wiki/alpha/a.md", &node.node.etag, "paid excerpt"), + excerpt_object("/Wiki/alpha/a.md", &node.node.etag, "paid excerpt"), + excerpt_object("/Wiki/alpha/a.md", &node.node.etag, "paid excerpt"), + excerpt_object("/Wiki/alpha/a.md", &node.node.etag, "paid excerpt"), + excerpt_object("/Wiki/alpha/a.md", &node.node.etag, "paid excerpt") + ), + excerpt_json("/Wiki/alpha/a.md", &node.node.etag, &"x".repeat(401)), + ] { + let mut request = market_listing_request("verified-excerpts", 100); + request.sample_excerpts_json = sample_excerpts_json; + assert!( + service.market_create_listing("seller", request, 4).is_err(), + "invalid sample excerpt must reject" + ); + } +} + +#[test] +fn market_listing_leaves_public_surface_when_seller_loses_owner_role() { + let service = service(); + service + .create_database("stale-owner-market", "seller", 1) + .expect("database should create"); + let listing = service + .market_create_listing( + "seller", + market_listing_request("stale-owner-market", 100), + 2, + ) + .expect("listing should create"); + let listing = service + .market_publish_listing("seller", &listing.listing_id, 3) + .expect("listing should publish"); + service + .grant_database_access( + "stale-owner-market", + "seller", + "successor", + DatabaseRole::Owner, + 4, + ) + .expect("successor owner should be granted"); + service + .revoke_database_access("stale-owner-market", "successor", "seller") + .expect("seller should lose owner access"); + credit_kinic_balance(&service, "buyer", 100, 30, 5); + + assert!( + service + .market_list_listings(None, 10) + .expect("public listings should load") + .listings + .is_empty(), + "seller without owner role must not stay in public marketplace" + ); + assert!( + service + .market_get_listing("buyer", &listing.listing_id) + .expect_err("buyer should not read stale public listing") + .contains("seller or admin required") + ); + assert!( + service + .market_preview_purchase("buyer", &listing.listing_id) + .expect_err("preview should reject stale seller") + .contains("principal has no access to database") + ); + assert!( + service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id.clone(), + price_e8s: listing.price_e8s, + }, + 6, + ) + .expect_err("purchase should reject stale seller") + .contains("principal has no access to database") + ); + assert_eq!( + service + .kinic_get_balance("buyer") + .expect("buyer balance should remain") + .balance_e8s, + 100 + ); + assert_eq!( + service + .kinic_get_balance("seller") + .expect("seller balance should remain") + .balance_e8s, + 0 + ); + + service + .market_get_listing("seller", &listing.listing_id) + .expect("original seller should still manage stale listing"); + let paused = service + .market_pause_listing("seller", &listing.listing_id, 7) + .expect("original seller should pause stale listing"); + assert_eq!(paused.listing_id, listing.listing_id); +} + +#[test] +fn market_listing_requires_active_database() { + let service = service(); + let pending = service + .reserve_pending_generated_database("Pending market", "owner", 1) + .expect("pending database should reserve"); + let pending_error = service + .market_create_listing( + "owner", + market_listing_request(&pending.database_id, 100), + 2, + ) + .expect_err("pending database listing should reject"); + assert!(pending_error.contains("database is pending")); + + service + .create_database("archive-market", "owner", 3) + .expect("database should create"); + let listing = service + .market_create_listing("owner", market_listing_request("archive-market", 100), 4) + .expect("active database listing should create"); + let listing = service + .market_publish_listing("owner", &listing.listing_id, 5) + .expect("active database listing should publish"); + assert_eq!( + service + .market_list_listings(None, 10) + .expect("active listings should load") + .listings + .len(), + 1 + ); + + let archive = service + .begin_database_archive("archive-market", "owner", 6) + .expect("archive should begin"); + assert!( + service + .market_list_listings(None, 10) + .expect("archiving listings should load") + .listings + .is_empty(), + "archiving database listing must leave public marketplace" + ); + assert!( + service + .market_get_listing("buyer", &listing.listing_id) + .expect_err("buyer should not read non-active database listing") + .contains("seller or admin required") + ); + service + .market_get_listing("owner", &listing.listing_id) + .expect("owner should still manage archived listing"); + + let bytes = read_archive_in_chunks(&service, "archive-market", archive.size_bytes, 17); + let snapshot_hash = sha256_bytes(&bytes); + service + .finalize_database_archive("archive-market", "owner", snapshot_hash.clone(), 7) + .expect("archive should finalize"); + credit_kinic_balance(&service, "buyer", 100, 21, 8); + let preview_error = service + .market_preview_purchase("buyer", &listing.listing_id) + .expect_err("archived database preview should reject"); + assert!(preview_error.contains("database is archived")); + let purchase_error = service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id.clone(), + price_e8s: listing.price_e8s, + }, + 9, + ) + .expect_err("archived database purchase should reject"); + assert!(purchase_error.contains("database is archived")); + assert_eq!( + service + .kinic_get_balance("buyer") + .expect("buyer balance should remain") + .balance_e8s, + 100 + ); + assert_eq!( + service + .kinic_get_balance("owner") + .expect("seller balance should remain") + .balance_e8s, + 0 + ); + + service + .begin_database_restore( + "archive-market", + "owner", + snapshot_hash, + archive.size_bytes, + 10, + ) + .expect("restore should begin"); + assert!( + service + .market_list_listings(None, 10) + .expect("restoring listings should load") + .listings + .is_empty(), + "restoring database listing must stay hidden" + ); + service + .write_database_restore_chunk("archive-market", "owner", 0, &bytes) + .expect("restore chunk should write"); + service + .finalize_database_restore("archive-market", "owner", 11) + .expect("restore should finalize"); + assert_eq!( + service + .market_list_listings(None, 10) + .expect("restored listings should load") + .listings + .len(), + 1 + ); + service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id, + price_e8s: listing.price_e8s, + }, + 12, + ) + .expect("restored active database purchase should succeed"); +} + +#[test] +fn delete_database_removes_marketplace_rows() { + let (service, root) = service_with_root(); + service + .create_database("market-delete", "seller", 1) + .expect("database should create"); + service + .write_node( + "seller", + WriteNodeRequest { + database_id: "market-delete".to_string(), + path: "/Wiki/private.md".to_string(), + kind: NodeKind::File, + content: "paid body".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 2, + ) + .expect("owner should write"); + let listing = service + .market_create_listing("seller", market_listing_request("market-delete", 100), 3) + .expect("listing should create"); + let listing = service + .market_publish_listing("seller", &listing.listing_id, 4) + .expect("listing should publish"); + credit_kinic_balance(&service, "buyer", 100, 10, 5); + service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id, + price_e8s: listing.price_e8s, + }, + 6, + ) + .expect("purchase should succeed"); + service + .read_node("market-delete", "buyer", "/Wiki/private.md") + .expect("entitled buyer should read before delete"); + assert_eq!( + market_row_count(&root, "market_listings", "market-delete"), + 1 + ); + assert_eq!(market_row_count(&root, "market_orders", "market-delete"), 1); + assert_eq!( + market_row_count(&root, "market_entitlements", "market-delete"), + 1 + ); + + service + .delete_database(delete_request("market-delete"), "seller", 7) + .expect("delete should succeed"); + assert_eq!( + market_row_count(&root, "market_listings", "market-delete"), + 0 + ); + assert_eq!(market_row_count(&root, "market_orders", "market-delete"), 1); + assert_eq!( + market_row_count(&root, "market_entitlements", "market-delete"), + 0 + ); + assert_eq!( + service + .market_list_orders("buyer", None, 10) + .expect("buyer order history should remain") + .orders + .len(), + 1 + ); + let deleted_read = service + .read_node("market-delete", "buyer", "/Wiki/private.md") + .expect_err("deleted database should not remain readable"); + assert!(deleted_read.contains("database not found")); +} + +#[test] +fn market_entitlement_allows_read_surface_but_not_export() { + let service = service(); + service + .create_database("read-market", "seller", 1) + .expect("database should create"); + service + .write_node( + "seller", + WriteNodeRequest { + database_id: "read-market".to_string(), + path: "/Wiki/private.md".to_string(), + kind: NodeKind::File, + content: "paid body".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 2, + ) + .expect("owner should write"); + let listing = service + .market_create_listing("seller", market_listing_request("read-market", 100), 3) + .expect("listing should create"); + let listing = service + .market_publish_listing("seller", &listing.listing_id, 4) + .expect("listing should publish"); + credit_kinic_balance(&service, "buyer", 100, 20, 6); + service + .market_purchase_access( + "buyer", + MarketPurchaseRequest { + listing_id: listing.listing_id, + price_e8s: listing.price_e8s, + }, + 7, + ) + .expect("purchase should succeed"); + + let node = service + .read_node("read-market", "buyer", "/Wiki/private.md") + .expect("entitled buyer should read") + .expect("node should exist"); + assert_eq!(node.content, "paid body"); + assert!( + service + .export_fs_snapshot( + "buyer", + vfs_types::ExportSnapshotRequest { + database_id: "read-market".to_string(), + prefix: None, + limit: 10, + cursor: None, + snapshot_session_id: None, + snapshot_revision: None, + }, + ) + .is_err(), + "entitlement must not allow export" + ); +} diff --git a/crates/vfs_store/src/fs_store.rs b/crates/vfs_store/src/fs_store.rs index 37b2ce17..754ed076 100644 --- a/crates/vfs_store/src/fs_store.rs +++ b/crates/vfs_store/src/fs_store.rs @@ -19,12 +19,13 @@ use vfs_types::{ EditNodeResult, ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, GlobNodeHit, GlobNodeType, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, IncomingLinksRequest, LinkEdge, ListChildrenRequest, - ListNodesRequest, MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, MoveNodeResult, - MultiEdit, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeContext, NodeContextRequest, - NodeEntry, NodeEntryKind, NodeKind, OutgoingLinksRequest, QueryContext, QueryContextRequest, - SearchNodeHit, SearchNodePathsRequest, SearchNodesRequest, SearchPreviewMode, SourceEvidence, - SourceEvidenceRef, SourceEvidenceRequest, Status, WriteNodeItem, WriteNodeRequest, - WriteNodeResult, WriteNodesRequest, + ListNodesRequest, MarketCategoryGraph, MarketCategoryGraphEdge, MarketCategoryGraphNode, + MarketListingPreview, MarketListingVerifiedStats, MkdirNodeRequest, MkdirNodeResult, + MoveNodeRequest, MoveNodeResult, MultiEdit, MultiEditNodeRequest, MultiEditNodeResult, Node, + NodeContext, NodeContextRequest, NodeEntry, NodeEntryKind, NodeKind, OutgoingLinksRequest, + QueryContext, QueryContextRequest, SearchNodeHit, SearchNodePathsRequest, SearchNodesRequest, + SearchPreviewMode, SourceEvidence, SourceEvidenceRef, SourceEvidenceRequest, Status, + WriteNodeItem, WriteNodeRequest, WriteNodeResult, WriteNodesRequest, }; use crate::{ @@ -168,6 +169,22 @@ impl FsStore { }) } + pub fn marketplace_preview( + &self, + ) -> Result<(MarketListingVerifiedStats, MarketListingPreview), String> { + self.read_conn(|conn| { + let mut stats = load_marketplace_verified_stats(conn)?; + stats.logical_size_bytes = logical_size_bytes_for_conn(conn)?; + let preview = MarketListingPreview { + top_level_paths: load_marketplace_top_level_paths(conn)?, + excerpts: Vec::new(), + category_graph: load_marketplace_category_graph(conn)?, + preview_stale: false, + }; + Ok((stats, preview)) + }) + } + pub fn logical_size_bytes(&self) -> Result { #[cfg(not(target_arch = "wasm32"))] { @@ -1437,6 +1454,181 @@ fn count_nodes(conn: &Connection, kind: &str) -> Result { u64::try_from(count).map_err(|error| error.to_string()) } +fn load_marketplace_verified_stats( + conn: &Connection, +) -> Result { + let ( + total_nodes, + wiki_nodes, + source_nodes, + folder_nodes, + markdown_chars, + source_chars, + last_content_updated_at_ms, + ) = conn + .query_row( + "SELECT COUNT(*), + SUM(CASE WHEN path = '/Wiki' OR path LIKE '/Wiki/%' THEN 1 ELSE 0 END), + SUM(CASE WHEN kind = 'source' THEN 1 ELSE 0 END), + SUM(CASE WHEN kind = 'folder' THEN 1 ELSE 0 END), + SUM(CASE WHEN kind = 'file' THEN length(content) ELSE 0 END), + SUM(CASE WHEN kind = 'source' THEN length(content) ELSE 0 END), + MAX(CASE WHEN kind IN ('file', 'source') THEN updated_at ELSE NULL END) + FROM fs_nodes", + params![], + |row| { + Ok(( + crate::sqlite::row_get::(row, 0)?, + crate::sqlite::row_get::>(row, 1)?, + crate::sqlite::row_get::>(row, 2)?, + crate::sqlite::row_get::>(row, 3)?, + crate::sqlite::row_get::>(row, 4)?, + crate::sqlite::row_get::>(row, 5)?, + crate::sqlite::row_get::>(row, 6)?, + )) + }, + ) + .map_err(|error| error.to_string())?; + let link_edges = conn + .query_row("SELECT COUNT(*) FROM fs_links", params![], |row| { + crate::sqlite::row_get::(row, 0) + }) + .map_err(|error| error.to_string())?; + Ok(MarketListingVerifiedStats { + total_nodes: nonnegative_i64_to_u64(total_nodes)?, + wiki_nodes: nonnegative_i64_to_u64(wiki_nodes.unwrap_or(0))?, + source_nodes: nonnegative_i64_to_u64(source_nodes.unwrap_or(0))?, + folder_nodes: nonnegative_i64_to_u64(folder_nodes.unwrap_or(0))?, + markdown_chars: nonnegative_i64_to_u64(markdown_chars.unwrap_or(0))?, + source_chars: nonnegative_i64_to_u64(source_chars.unwrap_or(0))?, + link_edges: nonnegative_i64_to_u64(link_edges)?, + logical_size_bytes: 0, + last_content_updated_at_ms, + }) +} + +fn load_marketplace_top_level_paths(conn: &Connection) -> Result, String> { + let mut stmt = conn + .prepare( + "SELECT child.path + FROM fs_nodes child + JOIN fs_nodes parent ON parent.id = child.parent_id + WHERE parent.path = '/Wiki' + ORDER BY CASE child.kind WHEN 'folder' THEN 0 WHEN 'file' THEN 1 ELSE 2 END, + child.path ASC + LIMIT 12", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map(&mut stmt, params![], |row| crate::sqlite::row_get(row, 0)) + .map_err(|error| error.to_string()) +} + +fn load_marketplace_category_graph(conn: &Connection) -> Result { + let mut stmt = conn + .prepare( + "SELECT path + FROM fs_nodes + WHERE path = '/Wiki' OR path LIKE '/Wiki/%' + ORDER BY path ASC", + ) + .map_err(|error| error.to_string())?; + let paths = crate::sqlite::query_map(&mut stmt, params![], |row| { + crate::sqlite::row_get::(row, 0) + }) + .map_err(|error| error.to_string())?; + let mut counts = BTreeMap::::new(); + for path in paths { + if let Some(category) = marketplace_top_category(&path) { + *counts.entry(category).or_insert(0) += 1; + } + } + let mut nodes = counts + .into_iter() + .map(|(category, node_count)| MarketCategoryGraphNode { + category, + node_count, + }) + .collect::>(); + nodes.sort_by(|left, right| { + right + .node_count + .cmp(&left.node_count) + .then_with(|| left.category.cmp(&right.category)) + }); + nodes.truncate(12); + let selected = nodes + .iter() + .map(|node| node.category.clone()) + .collect::>(); + + let mut stmt = conn + .prepare( + "SELECT source_path, target_path + FROM fs_links + WHERE (source_path = '/Wiki' OR source_path LIKE '/Wiki/%') + AND (target_path = '/Wiki' OR target_path LIKE '/Wiki/%')", + ) + .map_err(|error| error.to_string())?; + let edges = crate::sqlite::query_map(&mut stmt, params![], |row| { + Ok(( + crate::sqlite::row_get::(row, 0)?, + crate::sqlite::row_get::(row, 1)?, + )) + }) + .map_err(|error| error.to_string())?; + let mut edge_counts = BTreeMap::<(String, String), u64>::new(); + for (source_path, target_path) in edges { + let Some(source_category) = marketplace_top_category(&source_path) else { + continue; + }; + let Some(target_category) = marketplace_top_category(&target_path) else { + continue; + }; + if source_category == target_category + || !selected.contains(&source_category) + || !selected.contains(&target_category) + { + continue; + } + *edge_counts + .entry((source_category, target_category)) + .or_insert(0) += 1; + } + let mut edges = edge_counts + .into_iter() + .map( + |((source_category, target_category), link_count)| MarketCategoryGraphEdge { + source_category, + target_category, + link_count, + }, + ) + .collect::>(); + edges.sort_by(|left, right| { + right + .link_count + .cmp(&left.link_count) + .then_with(|| left.source_category.cmp(&right.source_category)) + .then_with(|| left.target_category.cmp(&right.target_category)) + }); + edges.truncate(30); + Ok(MarketCategoryGraph { nodes, edges }) +} + +fn marketplace_top_category(path: &str) -> Option { + let rest = path.strip_prefix("/Wiki/")?; + let segment = rest.split('/').next()?.trim(); + if segment.is_empty() { + None + } else { + Some(format!("/Wiki/{segment}")) + } +} + +fn nonnegative_i64_to_u64(value: i64) -> Result { + u64::try_from(value.max(0)).map_err(|error| error.to_string()) +} + fn logical_size_bytes_for_conn(conn: &Connection) -> Result { let page_count = conn .query_row("PRAGMA page_count", params![], |row| { diff --git a/crates/vfs_types/src/fs.rs b/crates/vfs_types/src/fs.rs index c0dc0d3d..14b7dde1 100644 --- a/crates/vfs_types/src/fs.rs +++ b/crates/vfs_types/src/fs.rs @@ -127,6 +127,228 @@ pub struct DatabaseCyclesPendingPurchase { pub required_action: String, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicBalance { + pub balance_e8s: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicPendingOperation { + pub operation_id: u64, + pub kind: String, + pub caller: String, + pub status: String, + pub amount_e8s: u64, + pub ledger_block_index: Option, + pub created_at_ms: i64, + pub required_action: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicPendingOperationsPageRequest { + pub cursor_operation_id: Option, + pub limit: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicPendingOperationsPage { + pub operations: Vec, + pub next_cursor_operation_id: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicDepositRequest { + pub amount_e8s: u64, + pub expected_fee_e8s: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicDepositResult { + pub block_index: u64, + pub amount_e8s: u64, + pub balance_e8s: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicFundDatabaseCyclesRequest { + pub database_id: String, + pub payment_amount_e8s: u64, + pub min_expected_cycles: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct KinicFundDatabaseCyclesResult { + pub payment_amount_e8s: u64, + pub amount_cycles: u64, + pub database_balance_cycles: u64, + pub kinic_balance_e8s: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +#[serde(rename_all = "snake_case")] +pub enum MarketListingStatus { + #[serde(alias = "Draft")] + Draft, + #[serde(alias = "Active")] + Active, + #[serde(alias = "Paused")] + Paused, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketListing { + pub listing_id: String, + pub seller_principal: String, + pub database_id: String, + pub title: String, + pub description: String, + pub llm_summary: Option, + pub summary_snapshot_revision: Option, + pub sample_excerpts_json: String, + pub tags_json: String, + pub price_e8s: u64, + pub status: MarketListingStatus, + pub revision: u64, + pub purchase_count: u64, + pub report_count: u64, + pub created_at_ms: i64, + pub updated_at_ms: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketListingVerifiedStats { + pub total_nodes: u64, + pub wiki_nodes: u64, + pub source_nodes: u64, + pub folder_nodes: u64, + pub markdown_chars: u64, + pub source_chars: u64, + pub link_edges: u64, + pub logical_size_bytes: u64, + pub last_content_updated_at_ms: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketPreviewExcerpt { + pub path: String, + pub etag: String, + pub excerpt: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketCategoryGraphNode { + pub category: String, + pub node_count: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketCategoryGraphEdge { + pub source_category: String, + pub target_category: String, + pub link_count: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketCategoryGraph { + pub nodes: Vec, + pub edges: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketListingPreview { + pub top_level_paths: Vec, + pub excerpts: Vec, + pub category_graph: MarketCategoryGraph, + pub preview_stale: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketListingDetail { + pub listing: MarketListing, + pub verified_stats: MarketListingVerifiedStats, + pub preview: MarketListingPreview, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketListingPage { + pub listings: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketCreateListingRequest { + pub database_id: String, + pub title: String, + pub description: String, + pub llm_summary: Option, + pub summary_snapshot_revision: Option, + pub sample_excerpts_json: String, + pub tags_json: String, + pub price_e8s: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketUpdateListingRequest { + pub listing_id: String, + pub expected_revision: u64, + pub title: String, + pub description: String, + pub llm_summary: Option, + pub summary_snapshot_revision: Option, + pub sample_excerpts_json: String, + pub tags_json: String, + pub price_e8s: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketPurchaseRequest { + pub listing_id: String, + pub price_e8s: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketPurchasePreview { + pub listing_id: String, + pub database_id: String, + pub price_e8s: u64, + pub buyer_balance_e8s: u64, + pub already_entitled: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketOrder { + pub order_id: String, + pub listing_id: String, + pub database_id: String, + pub buyer_principal: String, + pub seller_principal: String, + pub price_e8s: u64, + pub listing_revision: u64, + pub created_at_ms: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketOrderPage { + pub orders: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketEntitlement { + pub database_id: String, + pub buyer_principal: String, + pub listing_id: String, + pub order_id: String, + pub purchased_at_ms: i64, + pub status: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] +pub struct MarketEntitlementPage { + pub entitlements: Vec, + pub next_cursor: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] pub struct StorageBillingBatchRequest { pub cursor_mount_id: Option, diff --git a/extensions/wiki-clipper/scripts/check-candid-drift.mjs b/extensions/wiki-clipper/scripts/check-candid-drift.mjs index 574e7dfd..c8ea1f71 100644 --- a/extensions/wiki-clipper/scripts/check-candid-drift.mjs +++ b/extensions/wiki-clipper/scripts/check-candid-drift.mjs @@ -190,10 +190,10 @@ function normalizeDidResult(value) { if (normalized === "Result_1") return "ResultUnit"; if (normalized === "Result_9") return "ResultCyclesBillingConfig"; if (normalized === "Result_4") return "ResultCreateDatabase"; - if (normalized === "Result_16") return "ResultDatabases"; - if (normalized === "Result_18") return "ResultMkdirNode"; - if (normalized === "Result_24") return "ResultNode"; - if (normalized === "Result_30") return "ResultWriteSourceForGeneration"; + if (normalized === "Result_20") return "ResultDatabases"; + if (normalized === "Result_31") return "ResultMkdirNode"; + if (normalized === "Result_37") return "ResultNode"; + if (normalized === "Result_43") return "ResultWriteSourceForGeneration"; if (normalized === "Result") return "ResultWriteNode"; return normalized; } diff --git a/extensions/wiki-clipper/tests/settings.test.mjs b/extensions/wiki-clipper/tests/settings.test.mjs index 7169479d..3b782e2e 100644 --- a/extensions/wiki-clipper/tests/settings.test.mjs +++ b/extensions/wiki-clipper/tests/settings.test.mjs @@ -18,7 +18,8 @@ import { AUTH_SESSION_TTL_NS, MAINNET_II_PROVIDER_URL, WIKI_CANISTER_DERIVATION_ORIGIN, - derivationOriginForLocation + derivationOriginForLocation, + identityProviderUrlForLocation } from "../../../shared/ii-auth/index.js"; test("settings popup omits fixed runtime inputs", () => { @@ -210,7 +211,14 @@ test("Internet Identity options use 29 day TTL and derivation origin", () => { assert.equal(AUTH_OPTIONS.createOptions.idleOptions.disableDefaultIdleCallback, true); }); -test("CLI login derivation origin uses local origin only for local development", () => { +test("CLI login helpers use mainnet Internet Identity and canonical derivation origin", () => { + assert.equal( + identityProviderUrlForLocation({ + hostname: "localhost", + origin: "http://localhost:4943" + }), + MAINNET_II_PROVIDER_URL + ); assert.equal( derivationOriginForLocation({ hostname: "wiki.kinic.xyz", @@ -230,7 +238,7 @@ test("CLI login derivation origin uses local origin only for local development", hostname: "localhost", origin: "http://localhost:4943" }), - "http://localhost:4943" + WIKI_CANISTER_DERIVATION_ORIGIN ); }); diff --git a/scripts/build-vfs-canister.sh b/scripts/build-vfs-canister.sh index 7a41f899..22014e7e 100755 --- a/scripts/build-vfs-canister.sh +++ b/scripts/build-vfs-canister.sh @@ -16,6 +16,27 @@ OUTPUT_WASM="${INPUT_WASM}" # `icp deploy` sets this; standalone runs default to the repo artifact path. ICP_WASM_OUTPUT_PATH="${ICP_WASM_OUTPUT_PATH:-${OUTPUT_WASM}}" +guard_local_ii_origins() { + if [[ "${KINIC_VFS_LOCAL_II_ORIGINS:-}" != "1" ]]; then + return + fi + case "${ICP_ENVIRONMENT:-}" in + local | local-wiki) + return + ;; + *) + echo "KINIC_VFS_LOCAL_II_ORIGINS=1 is only allowed for ICP_ENVIRONMENT=local or local-wiki" >&2 + exit 1 + ;; + esac +} + +guard_local_ii_origins + +if [[ "${1:-}" == "--check-env-only" ]]; then + exit 0 +fi + EXTRA_FEATURES="" case "${VFS_CANISTER_DIAGNOSTIC_PROFILE:-baseline}" in baseline) diff --git a/scripts/candid-subset-check.mjs b/scripts/candid-subset-check.mjs new file mode 100644 index 00000000..d9fefc88 --- /dev/null +++ b/scripts/candid-subset-check.mjs @@ -0,0 +1,233 @@ +// Where: scripts/candid-subset-check.mjs +// What: Shared structural checks for hand-written Candid IDL subsets. +// Why: Browser-side hand-written IDL should fail CI when crates/vfs_canister/vfs.did drifts. +export function checkCandidSubset({ didSource, idlSource, expectedTypes, expectedMethods, didTypeAliases = {} }) { + const failures = []; + const didTypes = parseDidTypes(didSource); + const didMethods = parseDidMethods(didSource, didTypeAliases); + const idlTypes = parseIdlTypes(idlSource); + const idlMethods = parseIdlMethods(idlSource); + + for (const [name, shape] of Object.entries(expectedTypes)) { + compareShape(failures, `vfs.did type ${name}`, didTypes[didTypeAliases[name] ?? name], shape); + compareShape(failures, `hand-written IDL type ${name}`, idlTypes[name], shape); + } + + for (const [name, shape] of Object.entries(expectedMethods)) { + compareMethod(failures, `vfs.did method ${name}`, didMethods[name], shape, didTypeAliases); + compareMethod(failures, `hand-written IDL method ${name}`, idlMethods[name], shape, didTypeAliases); + } + + return failures; +} + +function parseDidTypes(source) { + const types = {}; + for (const match of source.matchAll(/^type\s+(\w+)\s*=\s*(record|variant)\s*\{([^]*?)\};/gm)) { + const [, name, kind, body] = match; + types[name] = kind === "record" ? { kind, fields: parseDidFields(body) } : { kind, cases: parseDidFields(body) }; + } + return types; +} + +function parseDidFields(body) { + const fields = {}; + for (const raw of body.split(";")) { + const line = raw.trim(); + if (!line) continue; + const match = line.match(/^"?(\w+)"?\s*(?::\s*(.+))?$/); + if (!match) continue; + fields[match[1]] = normalizeShape(match[2] ?? "null"); + } + return fields; +} + +function parseDidMethods(source, didTypeAliases) { + const service = source.match(/service\s*:\s*\([^)]*\)\s*->\s*\{([^]*?)\n\}/m)?.[1] ?? ""; + const methods = {}; + for (const raw of service.split(";")) { + const line = raw.trim(); + if (!line) continue; + const match = line.match(/^(\w+)\s*:\s*\(([^)]*)\)\s*->\s*\(([^)]*)\)(?:\s+(\w+))?$/); + if (!match) continue; + methods[match[1]] = { + input: splitShapes(match[2]), + output: normalizeResultAlias(match[3], didTypeAliases), + mode: match[4] ?? "update" + }; + } + return methods; +} + +function parseIdlTypes(source) { + const types = {}; + for (const declaration of extractIdlConstDeclarations(source)) { + const match = declaration.initializer.match(/^idl\.(Record|Variant)\(\{([^]*)\}\)$/m); + if (!match) continue; + const [, rawKind, body] = match; + const kind = rawKind === "Record" ? "record" : "variant"; + const fields = parseIdlFields(body); + types[declaration.name] = kind === "record" ? { kind, fields } : { kind, cases: fields }; + } + return types; +} + +function extractIdlConstDeclarations(source) { + const declarations = []; + const pattern = /const\s+(\w+)\s*=\s*/g; + let match; + while ((match = pattern.exec(source))) { + const name = match[1]; + const start = match.index + match[0].length; + const end = findStatementEnd(source, start); + if (end === -1) continue; + declarations.push({ name, initializer: source.slice(start, end).trim() }); + pattern.lastIndex = end + 1; + } + return declarations; +} + +function parseIdlFields(body) { + const fields = {}; + for (const raw of body.split(",")) { + const line = raw.trim(); + if (!line) continue; + const match = line.match(/^(\w+):\s*(.+)$/); + if (!match) continue; + fields[match[1]] = normalizeIdlShape(match[2]); + } + return fields; +} + +function parseIdlMethods(source) { + const service = source.match(/return\s+idl\.Service\(\{([^]*?)\n\s*\}\);/m)?.[1] ?? ""; + const methods = {}; + for (const match of service.matchAll(/^\s*(\w+):\s*idl\.Func\(\[\s*([^\]]*)\s*\],\s*\[\s*([^\]]+?)\s*\],\s*\[\s*(?:"(\w+)")?\s*\]\)/gm)) { + methods[match[1]] = { + input: splitIdlInputs(match[2]), + output: normalizeIdlShape(match[3]), + mode: match[4] ?? "update" + }; + } + return methods; +} + +function findStatementEnd(source, start) { + let depth = 0; + let inString = false; + for (let index = start; index < source.length; index += 1) { + const char = source[index]; + const previous = source[index - 1]; + if (char === "\"" && previous !== "\\") { + inString = !inString; + } + if (inString) continue; + if (char === "(" || char === "{" || char === "[") { + depth += 1; + } else if (char === ")" || char === "}" || char === "]") { + depth -= 1; + } else if (char === ";" && depth === 0) { + return index; + } + } + return -1; +} + +function normalizeIdlShape(value) { + const normalized = value + .trim() + .replace(/^idl\./, "") + .replace(/^Text$/, "text") + .replace(/^Int16$/, "int16") + .replace(/^Int64$/, "int64") + .replace(/^Nat64$/, "nat64") + .replace(/^Nat32$/, "nat32") + .replace(/^Nat16$/, "nat16") + .replace(/^Nat8$/, "nat8") + .replace(/^Nat$/, "nat") + .replace(/^Float32$/, "float32") + .replace(/^Bool$/, "bool") + .replace(/^Principal$/, "principal") + .replace(/^Null$/, "null") + .replace(/^Opt\((.+)\)$/, (_, inner) => `opt ${normalizeIdlShape(inner)}`) + .replace(/^Vec\((.+)\)$/, (_, inner) => `vec ${normalizeIdlShape(inner)}`); + return normalizeBlobAlias(normalized); +} + +function splitShapes(value) { + const trimmed = value.trim(); + if (!trimmed) return []; + return trimmed.split(",").map((part) => normalizeShape(part)); +} + +function normalizeShape(value) { + return normalizeBlobAlias(value.trim().replace(/\s+/g, " ")); +} + +function splitIdlInputs(value) { + const trimmed = value.trim(); + if (!trimmed) return []; + return trimmed.split(",").map((part) => normalizeIdlShape(part)); +} + +function normalizeBlobAlias(value) { + if (value === "vec nat8") return "blob"; + if (value === "opt vec nat8") return "opt blob"; + return value; +} + +function normalizeResultAlias(value, didTypeAliases) { + const normalized = normalizeShape(value).replace(/,$/, "").trim(); + const alias = Object.entries(didTypeAliases).find(([, didName]) => didName === normalized)?.[0]; + if (alias) return alias; + if (normalized === "Result") return "ResultWriteNode"; + return normalized; +} + +function compareShape(failures, label, actual, expected) { + if (!actual) { + failures.push(`${label} missing`); + return; + } + if (actual.kind !== expected.kind) { + failures.push(`${label} kind mismatch: ${actual.kind} != ${expected.kind}`); + return; + } + compareMap(failures, label, actual.fields ?? actual.cases, expected.fields ?? expected.cases); +} + +function compareMethod(failures, label, actual, expected, didTypeAliases) { + if (!actual) { + failures.push(`${label} missing`); + return; + } + const actualInput = actual.input.map((name) => canonicalTypeName(name, didTypeAliases)); + const expectedInput = expected.input.map((name) => canonicalTypeName(name, didTypeAliases)); + if (JSON.stringify(actualInput) !== JSON.stringify(expectedInput)) { + failures.push(`${label} input mismatch: ${actual.input.join(", ")} != ${expected.input.join(", ")}`); + } + if (actual.output !== expected.output) { + failures.push(`${label} output mismatch: ${actual.output} != ${expected.output}`); + } + if (actual.mode !== expected.mode) { + failures.push(`${label} mode mismatch: ${actual.mode} != ${expected.mode}`); + } +} + +function canonicalTypeName(name, didTypeAliases) { + return didTypeAliases[name] ?? name; +} + +function compareMap(failures, label, actual, expected) { + const actualKeys = Object.keys(actual).sort(); + const expectedKeys = Object.keys(expected).sort(); + if (JSON.stringify(actualKeys) !== JSON.stringify(expectedKeys)) { + failures.push(`${label} fields mismatch: ${actualKeys.join(", ")} != ${expectedKeys.join(", ")}`); + return; + } + for (const key of expectedKeys) { + if (actual[key] !== expected[key]) { + failures.push(`${label}.${key} mismatch: ${actual[key]} != ${expected[key]}`); + } + } +} diff --git a/scripts/check-build-vfs-canister-guard.mjs b/scripts/check-build-vfs-canister-guard.mjs new file mode 100644 index 00000000..9f7bf9a7 --- /dev/null +++ b/scripts/check-build-vfs-canister-guard.mjs @@ -0,0 +1,32 @@ +// Where: scripts/check-build-vfs-canister-guard.mjs +// What: Verify production builds reject local Internet Identity origins. +// Why: A compile-time local II origin flag must not leak into mainnet artifacts. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const script = join(root, "scripts", "build-vfs-canister.sh"); + +assert.equal(run({ KINIC_VFS_LOCAL_II_ORIGINS: "1", ICP_ENVIRONMENT: "ic" }).status, 1); +assert.match( + run({ KINIC_VFS_LOCAL_II_ORIGINS: "1", ICP_ENVIRONMENT: "ic" }).stderr, + /only allowed for ICP_ENVIRONMENT=local or local-wiki/ +); +assert.equal(run({ KINIC_VFS_LOCAL_II_ORIGINS: "1", ICP_ENVIRONMENT: "local-wiki" }).status, 0); +assert.equal(run({ KINIC_VFS_LOCAL_II_ORIGINS: "1" }).status, 1); +assert.equal(run({}).status, 0); + +console.log("Build VFS canister guard OK"); + +function run(env) { + const cleanEnv = { ...process.env }; + delete cleanEnv.KINIC_VFS_LOCAL_II_ORIGINS; + delete cleanEnv.ICP_ENVIRONMENT; + return spawnSync("bash", [script, "--check-env-only"], { + cwd: root, + env: { ...cleanEnv, ...env }, + encoding: "utf8" + }); +} diff --git a/scripts/local/deploy_wiki.sh b/scripts/local/deploy_wiki.sh index 706d72e0..f576dab4 100755 --- a/scripts/local/deploy_wiki.sh +++ b/scripts/local/deploy_wiki.sh @@ -63,4 +63,6 @@ if [[ "${1:-}" == "--dry-run" ]]; then fi cd "${REPO_ROOT}" +export ICP_ENVIRONMENT +export KINIC_VFS_LOCAL_II_ORIGINS=1 icp deploy wiki -e "${ICP_ENVIRONMENT}" --args-file "${ARGS_FILE}" "$@" diff --git a/scripts/mainnet/deploy_wiki.sh b/scripts/mainnet/deploy_wiki.sh index 224f522e..628e95df 100755 --- a/scripts/mainnet/deploy_wiki.sh +++ b/scripts/mainnet/deploy_wiki.sh @@ -49,4 +49,5 @@ if [[ "${1:-}" == "--dry-run" ]]; then fi cd "${REPO_ROOT}" +unset KINIC_VFS_LOCAL_II_ORIGINS icp deploy wiki -e ic --args-file "${ARGS_FILE}" "$@" diff --git a/scripts/setup-wikibrowser-ii-e2e.sh b/scripts/setup-wikibrowser-ii-e2e.sh index 79e3ed36..88804fd3 100755 --- a/scripts/setup-wikibrowser-ii-e2e.sh +++ b/scripts/setup-wikibrowser-ii-e2e.sh @@ -19,6 +19,46 @@ II_BACKEND_INIT_ARGS='(opt record { captcha_config = opt record { max_unsolved_c mkdir -p "$ARTIFACT_DIR" +current_identity_principal() { + icp identity principal +} + +resolve_wiki_canister_id() { + if [ -f "$MAPPING_FILE" ]; then + node -e ' + const fs = require("fs"); + const [file] = process.argv.slice(1); + const ids = JSON.parse(fs.readFileSync(file, "utf8")); + if (typeof ids.wiki !== "string" || ids.wiki.trim() === "") process.exit(1); + process.stdout.write(ids.wiki); + ' "$MAPPING_FILE" + return + fi + return 1 +} + +canister_has_module() { + local canister_id="$1" + icp canister status "$canister_id" -e local-wiki --json \ + | node -e ' + const fs = require("fs"); + const status = JSON.parse(fs.readFileSync(0, "utf8")); + process.exit(status.module_hash ? 0 : 1); + ' +} + +wiki_ledger_canister_id() { + icp canister call wiki get_cycles_billing_config '()' -e local-wiki -o candid 2>/dev/null \ + | awk -F'"' '/kinic_ledger_canister_id/ { print $2; exit }' +} + +deploy_wiki() { + ICP_ENVIRONMENT=local-wiki \ + KINIC_LEDGER_CANISTER_ID="$KINIC_LEDGER_CANISTER_ID" \ + BILLING_AUTHORITY_ID="$BILLING_AUTHORITY_ID" \ + bash "$ROOT_DIR/scripts/local/deploy_wiki.sh" "$@" +} + ensure_canister_id() { local id_file="$1" if [ -s "$id_file" ]; then @@ -31,6 +71,22 @@ ensure_canister_id() { icp canister create --detached -e local-wiki --quiet > "$id_file" } +ensure_distinct_canister_id() { + local id_file="$1" + shift + ensure_canister_id "$id_file" + local canister_id + canister_id="$(tr -d '[:space:]' < "$id_file")" + local forbidden + for forbidden in "$@"; do + if [ -n "$forbidden" ] && [ "$canister_id" = "$forbidden" ]; then + echo "canister id $canister_id in $id_file conflicts with reserved local canister; creating a new detached canister" >&2 + icp canister create --detached -e local-wiki --quiet > "$id_file" + return + fi + done +} + if [ ! -s "$BACKEND_WASM_GZ" ]; then curl -fsSL "$II_BACKEND_WASM_URL" -o "$BACKEND_WASM_GZ" fi @@ -51,12 +107,38 @@ if [ ! -s "$BACKEND_CANISTER_ID_FILE" ] && [ -s "$LEGACY_CANISTER_ID_FILE" ]; th cp "$LEGACY_CANISTER_ID_FILE" "$BACKEND_CANISTER_ID_FILE" fi -ensure_canister_id "$BACKEND_CANISTER_ID_FILE" -ensure_canister_id "$FRONTEND_CANISTER_ID_FILE" +if [ -z "${BILLING_AUTHORITY_ID:-}" ]; then + BILLING_AUTHORITY_ID="$(current_identity_principal)" +fi +LEDGER_SETUP_OUTPUT="$(ICP_ENVIRONMENT=local-wiki bash "$ROOT_DIR/scripts/local/setup_kinic_ledger.sh")" +KINIC_LEDGER_CANISTER_ID="${LEDGER_SETUP_OUTPUT#KINIC_LEDGER_CANISTER_ID=}" +export KINIC_LEDGER_CANISTER_ID +export BILLING_AUTHORITY_ID + +if ! WIKI_CANISTER_ID="$(resolve_wiki_canister_id)"; then + echo "local wiki canister id not found; deploying wiki to local-wiki" >&2 + deploy_wiki + WIKI_CANISTER_ID="$(resolve_wiki_canister_id)" +elif canister_has_module "$WIKI_CANISTER_ID" >/dev/null 2>&1; then + CURRENT_LEDGER_CANISTER_ID="$(wiki_ledger_canister_id || true)" + if [ "$CURRENT_LEDGER_CANISTER_ID" != "$KINIC_LEDGER_CANISTER_ID" ]; then + echo "wiki ledger mismatch (${CURRENT_LEDGER_CANISTER_ID:-missing}); reinstalling wiki for $KINIC_LEDGER_CANISTER_ID" >&2 + deploy_wiki --mode reinstall + else + deploy_wiki + fi +else + echo "local wiki canister $WIKI_CANISTER_ID missing installed module; deploying wiki to local-wiki" >&2 + deploy_wiki +fi +WIKI_CANISTER_ID="$(resolve_wiki_canister_id)" + +ensure_distinct_canister_id "$BACKEND_CANISTER_ID_FILE" "$KINIC_LEDGER_CANISTER_ID" "$WIKI_CANISTER_ID" II_BACKEND_CANISTER_ID="$(tr -d '[:space:]' < "$BACKEND_CANISTER_ID_FILE")" +ensure_distinct_canister_id "$FRONTEND_CANISTER_ID_FILE" "$KINIC_LEDGER_CANISTER_ID" "$WIKI_CANISTER_ID" "$II_BACKEND_CANISTER_ID" + II_FRONTEND_CANISTER_ID="$(tr -d '[:space:]' < "$FRONTEND_CANISTER_ID_FILE")" -WIKI_CANISTER_ID="$(node -e 'const fs=require("fs"); const file=process.argv[1]; const ids=JSON.parse(fs.readFileSync(file,"utf8")); if(!ids.wiki) throw new Error("wiki canister id is missing"); process.stdout.write(ids.wiki);' "$MAPPING_FILE")" II_FRONTEND_INIT_ARGS="$(printf '(record { backend_canister_id = principal "%s"; backend_origin = "http://%s.raw.localhost:8011"; related_origins = null; fetch_root_key = opt true; analytics_config = null; dummy_auth = opt opt record { prompt_for_index = false }; dev_csp = opt true })' "$II_BACKEND_CANISTER_ID" "$II_BACKEND_CANISTER_ID")" if ! icp canister install "$II_BACKEND_CANISTER_ID" \ @@ -95,9 +177,12 @@ fi { printf 'NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8011\n' printf 'NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID=%s\n' "$WIKI_CANISTER_ID" + printf 'NEXT_PUBLIC_ENABLE_LOCAL_II_E2E=1\n' printf 'NEXT_PUBLIC_II_PROVIDER_URL=http://%s.raw.localhost:8011\n' "$II_FRONTEND_CANISTER_ID" } > "$ENV_FILE" printf 'Wrote %s\n' "$ENV_FILE" printf 'NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID=%s\n' "$WIKI_CANISTER_ID" +printf 'NEXT_PUBLIC_ENABLE_LOCAL_II_E2E=1\n' printf 'NEXT_PUBLIC_II_PROVIDER_URL=http://%s.raw.localhost:8011\n' "$II_FRONTEND_CANISTER_ID" +printf 'For manual localhost testing, run: cp wikibrowser/.env.e2e.local wikibrowser/.env.local && pnpm -C wikibrowser dev -p 3010\n' diff --git a/shared/ii-auth/index.js b/shared/ii-auth/index.js index 8cf0d399..ed0f24c0 100644 --- a/shared/ii-auth/index.js +++ b/shared/ii-auth/index.js @@ -22,19 +22,11 @@ export function authClientCreateOptions(idleTimeoutMs = AUTH_SESSION_TTL_MS) { }; } -export function identityProviderUrlForLocation(locationLike) { - const hostname = locationLike?.hostname ?? ""; - if (hostname === "localhost" || hostname === "127.0.0.1" || hostname.endsWith(".localhost")) { - return `http://id.ai.localhost:${locationLike?.port || "8000"}`; - } +export function identityProviderUrlForLocation() { return MAINNET_II_PROVIDER_URL; } -export function derivationOriginForLocation(locationLike) { - const hostname = locationLike?.hostname ?? ""; - if (hostname === "localhost" || hostname === "127.0.0.1" || hostname.endsWith(".localhost")) { - return locationLike.origin; - } +export function derivationOriginForLocation() { return WIKI_CANISTER_DERIVATION_ORIGIN; } diff --git a/skill-registry-web/README.md b/skill-registry-web/README.md index 360dfdcb..d2ef3503 100644 --- a/skill-registry-web/README.md +++ b/skill-registry-web/README.md @@ -26,7 +26,6 @@ Required public environment: ```bash NEXT_PUBLIC_WIKI_IC_HOST=https://icp0.io -NEXT_PUBLIC_II_PROVIDER_URL=https://id.ai NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID=xis3j-paaaa-aaaai-axumq-cai ``` diff --git a/skill-registry-web/lib/auth.ts b/skill-registry-web/lib/auth.ts index 17b381f3..d49a72a1 100644 --- a/skill-registry-web/lib/auth.ts +++ b/skill-registry-web/lib/auth.ts @@ -4,6 +4,7 @@ const DELEGATION_DAYS = 29; const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000; export const DELEGATION_TTL_NS = BigInt(DELEGATION_DAYS) * HOURS_PER_DAY * NANOSECONDS_PER_HOUR; +export const MAINNET_II_PROVIDER_URL = "https://id.ai"; export const DERIVATION_ORIGIN = "https://xis3j-paaaa-aaaai-axumq-cai.icp0.io"; export const AUTH_CLIENT_CREATE_OPTIONS = { idleOptions: { @@ -13,14 +14,7 @@ export const AUTH_CLIENT_CREATE_OPTIONS = { }; export function identityProviderUrl(): string { - if (process.env.NEXT_PUBLIC_II_PROVIDER_URL) { - return process.env.NEXT_PUBLIC_II_PROVIDER_URL; - } - const host = window.location.hostname; - if (host === "localhost" || host === "127.0.0.1" || host.endsWith(".localhost")) { - return "http://id.ai.localhost:8000"; - } - return "https://id.ai"; + return MAINNET_II_PROVIDER_URL; } export function authLoginOptions() { diff --git a/skill-registry-web/next.config.ts b/skill-registry-web/next.config.ts index 3ecb638e..65c92d7f 100644 --- a/skill-registry-web/next.config.ts +++ b/skill-registry-web/next.config.ts @@ -4,7 +4,6 @@ const nextConfig: NextConfig = { allowedDevOrigins: ["127.0.0.1"], env: { NEXT_PUBLIC_WIKI_IC_HOST: process.env.NEXT_PUBLIC_WIKI_IC_HOST ?? "https://icp0.io", - NEXT_PUBLIC_II_PROVIDER_URL: process.env.NEXT_PUBLIC_II_PROVIDER_URL ?? "https://id.ai", NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID: process.env.NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID ?? "" }, reactStrictMode: true diff --git a/skill-registry-web/scripts/check-skill-registry-web.mjs b/skill-registry-web/scripts/check-skill-registry-web.mjs index 05833027..0637c83e 100644 --- a/skill-registry-web/scripts/check-skill-registry-web.mjs +++ b/skill-registry-web/scripts/check-skill-registry-web.mjs @@ -1,6 +1,8 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import ts from "typescript"; +import { checkCandidSubset } from "../../scripts/candid-subset-check.mjs"; +import { didTypeAliases, expectedMethods, expectedTypes } from "../../wikibrowser/scripts/candid-shapes.mjs"; const route = readFileSync(new URL("../app/skills/[databaseId]/page.tsx", import.meta.url), "utf8"); const client = readFileSync(new URL("../app/skills/skill-registry-client.tsx", import.meta.url), "utf8"); @@ -15,6 +17,7 @@ const types = readFileSync(new URL("../lib/types.ts", import.meta.url), "utf8"); const vfsClient = readFileSync(new URL("../lib/vfs-client.ts", import.meta.url), "utf8"); const vfsIdl = readFileSync(new URL("../lib/vfs-idl.ts", import.meta.url), "utf8"); const packageJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")); +const did = readFileSync(new URL("../../crates/vfs_canister/vfs.did", import.meta.url), "utf8"); assert.equal(packageJson.name, "kinic-skill-registry-web"); assert.match(route, //); @@ -68,6 +71,16 @@ assert.match(vfsIdl, /Active: idl\.Null/); assert.match(vfsIdl, /status: DatabaseStatus/); assert.match(vfsIdl, /Deleted: idl\.Null/); assert.match(vfsIdl, /deleted_at_ms: idl\.Opt\(idl\.Int64\)/); +assert.deepEqual( + checkCandidSubset({ + didSource: did, + idlSource: vfsIdl, + expectedTypes: pickUsedExpectedTypes(vfsIdl), + expectedMethods: pickUsedExpectedMethods(vfsIdl), + didTypeAliases + }), + [] +); assert.match(vfsClient, /function normalizeDatabaseStatus/); assert.match(vfsClient, /"Active" in status/); assert.match(vfsClient, /"Pending" in status/); @@ -129,6 +142,18 @@ assert.equal( console.log("Skill Registry web checks OK"); +function pickUsedExpectedTypes(source) { + return Object.fromEntries( + Object.entries(expectedTypes).filter(([name]) => new RegExp(`const\\s+${name}\\s*=`).test(source)) + ); +} + +function pickUsedExpectedMethods(source) { + return Object.fromEntries( + Object.entries(expectedMethods).filter(([name]) => new RegExp(`${name}:\\s*idl\\.Func`).test(source)) + ); +} + async function importSkillRegistryPackageForTest(relativePath) { const sourcePath = new URL(relativePath, import.meta.url); const source = readFileSync(sourcePath, "utf8") diff --git a/wikibrowser/README.md b/wikibrowser/README.md index ce8fe0e2..7f58b0b2 100644 --- a/wikibrowser/README.md +++ b/wikibrowser/README.md @@ -24,17 +24,15 @@ DB_ID="$(cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database grant "$DB_ID" 2vxsx-fae reader ``` -`database create ` creates a generated database ID and prints it on success. `NEXT_PUBLIC_WIKI_IC_HOST` controls the browser-side IC agent host. `NEXT_PUBLIC_II_PROVIDER_URL` overrides the Internet Identity frontend URL for local II. `NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID` selects the fixed wiki canister: +`database create ` creates a generated database ID and prints it on success. `NEXT_PUBLIC_WIKI_IC_HOST` controls the browser-side IC agent host. Internet Identity uses the mainnet provider `https://id.ai` by default. `NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID` selects the fixed wiki canister: ```bash # local icp network NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8011 -NEXT_PUBLIC_II_PROVIDER_URL=http://id.ai.localhost:8011 NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID= # mainnet / Cloudflare Workers NEXT_PUBLIC_WIKI_IC_HOST=https://icp0.io -NEXT_PUBLIC_II_PROVIDER_URL=https://id.ai NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID=xis3j-paaaa-aaaai-axumq-cai ``` @@ -100,15 +98,25 @@ pnpm typecheck pnpm build ``` -Internet Identity E2E requires a local wiki canister and the E2E setup script. The script deploys the pinned Internet Identity backend/frontend dev canisters with dummy auth and writes `.env.e2e.local`. Override `II_RELEASE` only when intentionally updating the tested Internet Identity release. +Internet Identity for `localhost` uses the local II canisters prepared by the E2E setup script. The script deploys the local wiki, KINIC ledger, and pinned Internet Identity backend/frontend dev canisters with dummy auth, then writes `.env.e2e.local` with `NEXT_PUBLIC_ENABLE_LOCAL_II_E2E=1`. Copy that file to `.env.local` for manual browser testing on `localhost`; restart the dev server after copying so Next picks up the new public env values. Mainnet II (`https://id.ai`) is reserved for production or preview origins, not `localhost`. Override `II_RELEASE` only when intentionally updating the tested Internet Identity release. ```bash +cd .. icp network start -d -e local-wiki -KINIC_LEDGER_CANISTER_ID= scripts/local/deploy_wiki.sh +cd wikibrowser pnpm e2e:ii:setup +cp .env.e2e.local .env.local +pnpm dev -p 3010 +``` + +Run E2E in another terminal from `wikibrowser/` while the dev server is running: + +```bash pnpm e2e:ii ``` +For production and preview deployments, leave `NEXT_PUBLIC_ENABLE_LOCAL_II_E2E` unset so auth uses `https://id.ai` with the production derivation origin. Do not add `localhost` or `127.0.0.1` to the production `ii-alternative-origins`; Internet Identity also rejects alternative-origin lists with more than 10 entries. + The wiki canister constructor requires cycles billing config; use the deploy wrapper instead of no-arg `icp deploy`. `next-env.d.ts` is generated by Next and is intentionally ignored. `pnpm typecheck` runs `next typegen` before `tsc` so clean checkouts do not need to commit that file. diff --git a/wikibrowser/app/app-header.tsx b/wikibrowser/app/app-header.tsx index 936aee32..d4254141 100644 --- a/wikibrowser/app/app-header.tsx +++ b/wikibrowser/app/app-header.tsx @@ -3,8 +3,13 @@ // Where: root wikibrowser layout. // What: renders the shared dashboard/cycles header with wallet and II controls. // Why: funding pages should keep the same wallet session and management shell. +import Link from "next/link"; import { usePathname } from "next/navigation"; +import { CircleAlert, X } from "lucide-react"; +import { useState } from "react"; import { AdminHeader } from "@/components/admin-header"; +import { parseDepositAmount } from "@/lib/kinic-deposit"; +import { depositKinicBalanceWithIdentity } from "@/lib/kinic-wallet"; import { formatTokenAmountFromE8s } from "@/lib/kinic-amount"; import { AuthControls, WalletControls } from "./home-ui"; import { connectedWalletPrincipal, useAppSession } from "./app-session-provider"; @@ -17,29 +22,52 @@ export function AppHeader() { authReady, connectWallet, disconnectWallet, + kinicBalance, + kinicBalanceError, + kinicBalanceLoading, login, logout, principal, + refreshKinicBalance, wallet, walletBalance, walletBalanceLoading, walletBusyProvider, walletControlsLocked } = useAppSession(); + const [kinicModalOpen, setKinicModalOpen] = useState(false); - if (pathname !== "/dashboard" && pathname !== "/cycles") return null; + const isMarketplace = pathname === "/marketplace" || pathname.startsWith("/marketplace/"); + if (pathname !== "/dashboard" && pathname !== "/cycles" && !isMarketplace) return null; - const title = pathname === "/cycles" ? "Database cycles purchase" : "Database dashboard"; + const title = pathname === "/cycles" ? "Database cycles purchase" : isMarketplace ? "Kinic marketplace" : "Database dashboard"; const connectedWalletLabel = wallet ? `${walletLabel(wallet.provider)} ${shortPrincipal(connectedWalletPrincipal(wallet))}` : null; const connectedWalletBalanceLabel = walletBalance ? formatTokenAmountFromE8s(walletBalance) : null; + const kinicBalanceLabel = kinicBalanceLoading + ? "Loading" + : kinicBalanceError + ? "Unavailable" + : principal && kinicBalance !== null + ? formatTokenAmountFromE8s(kinicBalance) + : "- KINIC"; return (
-
+
} actions={ <> + } /> + {kinicModalOpen ? ( + setKinicModalOpen(false)} + /> + ) : null}
); @@ -75,6 +117,136 @@ function walletLabel(provider: "oisy" | "plug"): string { return provider === "oisy" ? "OISY" : "Plug"; } +function HeaderNav({ pathname }: { pathname: string }) { + return ( + + ); +} + +type KinicDepositModalProps = { + authClient: ReturnType["authClient"]; + authReady: boolean; + balance: string | null; + balanceError: string | null; + balanceLoading: boolean; + canisterId: string; + login: () => Promise; + principal: string | null; + refreshKinicBalance: () => Promise; + onClose: () => void; +}; + +function KinicDepositModal({ + authClient, + authReady, + balance, + balanceError, + balanceLoading, + canisterId, + login, + principal, + refreshKinicBalance, + onClose +}: KinicDepositModalProps) { + const [amount, setAmount] = useState("1"); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [message, setMessage] = useState(null); + const currentBalance = balanceLoading ? "Loading" : balance !== null ? formatTokenAmountFromE8s(balance) : "-"; + + async function deposit() { + if (!authClient || !principal) { + setError("Login with Internet Identity first"); + return; + } + const amountE8s = parseDepositAmount(amount); + if (!amountE8s) { + setError("Enter an amount greater than 0 KINIC"); + return; + } + setBusy(true); + setError(null); + setMessage(null); + try { + const result = await depositKinicBalanceWithIdentity({ canisterId, amountE8s: BigInt(amountE8s) }, authClient.getIdentity()); + setMessage(`Deposit block ${result.depositBlockIndex}. Balance ${formatTokenAmountFromE8s(result.balanceE8s)}`); + await refreshKinicBalance(); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setBusy(false); + } + } + + return ( +
{ + if (event.target === event.currentTarget) onClose(); + }} + > +
+
+
+

Deposit KINIC

+

{principal ?? (authReady ? "Internet Identity disconnected" : "Loading identity")}

+
+ +
+ +
+

Balance

+

{currentBalance}

+ {balanceError ?

{balanceError}

: null} +
+ +
+ setAmount(event.target.value)} + /> + + {!principal ? ( + + ) : null} +
+ + {message ?

{message}

: null} + {error ? ( +

+ + {error} +

+ ) : null} +
+
+ ); +} + function shortPrincipal(value: string): string { if (value.length <= 16) return value; return `${value.slice(0, 8)}...${value.slice(-5)}`; diff --git a/wikibrowser/app/app-session-provider.tsx b/wikibrowser/app/app-session-provider.tsx index e48879b2..544f06b2 100644 --- a/wikibrowser/app/app-session-provider.tsx +++ b/wikibrowser/app/app-session-provider.tsx @@ -6,7 +6,8 @@ import { AuthClient } from "@icp-sdk/auth/client"; import { createContext, useCallback, useContext, useEffect, useRef, useState, type ReactNode } from "react"; import { AUTH_CLIENT_CREATE_OPTIONS, authLoginOptions } from "@/lib/auth"; -import { connectOisyWallet, connectPlugWallet, getConnectedWalletKinicBalance, type ConnectedKinicWallet } from "@/lib/cycles-wallet"; +import { connectOisyWallet, connectPlugWallet, getConnectedWalletKinicBalance, type ConnectedKinicWallet } from "@/lib/kinic-wallet"; +import { kinicGetBalance } from "@/lib/vfs-client"; import type { HeaderWalletProvider } from "./home-ui"; type AppSessionContext = { @@ -15,6 +16,9 @@ type AppSessionContext = { authLoading: boolean; authReady: boolean; principal: string | null; + kinicBalance: string | null; + kinicBalanceError: string | null; + kinicBalanceLoading: boolean; wallet: ConnectedKinicWallet | null; walletBalance: string | null; walletBalanceError: string | null; @@ -25,6 +29,7 @@ type AppSessionContext = { disconnectWallet: (provider: HeaderWalletProvider) => void; logout: () => Promise; login: () => Promise; + refreshKinicBalance: () => Promise; refreshWalletBalance: (wallet: ConnectedKinicWallet) => Promise; setWalletControlsLocked: (locked: boolean) => void; }; @@ -34,12 +39,16 @@ const AppSession = createContext(null); export function AppSessionProvider({ children }: { children: ReactNode }) { const canisterId = process.env.NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID ?? ""; + const kinicBalanceSeqRef = useRef(0); const walletBalanceSeqRef = useRef(0); const [authClient, setAuthClient] = useState(null); const [authError, setAuthError] = useState(null); const [authLoading, setAuthLoading] = useState(true); const [authReady, setAuthReady] = useState(false); const [principal, setPrincipal] = useState(null); + const [kinicBalance, setKinicBalance] = useState(null); + const [kinicBalanceError, setKinicBalanceError] = useState(null); + const [kinicBalanceLoading, setKinicBalanceLoading] = useState(false); const [wallet, setWallet] = useState(() => readStoredWallet()); const [walletBalance, setWalletBalance] = useState(null); const [walletBalanceError, setWalletBalanceError] = useState(null); @@ -75,6 +84,36 @@ export function AppSessionProvider({ children }: { children: ReactNode }) { clearStoredWallet(); }, [clearStoredWallet]); + const clearKinicBalance = useCallback(() => { + kinicBalanceSeqRef.current += 1; + setKinicBalance(null); + setKinicBalanceLoading(false); + setKinicBalanceError(null); + }, []); + + const refreshKinicBalance = useCallback(async () => { + const balanceSeq = (kinicBalanceSeqRef.current += 1); + const isCurrentBalance = () => balanceSeq === kinicBalanceSeqRef.current; + if (!authClient || !principal) { + clearKinicBalance(); + return; + } + setKinicBalanceLoading(true); + setKinicBalanceError(null); + try { + const balance = await kinicGetBalance(canisterId, authClient.getIdentity()); + if (!isCurrentBalance()) return; + setKinicBalance(balance.balanceE8s); + } catch (cause) { + if (!isCurrentBalance()) return; + setKinicBalance(null); + setKinicBalanceError(`KINIC balance unavailable: ${errorMessage(cause)}`); + } finally { + if (!isCurrentBalance()) return; + setKinicBalanceLoading(false); + } + }, [authClient, canisterId, clearKinicBalance, principal]); + const refreshWalletBalance = useCallback( async (nextWallet: ConnectedKinicWallet) => { const balanceSeq = (walletBalanceSeqRef.current += 1); @@ -128,10 +167,15 @@ export function AppSessionProvider({ children }: { children: ReactNode }) { [clearWallet, wallet, walletBusyProvider, walletControlsLocked] ); - const syncAuth = useCallback(async (client: AuthClient) => { - const authenticated = await client.isAuthenticated(); - setPrincipal(authenticated ? client.getIdentity().getPrincipal().toText() : null); - }, []); + const syncAuth = useCallback( + async (client: AuthClient) => { + const authenticated = await client.isAuthenticated(); + const nextPrincipal = authenticated ? client.getIdentity().getPrincipal().toText() : null; + setPrincipal(nextPrincipal); + if (!nextPrincipal) clearKinicBalance(); + }, + [clearKinicBalance] + ); const login = useCallback(async () => { if (!authClient) return; @@ -156,13 +200,14 @@ export function AppSessionProvider({ children }: { children: ReactNode }) { try { await authClient.logout(); setPrincipal(null); + clearKinicBalance(); clearWallet(); } catch (cause) { setAuthError(errorMessage(cause)); } finally { setAuthLoading(false); } - }, [authClient, clearWallet]); + }, [authClient, clearKinicBalance, clearWallet]); useEffect(() => { let cancelled = false; @@ -188,6 +233,18 @@ export function AppSessionProvider({ children }: { children: ReactNode }) { }; }, [syncAuth]); + useEffect(() => { + if (!authClient || !principal) return; + let cancelled = false; + queueMicrotask(() => { + if (cancelled) return; + void refreshKinicBalance(); + }); + return () => { + cancelled = true; + }; + }, [authClient, principal, refreshKinicBalance]); + useEffect(() => { if (!wallet) return; let cancelled = false; @@ -208,6 +265,9 @@ export function AppSessionProvider({ children }: { children: ReactNode }) { authLoading, authReady, principal, + kinicBalance, + kinicBalanceError, + kinicBalanceLoading, wallet, walletBalance, walletBalanceError, @@ -218,6 +278,7 @@ export function AppSessionProvider({ children }: { children: ReactNode }) { disconnectWallet, login, logout, + refreshKinicBalance, refreshWalletBalance, setWalletControlsLocked }} diff --git a/wikibrowser/app/create-database-dialog.tsx b/wikibrowser/app/create-database-dialog.tsx index f04d7340..e5fa07d2 100644 --- a/wikibrowser/app/create-database-dialog.tsx +++ b/wikibrowser/app/create-database-dialog.tsx @@ -35,7 +35,12 @@ export function CreateDatabaseDialog({ } return ( -
+
{ + if (!creating && event.target === event.currentTarget) onCancel(); + }} + >
diff --git a/wikibrowser/app/cycles/cycles-client.tsx b/wikibrowser/app/cycles/cycles-client.tsx index 18c2ab11..7cee5450 100644 --- a/wikibrowser/app/cycles/cycles-client.tsx +++ b/wikibrowser/app/cycles/cycles-client.tsx @@ -7,13 +7,17 @@ import { useRouter } from "next/navigation"; import { CheckCircle2, CircleAlert, Info, PlugZap, Wallet } from "lucide-react"; import { useMemo, useState } from "react"; import { useAppSession } from "@/app/app-session-provider"; +import { cyclesForPaymentAmountE8s } from "@/lib/cycles"; import { parseKinicAmountE8sInput, parseCyclesTarget } from "@/lib/cycles-url"; -import { purchaseCyclesWithOisy, purchaseCyclesWithPlug } from "@/lib/cycles-wallet"; +import { purchaseCyclesWithOisy, purchaseCyclesWithPlug } from "@/lib/kinic-wallet"; import { formatTokenAmountFromE8s } from "@/lib/kinic-amount"; +import { getCyclesBillingConfig, kinicFundDatabaseCycles } from "@/lib/vfs-client"; import type { DatabaseStatus } from "@/lib/types"; type CyclesStatus = "idle" | "running" | "success" | "error"; type CyclesProvider = "oisy" | "plug"; +type FundingProvider = CyclesProvider | "ii"; +type PaymentSource = "wallet" | "kinic"; type CyclesClientProps = { canisterId: string; @@ -23,10 +27,11 @@ type CyclesClientProps = { export function CyclesClient({ canisterId, databaseId, databaseStatus }: CyclesClientProps) { const router = useRouter(); - const { refreshWalletBalance, wallet, walletBalanceError, walletBusyProvider } = useAppSession(); + const { authClient, authLoading, login, principal, refreshKinicBalance, refreshWalletBalance, wallet, walletBalanceError, walletBusyProvider } = useAppSession(); const [status, setStatus] = useState("idle"); const [message, setMessage] = useState(null); const [amount, setAmount] = useState("1"); + const [paymentSource, setPaymentSource] = useState("wallet"); const configuredCanisterId = process.env.NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID ?? ""; const parsedTarget = useMemo(() => { const params = new URLSearchParams(); @@ -43,15 +48,48 @@ export function CyclesClient({ canisterId, databaseId, databaseStatus }: CyclesC const amountError = typeof parsedAmount === "string" ? parsedAmount : null; const busy = status === "running" || walletBusyProvider !== null; const selectedProvider = wallet?.provider ?? null; - const purchaseDisabled = !selectedProvider || Boolean(error) || Boolean(amountError) || busy; + const kinicBalancePendingDisabled = paymentSource === "kinic" && databaseStatus === "pending"; + const purchaseDisabled = + Boolean(error) || + Boolean(amountError) || + busy || + (paymentSource === "wallet" ? !selectedProvider : authLoading || !authClient || kinicBalancePendingDisabled); async function purchase() { - if (!wallet || !selectedProvider) return; if (typeof parsedTarget === "string" || typeof parsedAmount !== "bigint" || error) return; setStatus("running"); setMessage(null); try { const request = { canisterId, databaseId: parsedTarget.databaseId, paymentAmountE8s: parsedAmount }; + if (paymentSource === "kinic") { + if (!authClient || !principal) { + await login(); + setStatus("idle"); + return; + } + const config = await getCyclesBillingConfig(canisterId); + const minExpectedCycles = cyclesForPaymentAmountE8s(parsedAmount, BigInt(config.cyclesPerKinic)); + const result = await kinicFundDatabaseCycles( + canisterId, + authClient.getIdentity(), + parsedTarget.databaseId, + parsedAmount.toString(), + minExpectedCycles.toString() + ); + setMessage( + `Internet Identity funded cycles ${result.amountCycles}; paid ${formatTokenAmountFromE8s(result.paymentAmountE8s)} from KINIC balance; database cycles balance ${result.databaseBalanceCycles}; KINIC balance ${formatTokenAmountFromE8s(result.kinicBalanceE8s)}` + ); + await refreshKinicBalance(); + setStatus("success"); + router.replace(cyclesPurchaseSuccessHref({ + cycles: result.amountCycles, + databaseId: parsedTarget.databaseId, + kinic: formatTokenAmountFromE8s(result.paymentAmountE8s), + provider: "ii" + })); + return; + } + if (!wallet || !selectedProvider) return; const result = wallet.provider === "oisy" ? await purchaseCyclesWithOisy(request, wallet.connection) @@ -91,10 +129,30 @@ export function CyclesClient({ canisterId, databaseId, databaseStatus }: CyclesC /> {amountError ? {amountError} : null} +
+ Payment source +
+ + +
+
{databaseStatus === "pending" ? : null} - +
{error ? : null} - {walletBalanceError ? : null} + {paymentSource === "wallet" && walletBalanceError ? : null} + {paymentSource === "kinic" && !principal ? : null} {status === "success" && message ? : null} {status === "error" && message ? : null} @@ -117,7 +176,10 @@ export function CyclesClient({ canisterId, databaseId, databaseStatus }: CyclesC ); } -function purchaseButtonLabel(selectedProvider: CyclesProvider | null, status: CyclesStatus): string { +function purchaseButtonLabel(selectedProvider: CyclesProvider | null, status: CyclesStatus, paymentSource: PaymentSource): string { + if (paymentSource === "kinic") { + return status === "running" ? "Processing Internet Identity" : "Fund cycles from KINIC balance"; + } if (status === "running") { if (selectedProvider === "oisy") return "Processing OISY"; if (selectedProvider === "plug") return "Processing Plug"; @@ -136,7 +198,7 @@ function cyclesPurchaseSuccessHref({ cycles: string; databaseId: string; kinic: string; - provider: CyclesProvider; + provider: FundingProvider; }): string { const params = new URLSearchParams(); params.set("funding", "success"); diff --git a/wikibrowser/app/cycles/page.tsx b/wikibrowser/app/cycles/page.tsx index 6bce19fe..7895425d 100644 --- a/wikibrowser/app/cycles/page.tsx +++ b/wikibrowser/app/cycles/page.tsx @@ -29,7 +29,7 @@ function first(value: string | string[] | undefined): string { } function parseDatabaseStatus(value: string): DatabaseStatus | null { - if (value === "pending" || value === "active" || value === "restoring" || value === "archiving" || value === "archived" || value === "deleted") { + if (value === "pending" || value === "active" || value === "restoring" || value === "archiving" || value === "archived") { return value; } return null; diff --git a/wikibrowser/app/dashboard/dashboard-client.tsx b/wikibrowser/app/dashboard/dashboard-client.tsx index 47877c6b..9e41ec07 100644 --- a/wikibrowser/app/dashboard/dashboard-client.tsx +++ b/wikibrowser/app/dashboard/dashboard-client.tsx @@ -9,7 +9,7 @@ import { AuthControls, CyclesHistoryPanel, DashboardTabs, OwnerPanel, PendingDat import { AdminHeader } from "@/components/admin-header"; import { CycleBattery } from "@/components/cycle-battery"; import { AUTH_CLIENT_CREATE_OPTIONS, authLoginOptions } from "@/lib/auth"; -import type { CyclesBillingConfig, DatabaseCycleEntry, DatabaseCyclesPendingPurchase, DatabaseMember, DatabaseRole, DatabaseSummary } from "@/lib/types"; +import type { CyclesBillingConfig, DatabaseCycleEntry, DatabaseCyclesPendingPurchase, DatabaseMember, DatabaseRole, DatabaseSummary, MarketCreateListingRequest, MarketListing, MarketUpdateListingRequest } from "@/lib/types"; import { deleteDatabaseAuthenticated, getCyclesBillingConfig, @@ -20,6 +20,12 @@ import { listDatabaseMembersPublic, listDatabasesAuthenticated, listDatabasesPublic, + marketCountActiveEntitlements, + marketCreateListing, + marketListDatabaseListings, + marketPauseListing, + marketPublishListing, + marketUpdateListing, renameDatabaseAuthenticated, revokeDatabaseAccessAuthenticated } from "@/lib/vfs-client"; @@ -56,6 +62,10 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) const [pendingPurchases, setPendingPurchases] = useState([]); const [pendingPurchasesError, setPendingPurchasesError] = useState(null); const [pendingPurchasesLoading, setPendingPurchasesLoading] = useState(false); + const [marketListings, setMarketListings] = useState([]); + const [marketError, setMarketError] = useState(null); + const [activeEntitlementCount, setActiveEntitlementCount] = useState(null); + const [marketBusy, setMarketBusy] = useState(false); const database = useMemo(() => databases.find((item) => item.databaseId === databaseId) ?? null, [databaseId, databases]); const isActiveDatabase = database?.status === "active"; @@ -87,6 +97,9 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) setDatabases([]); setCyclesBillingConfig(null); setMembers([]); + setMarketListings([]); + setMarketError(null); + setActiveEntitlementCount(null); setError(null); setWarning(null); setMemberError(null); @@ -120,6 +133,9 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) setDatabases(nextDatabases); setCyclesBillingConfig(cyclesResult.status === "fulfilled" ? cyclesResult.value : null); setMembers([]); + setMarketListings([]); + setMarketError(null); + setActiveEntitlementCount(null); if (publicResult.status === "rejected") { setWarning(`Public database list unavailable: ${errorMessage(publicResult.reason)}`); } @@ -128,13 +144,26 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) } if (identity && nextDatabase?.role === "owner") { if (nextDatabase.status === "active") { - try { - const nextMembers = await listDatabaseMembersAuthenticated(canisterId, identity, nextDatabaseId); - if (!isCurrentRefresh()) return; - setMembers(nextMembers); - } catch (cause) { - if (!isCurrentRefresh()) return; - setMemberError(errorMessage(cause)); + const [membersResult, listingsResult, entitlementCountResult] = await Promise.allSettled([ + listDatabaseMembersAuthenticated(canisterId, identity, nextDatabaseId), + marketListDatabaseListings(canisterId, identity, nextDatabaseId), + marketCountActiveEntitlements(canisterId, identity, nextDatabaseId) + ]); + if (!isCurrentRefresh()) return; + if (membersResult.status === "fulfilled") { + setMembers(membersResult.value); + } else { + setMemberError(errorMessage(membersResult.reason)); + } + if (listingsResult.status === "fulfilled") { + setMarketListings(listingsResult.value); + } else { + setMarketError(errorMessage(listingsResult.reason)); + } + if (entitlementCountResult.status === "fulfilled") { + setActiveEntitlementCount(entitlementCountResult.value); + } else { + setMarketError((current) => [current, errorMessage(entitlementCountResult.reason)].filter(Boolean).join("; ")); } } } else if (nextDatabase?.publicReadable && nextDatabase.status === "active") { @@ -260,6 +289,9 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) setDatabases([]); setCyclesBillingConfig(null); setMembers([]); + setMarketListings([]); + setMarketError(null); + setActiveEntitlementCount(null); setError(null); setWarning(null); setMemberError(null); @@ -363,6 +395,70 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) } } + async function createMarketListing(request: MarketCreateListingRequest) { + if (!authClient || !databaseId) return; + setMarketBusy(true); + setActionMessage(null); + try { + await marketCreateListing(canisterId, authClient.getIdentity(), request); + setActionTone("info"); + setActionMessage("Listing created."); + await refresh(authClient, databaseId); + } catch (cause) { + setMarketError(errorMessage(cause)); + } finally { + setMarketBusy(false); + } + } + + async function updateMarketListing(request: MarketUpdateListingRequest) { + if (!authClient || !databaseId) return; + setMarketBusy(true); + setActionMessage(null); + try { + await marketUpdateListing(canisterId, authClient.getIdentity(), request); + setActionTone("info"); + setActionMessage("Listing updated."); + await refresh(authClient, databaseId); + } catch (cause) { + setMarketError(errorMessage(cause)); + } finally { + setMarketBusy(false); + } + } + + async function publishMarketListing(listingId: string) { + if (!authClient || !databaseId) return; + setMarketBusy(true); + setActionMessage(null); + try { + await marketPublishListing(canisterId, authClient.getIdentity(), listingId); + setActionTone("info"); + setActionMessage("Listing published."); + await refresh(authClient, databaseId); + } catch (cause) { + setMarketError(errorMessage(cause)); + } finally { + setMarketBusy(false); + } + } + + async function pauseMarketListing(listingId: string) { + if (!authClient || !databaseId) return; + setMarketBusy(true); + setActionMessage(null); + try { + await marketPauseListing(canisterId, authClient.getIdentity(), listingId); + setActionTone("info"); + setActionMessage("Listing paused."); + await refresh(authClient, databaseId); + } catch (cause) { + setMarketError(errorMessage(cause)); + } finally { + setMarketBusy(false); + } + } + return (
@@ -430,11 +526,19 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) busyAction={busyAction} databaseId={databaseId} databaseName={database.name} + activeEntitlementCount={activeEntitlementCount} + marketBusy={marketBusy} + marketError={marketError} + marketListings={marketListings} members={members} principal={principal ?? "anonymous"} + onCreateListing={createMarketListing} onDelete={deleteDatabase} onGrant={grantAccess} + onPauseListing={pauseMarketListing} + onPublishListing={publishMarketListing} onRevoke={revokeAccess} + onUpdateListing={updateMarketListing} /> ) : database.publicReadable ? ( diff --git a/wikibrowser/app/dashboard/dashboard-home-client.tsx b/wikibrowser/app/dashboard/dashboard-home-client.tsx index 661b445f..8109f108 100644 --- a/wikibrowser/app/dashboard/dashboard-home-client.tsx +++ b/wikibrowser/app/dashboard/dashboard-home-client.tsx @@ -9,13 +9,14 @@ import { CreateDatabaseDialog } from "../create-database-dialog"; import { DatabaseBody, OfficialKinicWikiPanel, StatusPanel } from "../home-ui"; import { KINIC_LEDGER_FEE_E8S } from "@/lib/cycles"; import { parseKinicAmountE8sInput } from "@/lib/cycles-url"; -import { purchaseCyclesWithOisy, purchaseCyclesWithPlug } from "@/lib/cycles-wallet"; +import { purchaseCyclesWithOisy, purchaseCyclesWithPlug } from "@/lib/kinic-wallet"; import { formatTokenAmountFromE8s } from "@/lib/kinic-amount"; import type { CyclesBillingConfig, DatabaseSummary } from "@/lib/types"; import { createDatabaseAuthenticated, getCyclesBillingConfig, listDatabasesAuthenticated, listDatabasesPublic } from "@/lib/vfs-client"; import type { DatabaseRow } from "../home-ui"; type LoadState = "idle" | "loading" | "ready" | "error"; +type FundingProvider = "oisy" | "plug" | "ii"; const CREATE_DATABASE_PURCHASE_KINIC = "1"; @@ -150,13 +151,13 @@ export function DashboardHomeClient() { setCreateDialogOpen(false); setNewDatabaseName(""); const paymentAmountE8s = createDatabasePurchaseAmountE8s(); - setWalletMessage(`Database created pending. Requesting ${walletLabel(wallet.provider)} approval for ${formatTokenAmountFromE8s(paymentAmountE8s)}.`); + setWalletMessage(`Database created pending. Requesting ${fundingProviderLabel(wallet.provider)} approval for ${formatTokenAmountFromE8s(paymentAmountE8s)}.`); const purchaseResult = wallet.provider === "oisy" ? await purchaseCyclesWithOisy({ canisterId, databaseId: result.database_id, paymentAmountE8s }, wallet.connection) : await purchaseCyclesWithPlug({ canisterId, databaseId: result.database_id, paymentAmountE8s }, wallet.connection); setWalletMessage( - `${walletLabel(wallet.provider)} purchased cycles ${purchaseResult.purchasedCycles}; paid ${formatTokenAmountFromE8s(purchaseResult.paymentAmountE8s)}; database activation can complete.` + `${fundingProviderLabel(wallet.provider)} purchased cycles ${purchaseResult.purchasedCycles}; paid ${formatTokenAmountFromE8s(purchaseResult.paymentAmountE8s)}; database activation can complete.` ); await refreshWalletBalance(wallet); await refreshDatabases(authClient); @@ -282,8 +283,10 @@ function createDatabaseRequiredBalanceE8s(): bigint { return createDatabasePurchaseAmountE8s() + KINIC_LEDGER_FEE_E8S * 2n; } -function walletLabel(provider: "oisy" | "plug"): string { - return provider === "oisy" ? "OISY" : "Plug"; +function fundingProviderLabel(provider: FundingProvider): string { + if (provider === "oisy") return "OISY"; + if (provider === "plug") return "Plug"; + return "Internet Identity"; } function walletCanFundCreate(balanceE8s: string | null): boolean { @@ -319,10 +322,15 @@ function dashboardFundingSuccessMessage(params: { get(name: string): string | nu const kinic = params.get("kinic") ?? ""; const cycles = params.get("cycles") ?? ""; if (!/^[a-zA-Z0-9_-]+$/.test(databaseId)) return null; - if (provider !== "oisy" && provider !== "plug") return null; + if (!isFundingProvider(provider)) return null; if (!/^(?:<0\.001|[0-9]+\.[0-9]{3}) KINIC$/.test(kinic)) return null; if (!/^(?:[0-9]+|[0-9]{1,3}(?:,[0-9]{3})+)$/.test(cycles)) return null; - return `${walletLabel(provider)} purchased ${cycles} cycles for ${databaseId}; paid ${kinic}.`; + const verb = provider === "ii" ? "funded" : "purchased"; + return `${fundingProviderLabel(provider)} ${verb} ${cycles} cycles for ${databaseId}; paid ${kinic}.`; +} + +function isFundingProvider(provider: string): provider is FundingProvider { + return provider === "oisy" || provider === "plug" || provider === "ii"; } function errorMessage(cause: unknown): string { diff --git a/wikibrowser/app/dashboard/dashboard-ui.tsx b/wikibrowser/app/dashboard/dashboard-ui.tsx index 311369a0..7762c1cd 100644 --- a/wikibrowser/app/dashboard/dashboard-ui.tsx +++ b/wikibrowser/app/dashboard/dashboard-ui.tsx @@ -11,7 +11,7 @@ import { MemberTable } from "./member-table"; import { formatRawCycles } from "@/lib/cycles"; import { databaseCyclesView, databaseCyclesHref } from "@/lib/cycles-state"; import { formatTokenAmountFromE8s } from "@/lib/kinic-amount"; -import type { CyclesBillingConfig, DatabaseCycleEntry, DatabaseCyclesPendingPurchase, DatabaseMember, DatabaseRole, DatabaseSummary } from "@/lib/types"; +import type { CyclesBillingConfig, DatabaseCycleEntry, DatabaseCyclesPendingPurchase, DatabaseMember, DatabaseRole, DatabaseSummary, MarketCreateListingRequest, MarketListing, MarketUpdateListingRequest } from "@/lib/types"; import { isRoutableDatabaseId, publicDatabasePath, xShareDatabaseHref } from "@/lib/share-links"; type PendingAclAction = { @@ -55,7 +55,7 @@ export function SummaryPanel({ }) { const routable = isRoutableDatabaseId(databaseId); const active = database?.status === "active"; - const openHref = active && routable ? (publicReadable ? publicDatabasePath(databaseId) : `/${encodeURIComponent(databaseId)}/Wiki`) : null; + const openHref = active && routable ? publicDatabasePath(databaseId) : null; const cycles = databaseCyclesView(database, cyclesConfig); const purchaseHref = database ? databaseCyclesHref(database) : null; return ( @@ -97,7 +97,7 @@ export function PendingDatabasePanel(props: {

Reserved database

This database is reserved until the first cycle purchase completes. VFS, skills, and member management are available after activation.

- + ); } @@ -108,11 +108,19 @@ export function OwnerPanel(props: { busyAction: BusyAction | null; databaseId: string; databaseName: string; + activeEntitlementCount: string | null; + marketBusy: boolean; + marketError: string | null; + marketListings: MarketListing[]; members: DatabaseMember[]; principal: string; + onCreateListing: (request: MarketCreateListingRequest) => void; onDelete: () => Promise; onGrant: (principalText: string, role: DatabaseRole) => void; + onPauseListing: (listingId: string) => void; + onPublishListing: (listingId: string) => void; onRevoke: (principalText: string) => void; + onUpdateListing: (request: MarketUpdateListingRequest) => void; }) { const [pendingAction, setPendingAction] = useState(null); const publicMember = props.members.find((member) => member.principal === ANONYMOUS_PRINCIPAL); @@ -242,7 +250,19 @@ export function OwnerPanel(props: { {pendingAction ? setPendingAction(null)} onConfirm={confirmPendingAction} /> : null} + void; + onPause: (listingId: string) => void; + onPublish: (listingId: string) => void; + onUpdate: (request: MarketUpdateListingRequest) => void; +}) { + const [selectedListingId, setSelectedListingId] = useState(""); + const [title, setTitle] = useState(props.databaseName); + const [description, setDescription] = useState(""); + const [price, setPrice] = useState("1"); + const [tags, setTags] = useState(""); + const selected = props.listings.find((listing) => listing.listingId === selectedListingId) ?? null; + const priceE8s = parseKinicInput(price); + const submitDisabled = props.busy || !title.trim() || !description.trim() || !priceE8s; + + function selectListing(listing: MarketListing) { + setSelectedListingId(listing.listingId); + setTitle(listing.title); + setDescription(listing.description); + setPrice(decimalFromE8s(listing.priceE8s)); + setTags(tagsFromJson(listing.tagsJson)); + } + + function submit(event: FormEvent) { + event.preventDefault(); + if (submitDisabled || !priceE8s) return; + const base = { + title: title.trim(), + description: description.trim(), + llmSummary: null, + summarySnapshotRevision: null, + sampleExcerptsJson: "[]", + tagsJson: tagsJsonFromInput(tags), + priceE8s + }; + if (selected) { + props.onUpdate({ + ...base, + listingId: selected.listingId, + expectedRevision: selected.revision + }); + } else { + props.onCreate({ + ...base, + databaseId: props.databaseId + }); + } + } + + return ( +
+
+
+

Marketplace

+

DB owner can sell paid reader access.

+
+ + Marketplace + +
+ {props.error ?

{props.error}

: null} + {props.listings.length ? ( +
+ {props.listings.map((listing) => ( +
+
+

{listing.title}

+

+ {listing.status} / {formatTokenAmountFromE8s(listing.priceE8s)} / {listing.purchaseCount} purchases +

+
+
+ selectListing(listing)} variant="secondary"> + Edit + + {listing.status === "Active" ? ( + props.onPause(listing.listingId)} variant="secondary"> + Pause + + ) : ( + props.onPublish(listing.listingId)} variant="primary"> + Publish + + )} +
+
+ ))} +
+ ) : null} + +
+ + +
+