diff --git a/.gitignore b/.gitignore index f9486877..52c89d98 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ node_modules .env.development.local .env.test.local .env.production.local +.claude/ npm-debug.log* yarn-debug.log* diff --git a/client/swagger/models/deal_list_deal_request.go b/client/swagger/models/deal_list_deal_request.go index 0bbad9ab..ef88917f 100644 --- a/client/swagger/models/deal_list_deal_request.go +++ b/client/swagger/models/deal_list_deal_request.go @@ -20,6 +20,9 @@ import ( // swagger:model deal.ListDealRequest type DealListDealRequest struct { + // deal type filter (market for f05, pdp for f41) + DealTypes []ModelDealType `json:"dealTypes"` + // preparation ID or name filter Preparations []string `json:"preparations"` @@ -40,6 +43,10 @@ type DealListDealRequest struct { func (m *DealListDealRequest) Validate(formats strfmt.Registry) error { var res []error + if err := m.validateDealTypes(formats); err != nil { + res = append(res, err) + } + if err := m.validateStates(formats); err != nil { res = append(res, err) } @@ -50,6 +57,31 @@ func (m *DealListDealRequest) Validate(formats strfmt.Registry) error { return nil } +func (m *DealListDealRequest) validateDealTypes(formats strfmt.Registry) error { + if swag.IsZero(m.DealTypes) { // not required + return nil + } + + for i := 0; i < len(m.DealTypes); i++ { + + if err := m.DealTypes[i].Validate(formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("dealTypes" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("dealTypes" + "." + strconv.Itoa(i)) + } + + return err + } + + } + + return nil +} + func (m *DealListDealRequest) validateStates(formats strfmt.Registry) error { if swag.IsZero(m.States) { // not required return nil @@ -79,6 +111,10 @@ func (m *DealListDealRequest) validateStates(formats strfmt.Registry) error { func (m *DealListDealRequest) ContextValidate(ctx context.Context, formats strfmt.Registry) error { var res []error + if err := m.contextValidateDealTypes(ctx, formats); err != nil { + res = append(res, err) + } + if err := m.contextValidateStates(ctx, formats); err != nil { res = append(res, err) } @@ -89,6 +125,32 @@ func (m *DealListDealRequest) ContextValidate(ctx context.Context, formats strfm return nil } +func (m *DealListDealRequest) contextValidateDealTypes(ctx context.Context, formats strfmt.Registry) error { + + for i := 0; i < len(m.DealTypes); i++ { + + if swag.IsZero(m.DealTypes[i]) { // not required + return nil + } + + if err := m.DealTypes[i].ContextValidate(ctx, formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("dealTypes" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("dealTypes" + "." + strconv.Itoa(i)) + } + + return err + } + + } + + return nil +} + func (m *DealListDealRequest) contextValidateStates(ctx context.Context, formats strfmt.Registry) error { for i := 0; i < len(m.States); i++ { diff --git a/client/swagger/models/model_deal.go b/client/swagger/models/model_deal.go index 252591e9..74f3bcb2 100644 --- a/client/swagger/models/model_deal.go +++ b/client/swagger/models/model_deal.go @@ -28,6 +28,9 @@ type ModelDeal struct { // deal Id DealID int64 `json:"dealId,omitempty"` + // deal type + DealType ModelDealType `json:"dealType,omitempty"` + // end epoch EndEpoch int64 `json:"endEpoch,omitempty"` @@ -43,6 +46,9 @@ type ModelDeal struct { // LastVerifiedAt is the last time the deal was verified as active by the tracker LastVerifiedAt string `json:"lastVerifiedAt,omitempty"` + // NextChallengeEpoch is the next epoch when a challenge proof is due + NextChallengeEpoch int64 `json:"nextChallengeEpoch,omitempty"` + // piece cid PieceCid string `json:"pieceCid,omitempty"` @@ -52,6 +58,12 @@ type ModelDeal struct { // price Price string `json:"price,omitempty"` + // PDP-specific fields (only populated for DealTypePDP) + ProofSetID int64 `json:"proofSetId,omitempty"` + + // ProofSetLive indicates if the proof set is live (actively being challenged) + ProofSetLive bool `json:"proofSetLive,omitempty"` + // proposal Id ProposalID string `json:"proposalId,omitempty"` @@ -81,6 +93,10 @@ type ModelDeal struct { func (m *ModelDeal) Validate(formats strfmt.Registry) error { var res []error + if err := m.validateDealType(formats); err != nil { + res = append(res, err) + } + if err := m.validateState(formats); err != nil { res = append(res, err) } @@ -91,6 +107,27 @@ func (m *ModelDeal) Validate(formats strfmt.Registry) error { return nil } +func (m *ModelDeal) validateDealType(formats strfmt.Registry) error { + if swag.IsZero(m.DealType) { // not required + return nil + } + + if err := m.DealType.Validate(formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("dealType") + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("dealType") + } + + return err + } + + return nil +} + func (m *ModelDeal) validateState(formats strfmt.Registry) error { if swag.IsZero(m.State) { // not required return nil @@ -116,6 +153,10 @@ func (m *ModelDeal) validateState(formats strfmt.Registry) error { func (m *ModelDeal) ContextValidate(ctx context.Context, formats strfmt.Registry) error { var res []error + if err := m.contextValidateDealType(ctx, formats); err != nil { + res = append(res, err) + } + if err := m.contextValidateState(ctx, formats); err != nil { res = append(res, err) } @@ -126,6 +167,28 @@ func (m *ModelDeal) ContextValidate(ctx context.Context, formats strfmt.Registry return nil } +func (m *ModelDeal) contextValidateDealType(ctx context.Context, formats strfmt.Registry) error { + + if swag.IsZero(m.DealType) { // not required + return nil + } + + if err := m.DealType.ContextValidate(ctx, formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("dealType") + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("dealType") + } + + return err + } + + return nil +} + func (m *ModelDeal) contextValidateState(ctx context.Context, formats strfmt.Registry) error { if swag.IsZero(m.State) { // not required diff --git a/client/swagger/models/model_deal_type.go b/client/swagger/models/model_deal_type.go new file mode 100644 index 00000000..5d375164 --- /dev/null +++ b/client/swagger/models/model_deal_type.go @@ -0,0 +1,78 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "encoding/json" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/validate" +) + +// ModelDealType model deal type +// +// swagger:model model.DealType +type ModelDealType string + +func NewModelDealType(value ModelDealType) *ModelDealType { + return &value +} + +// Pointer returns a pointer to a freshly-allocated ModelDealType. +func (m ModelDealType) Pointer() *ModelDealType { + return &m +} + +const ( + + // ModelDealTypeMarket captures enum value "market" + ModelDealTypeMarket ModelDealType = "market" + + // ModelDealTypePdp captures enum value "pdp" + ModelDealTypePdp ModelDealType = "pdp" +) + +// for schema +var modelDealTypeEnum []any + +func init() { + var res []ModelDealType + if err := json.Unmarshal([]byte(`["market","pdp"]`), &res); err != nil { + panic(err) + } + for _, v := range res { + modelDealTypeEnum = append(modelDealTypeEnum, v) + } +} + +func (m ModelDealType) validateModelDealTypeEnum(path, location string, value ModelDealType) error { + if err := validate.EnumCase(path, location, value, modelDealTypeEnum, true); err != nil { + return err + } + return nil +} + +// Validate validates this model deal type +func (m ModelDealType) Validate(formats strfmt.Registry) error { + var res []error + + // value enum + if err := m.validateModelDealTypeEnum("", "body", m); err != nil { + return err + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +// ContextValidate validates this model deal type based on context it is used +func (m ModelDealType) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} diff --git a/cmd/app.go b/cmd/app.go index 40165856..70eae983 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -154,6 +154,7 @@ Upgrading: run.DatasetWorkerCmd, run.ContentProviderCmd, run.DealTrackerCmd, + run.PDPTrackerCmd, run.DealPusherCmd, run.DownloadServerCmd, }, diff --git a/cmd/deal/list.go b/cmd/deal/list.go index 72e33e5b..394717eb 100644 --- a/cmd/deal/list.go +++ b/cmd/deal/list.go @@ -34,6 +34,10 @@ var ListCmd = &cli.Command{ Name: "state", Usage: "Filter deals by state: proposed, published, active, expired, proposal_expired, slashed", }, + &cli.StringSliceFlag{ + Name: "deal-type", + Usage: "Filter deals by type: market (legacy f05), pdp (f41 PDP deals)", + }, }, Action: func(c *cli.Context) error { db, closer, err := database.OpenFromCLI(c) @@ -47,6 +51,7 @@ var ListCmd = &cli.Command{ Schedules: underscore.Map(c.IntSlice("schedules"), func(i int) uint32 { return uint32(i) }), Providers: c.StringSlice("provider"), States: underscore.Map(c.StringSlice("state"), func(s string) model.DealState { return model.DealState(s) }), + DealTypes: underscore.Map(c.StringSlice("deal-type"), func(s string) model.DealType { return model.DealType(s) }), }) if err != nil { return errors.WithStack(err) diff --git a/cmd/run/pdptracker.go b/cmd/run/pdptracker.go new file mode 100644 index 00000000..8b60b5be --- /dev/null +++ b/cmd/run/pdptracker.go @@ -0,0 +1,61 @@ +package run + +import ( + "time" + + "github.com/data-preservation-programs/singularity/database" + "github.com/data-preservation-programs/singularity/service" + "github.com/data-preservation-programs/singularity/service/pdptracker" + "github.com/urfave/cli/v2" +) + +var PDPTrackerCmd = &cli.Command{ + Name: "pdp-tracker", + Usage: "Start a PDP deal tracker that tracks f41 PDP deals for all relevant wallets", + Description: `The PDP tracker monitors Proof of Data Possession (PDP) deals on the Filecoin network. +Unlike legacy f05 market deals, PDP deals use proof sets managed through the PDPVerifier contract +where data is verified through cryptographic challenges. + +This tracker: +- Monitors proof sets for tracked wallets +- Updates deal status based on on-chain proof set state +- Tracks challenge epochs and live status + +Note: Full functionality requires the go-synapse library integration. +See: https://github.com/data-preservation-programs/go-synapse`, + Flags: []cli.Flag{ + &cli.DurationFlag{ + Name: "interval", + Usage: "How often to check for PDP deal updates", + Value: 10 * time.Minute, + }, + &cli.StringFlag{ + Name: "lotus-api", + Usage: "Lotus RPC API endpoint", + EnvVars: []string{"LOTUS_API"}, + }, + }, + Action: func(c *cli.Context) error { + db, closer, err := database.OpenFromCLI(c) + if err != nil { + return err + } + defer closer.Close() + + pdpClient, err := pdptracker.NewPDPClient(c.Context, c.String("lotus-api")) + if err != nil { + return err + } + defer pdpClient.Close() + + tracker := pdptracker.NewPDPTracker( + db, + c.Duration("interval"), + c.String("lotus-api"), + pdpClient, + false, + ) + + return service.StartServers(c.Context, pdptracker.Logger, &tracker) + }, +} diff --git a/docs/en/SUMMARY.md b/docs/en/SUMMARY.md index cd214212..e1668d73 100644 --- a/docs/en/SUMMARY.md +++ b/docs/en/SUMMARY.md @@ -57,6 +57,7 @@ * [Dataset Worker](cli-reference/run/dataset-worker.md) * [Content Provider](cli-reference/run/content-provider.md) * [Deal Tracker](cli-reference/run/deal-tracker.md) + * [Pdp Tracker](cli-reference/run/pdp-tracker.md) * [Deal Pusher](cli-reference/run/deal-pusher.md) * [Download Server](cli-reference/run/download-server.md) * [Wallet](cli-reference/wallet/README.md) diff --git a/docs/en/cli-reference/deal/list.md b/docs/en/cli-reference/deal/list.md index d72a23c8..53a6f999 100644 --- a/docs/en/cli-reference/deal/list.md +++ b/docs/en/cli-reference/deal/list.md @@ -14,6 +14,7 @@ OPTIONS: --schedule value [ --schedule value ] Filter deals by schedule --provider value [ --provider value ] Filter deals by provider --state value [ --state value ] Filter deals by state: proposed, published, active, expired, proposal_expired, slashed + --deal-type value [ --deal-type value ] Filter deals by type: market (legacy f05), pdp (f41 PDP deals) --help, -h show help ``` {% endcode %} diff --git a/docs/en/cli-reference/run/README.md b/docs/en/cli-reference/run/README.md index 5ef815a4..64452e5d 100644 --- a/docs/en/cli-reference/run/README.md +++ b/docs/en/cli-reference/run/README.md @@ -13,6 +13,7 @@ COMMANDS: dataset-worker Start a dataset preparation worker to process dataset scanning and preparation tasks content-provider Start a content provider that serves retrieval requests deal-tracker Start a deal tracker that tracks the deal for all relevant wallets + pdp-tracker Start a PDP deal tracker that tracks f41 PDP deals for all relevant wallets deal-pusher Start a deal pusher that monitors deal schedules and pushes deals to storage providers download-server An HTTP server connecting to remote metadata API to offer CAR file downloads help, h Shows a list of commands or help for one command diff --git a/docs/en/cli-reference/run/pdp-tracker.md b/docs/en/cli-reference/run/pdp-tracker.md new file mode 100644 index 00000000..4e7f0b20 --- /dev/null +++ b/docs/en/cli-reference/run/pdp-tracker.md @@ -0,0 +1,29 @@ +# Start a PDP deal tracker that tracks f41 PDP deals for all relevant wallets + +{% code fullWidth="true" %} +``` +NAME: + singularity run pdp-tracker - Start a PDP deal tracker that tracks f41 PDP deals for all relevant wallets + +USAGE: + singularity run pdp-tracker [command options] + +DESCRIPTION: + The PDP tracker monitors Proof of Data Possession (PDP) deals on the Filecoin network. + Unlike legacy f05 market deals, PDP deals use proof sets managed through the PDPVerifier contract + where data is verified through cryptographic challenges. + + This tracker: + - Monitors proof sets for tracked wallets + - Updates deal status based on on-chain proof set state + - Tracks challenge epochs and live status + + Note: Full functionality requires the go-synapse library integration. + See: https://github.com/data-preservation-programs/go-synapse + +OPTIONS: + --interval value How often to check for PDP deal updates (default: 10m0s) + --lotus-api value Lotus RPC API endpoint [$LOTUS_API] + --help, -h show help +``` +{% endcode %} diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 846b98b5..364b2c6f 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -6162,6 +6162,13 @@ const docTemplate = `{ "deal.ListDealRequest": { "type": "object", "properties": { + "dealTypes": { + "description": "deal type filter (market for f05, pdp for f41)", + "type": "array", + "items": { + "$ref": "#/definitions/model.DealType" + } + }, "preparations": { "description": "preparation ID or name filter", "type": "array", @@ -6489,6 +6496,9 @@ const docTemplate = `{ "dealId": { "type": "integer" }, + "dealType": { + "$ref": "#/definitions/model.DealType" + }, "endEpoch": { "type": "integer" }, @@ -6505,6 +6515,10 @@ const docTemplate = `{ "description": "LastVerifiedAt is the last time the deal was verified as active by the tracker", "type": "string" }, + "nextChallengeEpoch": { + "description": "NextChallengeEpoch is the next epoch when a challenge proof is due", + "type": "integer" + }, "pieceCid": { "type": "string" }, @@ -6514,6 +6528,14 @@ const docTemplate = `{ "price": { "type": "string" }, + "proofSetId": { + "description": "PDP-specific fields (only populated for DealTypePDP)", + "type": "integer" + }, + "proofSetLive": { + "description": "ProofSetLive indicates if the proof set is live (actively being challenged)", + "type": "boolean" + }, "proposalId": { "type": "string" }, @@ -6564,6 +6586,17 @@ const docTemplate = `{ "DealErrored" ] }, + "model.DealType": { + "type": "string", + "enum": [ + "market", + "pdp" + ], + "x-enum-varnames": [ + "DealTypeMarket", + "DealTypePDP" + ] + }, "model.File": { "type": "object", "properties": { diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 0d0e6e31..2c27e1b6 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -6155,6 +6155,13 @@ "deal.ListDealRequest": { "type": "object", "properties": { + "dealTypes": { + "description": "deal type filter (market for f05, pdp for f41)", + "type": "array", + "items": { + "$ref": "#/definitions/model.DealType" + } + }, "preparations": { "description": "preparation ID or name filter", "type": "array", @@ -6482,6 +6489,9 @@ "dealId": { "type": "integer" }, + "dealType": { + "$ref": "#/definitions/model.DealType" + }, "endEpoch": { "type": "integer" }, @@ -6498,6 +6508,10 @@ "description": "LastVerifiedAt is the last time the deal was verified as active by the tracker", "type": "string" }, + "nextChallengeEpoch": { + "description": "NextChallengeEpoch is the next epoch when a challenge proof is due", + "type": "integer" + }, "pieceCid": { "type": "string" }, @@ -6507,6 +6521,14 @@ "price": { "type": "string" }, + "proofSetId": { + "description": "PDP-specific fields (only populated for DealTypePDP)", + "type": "integer" + }, + "proofSetLive": { + "description": "ProofSetLive indicates if the proof set is live (actively being challenged)", + "type": "boolean" + }, "proposalId": { "type": "string" }, @@ -6557,6 +6579,17 @@ "DealErrored" ] }, + "model.DealType": { + "type": "string", + "enum": [ + "market", + "pdp" + ], + "x-enum-varnames": [ + "DealTypeMarket", + "DealTypePDP" + ] + }, "model.File": { "type": "object", "properties": { diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index c99d16d4..d6c215da 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -151,6 +151,11 @@ definitions: type: object deal.ListDealRequest: properties: + dealTypes: + description: deal type filter (market for f05, pdp for f41) + items: + $ref: '#/definitions/model.DealType' + type: array preparations: description: preparation ID or name filter items: @@ -392,6 +397,8 @@ definitions: type: string dealId: type: integer + dealType: + $ref: '#/definitions/model.DealType' endEpoch: type: integer errorMessage: @@ -404,12 +411,23 @@ definitions: description: LastVerifiedAt is the last time the deal was verified as active by the tracker type: string + nextChallengeEpoch: + description: NextChallengeEpoch is the next epoch when a challenge proof is + due + type: integer pieceCid: type: string pieceSize: type: integer price: type: string + proofSetId: + description: PDP-specific fields (only populated for DealTypePDP) + type: integer + proofSetLive: + description: ProofSetLive indicates if the proof set is live (actively being + challenged) + type: boolean proposalId: type: string provider: @@ -448,6 +466,14 @@ definitions: - DealRejected - DealSlashed - DealErrored + model.DealType: + enum: + - market + - pdp + type: string + x-enum-varnames: + - DealTypeMarket + - DealTypePDP model.File: properties: attachmentId: diff --git a/go.mod b/go.mod index fc49ae75..4551c5e0 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/data-preservation-programs/singularity go 1.24.6 - require ( github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b github.com/avast/retry-go v3.0.0+incompatible @@ -100,6 +99,7 @@ require ( github.com/ProtonMail/go-crypto v1.3.0 // indirect github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect github.com/PuerkitoBio/goquery v1.10.3 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2 v1.39.4 // indirect @@ -124,6 +124,7 @@ require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bep/debounce v1.2.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/bradenaw/juniper v0.15.3 // indirect github.com/calebcase/tmpfile v1.0.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -134,15 +135,21 @@ require ( github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/colinmarc/hdfs/v2 v2.4.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/data-preservation-programs/go-synapse v0.0.0-20260206105716-b6a5e7e6808e // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/dchest/blake2b v1.0.0 // indirect + github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v28.5.1+incompatible // indirect @@ -154,6 +161,9 @@ require ( github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 // indirect github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethereum/go-ethereum v1.14.12 // indirect + github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/filecoin-project/filecoin-ffi v1.34.0 // indirect github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect @@ -161,6 +171,7 @@ require ( github.com/filecoin-project/go-bitfield v0.2.4 // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/filecoin-project/go-commp-utils v0.1.4 // indirect + github.com/filecoin-project/go-commp-utils/v2 v2.1.0 // indirect github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc8 // indirect github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect @@ -207,6 +218,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/holiman/uint256 v1.3.1 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect @@ -271,6 +283,7 @@ require ( github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -333,6 +346,7 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/samber/lo v1.52.0 // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shoenig/go-m1cpu v0.1.7 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -342,6 +356,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/supranational/blst v0.3.13 // indirect github.com/swaggo/files/v2 v2.0.0 // indirect github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect @@ -398,6 +413,7 @@ require ( modernc.org/memory v1.5.0 // indirect modernc.org/sqlite v1.21.1 // indirect moul.io/http2curl v1.0.0 // indirect + rsc.io/tmplfunc v0.0.3 // indirect storj.io/common v0.0.0-20251022143549-19bf6a9f274a // indirect storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect diff --git a/go.sum b/go.sum index afb571c3..c0cafd48 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiU github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 h1:hhdWprfSpFbN7lz3W1gM40vOgvSh1WCSMxYD6gGB4Hs= github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3/go.mod h1:XaUnRxSCYgL3kkgX0QHIV0D+znljPIDImxlv2kbGv0Y= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= @@ -164,6 +166,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo= github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -204,6 +208,10 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0= github.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -220,11 +228,17 @@ github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo= github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= +github.com/data-preservation-programs/go-synapse v0.0.0-20260206105716-b6a5e7e6808e h1:xqTd8FN2XAxKVwbkNmkSeemTFkuGp6eDMTvTDhumCxQ= +github.com/data-preservation-programs/go-synapse v0.0.0-20260206105716-b6a5e7e6808e/go.mod h1:5pXdfN2ywCsZK5gbhtmR0Nv6ttEAeUgHIq1gW3QCMPg= github.com/data-preservation-programs/table v0.0.3 h1:hboeauxPXybE8KlMA+RjDXz/J4xaG5CAFCcxyOm8yWo= github.com/data-preservation-programs/table v0.0.3/go.mod h1:sRGP/IuuqFc/y9QfmDyb5h6Q2wrnhhnBofEOj9aDRJg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -235,6 +249,8 @@ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s= github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s= +github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= +github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/chaincfg/chainhash v1.0.2/go.mod h1:BpbrGgrPTr3YJYRN3Bm+D9NuaFd+zGyNeIKgrhCXK60= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= @@ -278,6 +294,12 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.14.12 h1:8hl57x77HSUo+cXExrURjU/w1VhL+ShCTJrTwcCQSe4= +github.com/ethereum/go-ethereum v1.14.12/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY= +github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= +github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -309,6 +331,8 @@ github.com/filecoin-project/go-commp-utils v0.1.3/go.mod h1:3ENlD1pZySaUout0p9AN github.com/filecoin-project/go-commp-utils v0.1.4 h1:/WSsrAb0xupo+aRWRyD80lRUXAXJvYoTgDQS1pYZ1Mk= github.com/filecoin-project/go-commp-utils v0.1.4/go.mod h1:Sekocu5q9b4ECAUFu853GFUbm8I7upAluummHFe2kFo= github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837/go.mod h1:e2YBjSblNVoBckkbv3PPqsq71q98oFkFqL7s1etViGo= +github.com/filecoin-project/go-commp-utils/v2 v2.1.0 h1:KWNRalUp2bhN1SW7STsJS2AHs9mnfGKk9LnQgzDe+gI= +github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj/gysLNu26kYII83IV5iNrAO9iI= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0= github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI= @@ -401,6 +425,7 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -545,6 +570,7 @@ github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxV github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -601,6 +627,8 @@ github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0 github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= +github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= +github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -908,6 +936,9 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= @@ -1122,6 +1153,8 @@ github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRo github.com/sashabaranov/go-openai v1.14.1 h1:jqfkdj8XHnBF84oi2aNtT8Ktp3EJ0MfuVjvcMkfI0LA= github.com/sashabaranov/go-openai v1.14.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.1.7 h1:C76Yd0ObKR82W4vhfjZiCp0HxcSZ8Nqd84v+HZ0qyI0= @@ -1201,6 +1234,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= +github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/swaggo/echo-swagger v1.4.0 h1:RCxLKySw1SceHLqnmc41pKyiIeE+OiD7NSI7FUOBlLo= github.com/swaggo/echo-swagger v1.4.0/go.mod h1:Wh3VlwjZGZf/LH0s81tz916JokuPG7y/ZqaqnckYqoQ= github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw= @@ -1845,6 +1880,8 @@ moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHc rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= storj.io/common v0.0.0-20251022143549-19bf6a9f274a h1:g3stKez/XuRJ25RYRSUthdhG86DgFpBAvnOqHufHdDM= diff --git a/handler/deal/list.go b/handler/deal/list.go index 658b78cc..2977b79b 100644 --- a/handler/deal/list.go +++ b/handler/deal/list.go @@ -15,6 +15,7 @@ type ListDealRequest struct { Schedules []uint32 `json:"schedules"` // schedule id filter Providers []string `json:"providers"` // provider filter States []model.DealState `json:"states"` // state filter + DealTypes []model.DealType `json:"dealTypes"` // deal type filter (market for f05, pdp for f41) } // ListHandler retrieves a list of deals from the database based on the specified filtering criteria in ListDealRequest. @@ -84,6 +85,10 @@ func (DefaultHandler) ListHandler(ctx context.Context, db *gorm.DB, request List statement = statement.Where("state IN ?", request.States) } + if len(request.DealTypes) > 0 { + statement = statement.Where("deal_type IN ?", request.DealTypes) + } + // We did not create indexes for all above query and it should be fine for now err := db.Where(statement).Find(&deals).Error if err != nil { diff --git a/model/basetypes.go b/model/basetypes.go index 2341d85f..34d6647b 100644 --- a/model/basetypes.go +++ b/model/basetypes.go @@ -287,6 +287,7 @@ const ( DealTracker WorkerType = "deal_tracker" DealPusher WorkerType = "deal_pusher" DatasetWorker WorkerType = "dataset_worker" + PDPTracker WorkerType = "pdp_tracker" ) const ( diff --git a/model/migrate.go b/model/migrate.go index db56440a..c856bffc 100644 --- a/model/migrate.go +++ b/model/migrate.go @@ -117,6 +117,11 @@ func AutoMigrate(db *gorm.DB) error { return errors.Wrap(err, "failed to infer piece types") } + // Set deal_type for existing deals that predate the column + if err := inferDealTypes(db); err != nil { + return errors.Wrap(err, "failed to infer deal types") + } + return nil } @@ -306,6 +311,35 @@ func inferPieceTypes(db *gorm.DB) error { return nil } +// inferDealTypes sets deal_type for deals that predate the column. +// All existing deals are assumed to be legacy market deals (f05). +// This is idempotent - only updates rows where deal_type is NULL or empty. +func inferDealTypes(db *gorm.DB) error { + // check if any deals need updating + var count int64 + err := db.Raw(`SELECT COUNT(*) FROM deals WHERE deal_type IS NULL OR deal_type = ''`).Scan(&count).Error + if err != nil { + // table might not exist or column missing + logger.Debugw("skipping deal type inference", "error", err) + return nil + } + + if count == 0 { + return nil + } + + logger.Infow("setting deal type for legacy deals", "count", count) + + // All existing deals are legacy market deals + result := db.Exec(`UPDATE deals SET deal_type = 'market' WHERE deal_type IS NULL OR deal_type = ''`) + if result.Error != nil { + return errors.Wrap(result.Error, "failed to set deal types") + } + + logger.Infow("set deal types", "updated", result.RowsAffected) + return nil +} + // DropAll removes all tables specified in the Tables slice from the database. // // This function is typically used during development or testing where a clean database diff --git a/model/replication.go b/model/replication.go index 4bcada94..08150da6 100644 --- a/model/replication.go +++ b/model/replication.go @@ -10,6 +10,9 @@ type DealState string type ScheduleState string +// DealType represents the type of deal (legacy market vs PDP) +type DealType string + const ( DealProposed DealState = "proposed" DealPublished DealState = "published" @@ -21,6 +24,13 @@ const ( DealErrored DealState = "error" ) +const ( + // DealTypeMarket represents legacy f05 market actor deals + DealTypeMarket DealType = "market" + // DealTypePDP represents f41 PDP (Proof of Data Possession) deals + DealTypePDP DealType = "pdp" +) + var DealStateStrings = []string{ string(DealProposed), string(DealPublished), @@ -43,6 +53,16 @@ var DealStates = []DealState{ DealErrored, } +var DealTypeStrings = []string{ + string(DealTypeMarket), + string(DealTypePDP), +} + +var DealTypes = []DealType{ + DealTypeMarket, + DealTypePDP, +} + const ( ScheduleActive ScheduleState = "active" SchedulePaused ScheduleState = "paused" @@ -81,6 +101,7 @@ type Deal struct { LastVerifiedAt *time.Time `json:"lastVerifiedAt" table:"verbose;format:2006-01-02 15:04:05"` // LastVerifiedAt is the last time the deal was verified as active by the tracker DealID *uint64 `gorm:"unique" json:"dealId"` State DealState `gorm:"index:idx_pending" json:"state"` + DealType DealType `gorm:"index;default:'market'" json:"dealType"` Provider string `json:"provider"` ProposalID string `json:"proposalId" table:"verbose"` Label string `json:"label" table:"verbose"` @@ -93,6 +114,11 @@ type Deal struct { Verified bool `json:"verified"` ErrorMessage string `json:"errorMessage" table:"verbose"` + // PDP-specific fields (only populated for DealTypePDP) + ProofSetID *uint64 `json:"proofSetId,omitempty" table:"verbose"` // ProofSetID is the on-chain proof set ID for PDP deals + ProofSetLive *bool `json:"proofSetLive,omitempty" table:"verbose"` // ProofSetLive indicates if the proof set is live (actively being challenged) + NextChallengeEpoch *int32 `json:"nextChallengeEpoch,omitempty" table:"verbose"` // NextChallengeEpoch is the next epoch when a challenge proof is due + // Associations ScheduleID *ScheduleID `json:"scheduleId" table:"verbose"` Schedule *Schedule `gorm:"foreignKey:ScheduleID;constraint:OnDelete:SET NULL" json:"schedule,omitempty" swaggerignore:"true" table:"expand"` diff --git a/service/dealtracker/dealtracker.go b/service/dealtracker/dealtracker.go index 36a2714c..c878d27f 100644 --- a/service/dealtracker/dealtracker.go +++ b/service/dealtracker/dealtracker.go @@ -556,6 +556,7 @@ func (d *DealTracker) runOnce(ctx context.Context) error { return db.Create(&model.Deal{ DealID: &dealID, State: newState, + DealType: model.DealTypeMarket, // Legacy market deal (f05) ClientID: deal.Proposal.Client, Provider: deal.Proposal.Provider, Label: deal.Proposal.Label, diff --git a/service/pdptracker/pdpclient.go b/service/pdptracker/pdpclient.go new file mode 100644 index 00000000..d874fc5c --- /dev/null +++ b/service/pdptracker/pdpclient.go @@ -0,0 +1,309 @@ +package pdptracker + +import ( + "context" + "fmt" + "math" + "math/big" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/go-synapse" + "github.com/data-preservation-programs/go-synapse/constants" + "github.com/data-preservation-programs/go-synapse/contracts" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-varint" +) + +const pdpDefaultPageSize uint64 = 100 + +type activePiecesResult struct { + Pieces []contracts.CidsCid + HasMore bool +} + +type pdpVerifierAPI interface { + GetNextDataSetId(opts *bind.CallOpts) (uint64, error) + GetDataSetListener(opts *bind.CallOpts, setId *big.Int) (common.Address, error) + GetDataSetStorageProvider(opts *bind.CallOpts, setId *big.Int) (common.Address, common.Address, error) + DataSetLive(opts *bind.CallOpts, setId *big.Int) (bool, error) + GetNextChallengeEpoch(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) + GetActivePieces(opts *bind.CallOpts, setId *big.Int, offset *big.Int, limit *big.Int) (activePiecesResult, error) +} + +type pdpVerifierContract struct { + contract *contracts.PDPVerifier +} + +func (c pdpVerifierContract) GetNextDataSetId(opts *bind.CallOpts) (uint64, error) { + return c.contract.GetNextDataSetId(opts) +} + +func (c pdpVerifierContract) GetDataSetListener(opts *bind.CallOpts, setId *big.Int) (common.Address, error) { + return c.contract.GetDataSetListener(opts, setId) +} + +func (c pdpVerifierContract) GetDataSetStorageProvider(opts *bind.CallOpts, setId *big.Int) (common.Address, common.Address, error) { + return c.contract.GetDataSetStorageProvider(opts, setId) +} + +func (c pdpVerifierContract) DataSetLive(opts *bind.CallOpts, setId *big.Int) (bool, error) { + return c.contract.DataSetLive(opts, setId) +} + +func (c pdpVerifierContract) GetNextChallengeEpoch(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { + return c.contract.GetNextChallengeEpoch(opts, setId) +} + +func (c pdpVerifierContract) GetActivePieces(opts *bind.CallOpts, setId *big.Int, offset *big.Int, limit *big.Int) (activePiecesResult, error) { + result, err := c.contract.GetActivePieces(opts, setId, offset, limit) + if err != nil { + return activePiecesResult{}, err + } + return activePiecesResult{ + Pieces: result.Pieces, + HasMore: result.HasMore, + }, nil +} + +// ChainPDPClient implements PDPClient using the PDPVerifier contract on-chain. +type ChainPDPClient struct { + ethClient *ethclient.Client + contract pdpVerifierAPI + pageSize uint64 +} + +// NewPDPClient creates a new PDP client backed by the PDPVerifier contract. +func NewPDPClient(ctx context.Context, rpcURL string) (*ChainPDPClient, error) { + if rpcURL == "" { + return nil, errors.New("rpc URL is required") + } + + ethClient, err := ethclient.DialContext(ctx, rpcURL) + if err != nil { + return nil, errors.Wrap(err, "failed to connect to PDP RPC") + } + + network, _, err := synapse.DetectNetwork(ctx, ethClient) + if err != nil { + ethClient.Close() + return nil, errors.Wrap(err, "failed to detect PDP network") + } + + contractAddr := constants.GetPDPVerifierAddress(network) + if contractAddr == (common.Address{}) { + ethClient.Close() + return nil, errors.New("unsupported PDP network: missing contract address") + } + + verifier, err := contracts.NewPDPVerifier(contractAddr, ethClient) + if err != nil { + ethClient.Close() + return nil, errors.Wrap(err, "failed to initialize PDP verifier contract") + } + + return &ChainPDPClient{ + ethClient: ethClient, + contract: pdpVerifierContract{contract: verifier}, + pageSize: pdpDefaultPageSize, + }, nil +} + +// Close releases the underlying RPC client. +func (c *ChainPDPClient) Close() error { + if c.ethClient == nil { + return nil + } + c.ethClient.Close() + return nil +} + +// GetProofSetsForClient returns all proof sets associated with a client address. +func (c *ChainPDPClient) GetProofSetsForClient(ctx context.Context, clientAddress string) ([]ProofSetInfo, error) { + listenerAddr, err := parseDelegatedAddress(clientAddress) + if err != nil { + return nil, err + } + + nextID, err := c.contract.GetNextDataSetId(&bind.CallOpts{Context: ctx}) + if err != nil { + return nil, errors.Wrap(err, "failed to get next data set ID") + } + + var proofSets []ProofSetInfo + for setID := uint64(0); setID < nextID; setID++ { + setIDBig := new(big.Int).SetUint64(setID) + + listener, err := c.contract.GetDataSetListener(&bind.CallOpts{Context: ctx}, setIDBig) + if err != nil { + Logger.Debugw("failed to get PDP data set listener", "setID", setID, "error", err) + continue + } + if listener != listenerAddr { + continue + } + + info, err := c.buildProofSetInfo(ctx, setID, listener) + if err != nil { + Logger.Warnw("failed to build PDP proof set info", "setID", setID, "error", err) + continue + } + proofSets = append(proofSets, *info) + } + + return proofSets, nil +} + +// GetProofSetInfo returns detailed information about a specific proof set. +func (c *ChainPDPClient) GetProofSetInfo(ctx context.Context, proofSetID uint64) (*ProofSetInfo, error) { + listener, err := c.contract.GetDataSetListener(&bind.CallOpts{Context: ctx}, new(big.Int).SetUint64(proofSetID)) + if err != nil { + return nil, errors.Wrap(err, "failed to get PDP data set listener") + } + return c.buildProofSetInfo(ctx, proofSetID, listener) +} + +// IsProofSetLive checks if a proof set is actively being challenged. +func (c *ChainPDPClient) IsProofSetLive(ctx context.Context, proofSetID uint64) (bool, error) { + live, err := c.contract.DataSetLive(&bind.CallOpts{Context: ctx}, new(big.Int).SetUint64(proofSetID)) + if err != nil { + return false, errors.Wrap(err, "failed to check PDP data set live status") + } + return live, nil +} + +// GetNextChallengeEpoch returns the next challenge epoch for a proof set. +func (c *ChainPDPClient) GetNextChallengeEpoch(ctx context.Context, proofSetID uint64) (int32, error) { + epoch, err := c.contract.GetNextChallengeEpoch(&bind.CallOpts{Context: ctx}, new(big.Int).SetUint64(proofSetID)) + if err != nil { + return 0, errors.Wrap(err, "failed to get PDP next challenge epoch") + } + if !epoch.IsInt64() || epoch.Int64() > math.MaxInt32 { + return 0, fmt.Errorf("PDP next challenge epoch out of range: %s", epoch.String()) + } + return int32(epoch.Int64()), nil +} + +func (c *ChainPDPClient) buildProofSetInfo(ctx context.Context, setID uint64, listener common.Address) (*ProofSetInfo, error) { + setIDBig := new(big.Int).SetUint64(setID) + + storageProvider, _, err := c.contract.GetDataSetStorageProvider(&bind.CallOpts{Context: ctx}, setIDBig) + if err != nil { + return nil, errors.Wrap(err, "failed to get PDP data set storage provider") + } + + isLive, err := c.contract.DataSetLive(&bind.CallOpts{Context: ctx}, setIDBig) + if err != nil { + return nil, errors.Wrap(err, "failed to check PDP data set live status") + } + + nextChallenge, err := c.contract.GetNextChallengeEpoch(&bind.CallOpts{Context: ctx}, setIDBig) + if err != nil { + return nil, errors.Wrap(err, "failed to get PDP next challenge epoch") + } + if !nextChallenge.IsInt64() || nextChallenge.Int64() > math.MaxInt32 { + return nil, fmt.Errorf("PDP next challenge epoch out of range: %s", nextChallenge.String()) + } + + pieces, err := c.getPieceCIDs(ctx, setIDBig) + if err != nil { + return nil, errors.Wrap(err, "failed to get PDP active pieces") + } + + clientAddr, err := formatDelegatedAddress(listener.Bytes()) + if err != nil { + return nil, err + } + providerAddr, err := formatDelegatedAddress(storageProvider.Bytes()) + if err != nil { + return nil, err + } + + return &ProofSetInfo{ + ProofSetID: setID, + ClientAddress: clientAddr, + ProviderAddress: providerAddr, + IsLive: isLive, + NextChallengeEpoch: int32(nextChallenge.Int64()), + PieceCIDs: pieces, + }, nil +} + +func (c *ChainPDPClient) getPieceCIDs(ctx context.Context, setID *big.Int) ([]string, error) { + var ( + offset uint64 + result []string + ) + + for { + pieces, err := c.contract.GetActivePieces( + &bind.CallOpts{Context: ctx}, + setID, + new(big.Int).SetUint64(offset), + new(big.Int).SetUint64(c.pageSize), + ) + if err != nil { + return nil, err + } + + for i, piece := range pieces.Pieces { + parsed, err := cid.Cast(piece.Data) + if err != nil { + return nil, errors.Wrapf(err, "invalid piece CID at index %d", i) + } + result = append(result, parsed.String()) + } + + if !pieces.HasMore || len(pieces.Pieces) == 0 { + break + } + offset += uint64(len(pieces.Pieces)) + } + + return result, nil +} + +func parseDelegatedAddress(addressStr string) (common.Address, error) { + if addressStr == "" { + return common.Address{}, errors.New("client address is required") + } + if common.IsHexAddress(addressStr) { + return common.HexToAddress(addressStr), nil + } + + addr, err := address.NewFromString(addressStr) + if err != nil { + return common.Address{}, errors.Wrap(err, "failed to parse client address") + } + if addr.Protocol() != address.Delegated { + return common.Address{}, fmt.Errorf("client address must be delegated (f4), got protocol %d", addr.Protocol()) + } + + namespace, n, err := varint.FromUvarint(addr.Payload()) + if err != nil { + return common.Address{}, errors.Wrap(err, "failed to decode delegated namespace") + } + subaddr := addr.Payload()[n:] + if namespace != 10 { + return common.Address{}, fmt.Errorf("unsupported delegated namespace %d", namespace) + } + if len(subaddr) != common.AddressLength { + return common.Address{}, fmt.Errorf("invalid delegated address length: %d", len(subaddr)) + } + + return common.BytesToAddress(subaddr), nil +} + +func formatDelegatedAddress(subaddr []byte) (string, error) { + if len(subaddr) != common.AddressLength { + return "", fmt.Errorf("invalid delegated subaddress length: %d", len(subaddr)) + } + addr, err := address.NewDelegatedAddress(10, subaddr) + if err != nil { + return "", errors.Wrap(err, "failed to encode delegated address") + } + return addr.String(), nil +} diff --git a/service/pdptracker/pdpclient_test.go b/service/pdptracker/pdpclient_test.go new file mode 100644 index 00000000..4dcaff2d --- /dev/null +++ b/service/pdptracker/pdpclient_test.go @@ -0,0 +1,159 @@ +package pdptracker + +import ( + "context" + "errors" + "math/big" + "testing" + + "github.com/data-preservation-programs/go-synapse/contracts" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" +) + +type mockDataSet struct { + listener common.Address + provider common.Address + live bool + nextChallenge uint64 + pieces []cid.Cid +} + +type mockPDPVerifier struct { + dataSets map[uint64]*mockDataSet +} + +func (m *mockPDPVerifier) GetNextDataSetId(_ *bind.CallOpts) (uint64, error) { + var max uint64 + for id := range m.dataSets { + if id > max { + max = id + } + } + return max + 1, nil +} + +func (m *mockPDPVerifier) GetDataSetListener(_ *bind.CallOpts, setId *big.Int) (common.Address, error) { + data, ok := m.dataSets[setId.Uint64()] + if !ok { + return common.Address{}, errors.New("not found") + } + return data.listener, nil +} + +func (m *mockPDPVerifier) GetDataSetStorageProvider(_ *bind.CallOpts, setId *big.Int) (common.Address, common.Address, error) { + data, ok := m.dataSets[setId.Uint64()] + if !ok { + return common.Address{}, common.Address{}, errors.New("not found") + } + return data.provider, common.Address{}, nil +} + +func (m *mockPDPVerifier) DataSetLive(_ *bind.CallOpts, setId *big.Int) (bool, error) { + data, ok := m.dataSets[setId.Uint64()] + if !ok { + return false, errors.New("not found") + } + return data.live, nil +} + +func (m *mockPDPVerifier) GetNextChallengeEpoch(_ *bind.CallOpts, setId *big.Int) (*big.Int, error) { + data, ok := m.dataSets[setId.Uint64()] + if !ok { + return nil, errors.New("not found") + } + return new(big.Int).SetUint64(data.nextChallenge), nil +} + +func (m *mockPDPVerifier) GetActivePieces(_ *bind.CallOpts, setId *big.Int, offset *big.Int, limit *big.Int) (activePiecesResult, error) { + data, ok := m.dataSets[setId.Uint64()] + if !ok { + return activePiecesResult{}, errors.New("not found") + } + + start := int(offset.Uint64()) + if start >= len(data.pieces) { + return activePiecesResult{Pieces: nil, HasMore: false}, nil + } + end := start + int(limit.Uint64()) + if end > len(data.pieces) { + end = len(data.pieces) + } + + out := make([]contracts.CidsCid, 0, end-start) + for _, piece := range data.pieces[start:end] { + out = append(out, contracts.CidsCid{Data: piece.Bytes()}) + } + + return activePiecesResult{ + Pieces: out, + HasMore: end < len(data.pieces), + }, nil +} + +func TestChainPDPClient_GetProofSetsForClient(t *testing.T) { + address.CurrentNetwork = address.Mainnet + + listener := common.HexToAddress("0x1111111111111111111111111111111111111111") + provider := common.HexToAddress("0x2222222222222222222222222222222222222222") + + listenerAddr, err := address.NewDelegatedAddress(10, listener.Bytes()) + require.NoError(t, err) + providerAddr, err := address.NewDelegatedAddress(10, provider.Bytes()) + require.NoError(t, err) + + piece1, err := cid.Decode("baga6ea4seaqao7s73y24kcutaosvacpdjgfe5pw76ooefnyqw4ynr3d2y6x2mpq") + require.NoError(t, err) + piece2, err := cid.Decode("baga6ea4seaqgwm2a6rfh53y5a4qbm5zhqyixwut3wst6dfrlghm2f5l6t4o2mry") + require.NoError(t, err) + + mock := &mockPDPVerifier{ + dataSets: map[uint64]*mockDataSet{ + 1: { + listener: listener, + provider: provider, + live: true, + nextChallenge: 42, + pieces: []cid.Cid{piece1, piece2}, + }, + 2: { + listener: common.HexToAddress("0x3333333333333333333333333333333333333333"), + provider: provider, + live: false, + pieces: []cid.Cid{piece1}, + }, + }, + } + + client := &ChainPDPClient{ + contract: mock, + pageSize: 1, + } + + proofSets, err := client.GetProofSetsForClient(context.Background(), listenerAddr.String()) + require.NoError(t, err) + require.Len(t, proofSets, 1) + + proofSet := proofSets[0] + require.EqualValues(t, 1, proofSet.ProofSetID) + require.Equal(t, listenerAddr.String(), proofSet.ClientAddress) + require.Equal(t, providerAddr.String(), proofSet.ProviderAddress) + require.True(t, proofSet.IsLive) + require.EqualValues(t, 42, proofSet.NextChallengeEpoch) + require.Len(t, proofSet.PieceCIDs, 2) + require.Equal(t, piece1.String(), proofSet.PieceCIDs[0]) + require.Equal(t, piece2.String(), proofSet.PieceCIDs[1]) +} + +func TestChainPDPClient_GetProofSetsForClient_InvalidAddress(t *testing.T) { + client := &ChainPDPClient{ + contract: &mockPDPVerifier{dataSets: map[uint64]*mockDataSet{}}, + pageSize: 1, + } + + _, err := client.GetProofSetsForClient(context.Background(), "f0100") + require.Error(t, err) +} diff --git a/service/pdptracker/pdptracker.go b/service/pdptracker/pdptracker.go new file mode 100644 index 00000000..47d94ccf --- /dev/null +++ b/service/pdptracker/pdptracker.go @@ -0,0 +1,331 @@ +// Package pdptracker provides a service for tracking PDP (Proof of Data Possession) deals +// using the f41 actor on Filecoin. This is distinct from legacy f05 market deals. +// +// PDP deals use proof sets managed through the PDPVerifier contract, where data is verified +// through cryptographic challenges rather than the traditional sector sealing process. +// +// Note: This package requires the go-synapse library for full functionality. +// See: https://github.com/data-preservation-programs/go-synapse +package pdptracker + +import ( + "context" + "time" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/database" + "github.com/data-preservation-programs/singularity/model" + "github.com/data-preservation-programs/singularity/service/healthcheck" + "github.com/google/uuid" + "github.com/gotidy/ptr" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-log/v2" + "gorm.io/gorm" +) + +var ErrAlreadyRunning = errors.New("another PDP tracker worker already running") + +const ( + healthRegisterRetryInterval = time.Minute + cleanupTimeout = 5 * time.Second +) + +var Logger = log.Logger("pdptracker") + +// ProofSetInfo contains information about a PDP proof set retrieved from on-chain state +type ProofSetInfo struct { + ProofSetID uint64 + ClientAddress string // f4 address of the client + ProviderAddress string // Provider/record keeper address + IsLive bool // Whether the proof set is actively being challenged + NextChallengeEpoch int32 // Next epoch when a challenge is due + PieceCIDs []string +} + +// PDPClient is the interface for interacting with PDP on-chain state. +// This will be implemented using the go-synapse library once it's available. +type PDPClient interface { + // GetProofSetsForClient returns all proof sets associated with a client address + GetProofSetsForClient(ctx context.Context, clientAddress string) ([]ProofSetInfo, error) + // GetProofSetInfo returns detailed information about a specific proof set + GetProofSetInfo(ctx context.Context, proofSetID uint64) (*ProofSetInfo, error) + // IsProofSetLive checks if a proof set is actively being challenged + IsProofSetLive(ctx context.Context, proofSetID uint64) (bool, error) + // GetNextChallengeEpoch returns the next challenge epoch for a proof set + GetNextChallengeEpoch(ctx context.Context, proofSetID uint64) (int32, error) +} + +// PDPTracker tracks PDP deals (f41 actor) on the Filecoin network. +// It monitors proof sets and updates deal status based on on-chain state. +type PDPTracker struct { + workerID uuid.UUID + dbNoContext *gorm.DB + interval time.Duration + pdpClient PDPClient + rpcURL string + once bool +} + +// NewPDPTracker creates a new PDP deal tracker. +// +// Parameters: +// - db: Database connection for storing deal information +// - interval: How often to check for updates +// - rpcURL: Filecoin RPC endpoint URL +// - pdpClient: Client for interacting with PDP contracts (can be nil for stub mode) +// - once: If true, run only once instead of continuously +func NewPDPTracker( + db *gorm.DB, + interval time.Duration, + rpcURL string, + pdpClient PDPClient, + once bool, +) PDPTracker { + return PDPTracker{ + workerID: uuid.New(), + dbNoContext: db, + interval: interval, + rpcURL: rpcURL, + pdpClient: pdpClient, + once: once, + } +} + +func (*PDPTracker) Name() string { + return "PDPTracker" +} + +// Start begins the PDP tracker service. +func (p *PDPTracker) Start(ctx context.Context, exitErr chan<- error) error { + if p.pdpClient == nil { + Logger.Warn("PDP client not configured - PDP tracking will be disabled until go-synapse is integrated") + if exitErr != nil { + exitErr <- nil + } + return nil + } + + var regTimer *time.Timer + for { + alreadyRunning, err := healthcheck.Register(ctx, p.dbNoContext, p.workerID, model.PDPTracker, false) + if err != nil { + return errors.WithStack(err) + } + if !alreadyRunning { + break + } + + Logger.Warnw("another PDP tracker worker already running") + if p.once { + return ErrAlreadyRunning + } + Logger.Warn("retrying in 1 minute") + if regTimer == nil { + regTimer = time.NewTimer(healthRegisterRetryInterval) + defer regTimer.Stop() + } else { + regTimer.Reset(healthRegisterRetryInterval) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-regTimer.C: + } + } + + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + + healthcheckDone := make(chan struct{}) + go func() { + defer close(healthcheckDone) + healthcheck.StartReportHealth(ctx, p.dbNoContext, p.workerID, model.PDPTracker) + Logger.Info("PDP tracker health report stopped") + }() + + go func() { + var timer *time.Timer + var runErr error + for { + runErr = p.runOnce(ctx) + if runErr != nil { + if ctx.Err() != nil { + if errors.Is(runErr, context.Canceled) { + runErr = nil + } + Logger.Info("PDP tracker run stopped") + break + } + Logger.Errorw("failed to run PDP tracker once", "error", runErr) + } + if p.once { + Logger.Info("PDP tracker run once done") + break + } + if timer == nil { + timer = time.NewTimer(p.interval) + defer timer.Stop() + } else { + timer.Reset(p.interval) + } + + var stopped bool + select { + case <-ctx.Done(): + stopped = true + case <-timer.C: + } + if stopped { + Logger.Info("PDP tracker run stopped") + break + } + } + + cancel() + + ctx2, cancel2 := context.WithTimeout(context.Background(), cleanupTimeout) + defer cancel2() + //nolint:contextcheck + err := p.cleanup(ctx2) + if err != nil { + Logger.Errorw("failed to cleanup PDP tracker", "error", err) + } else { + Logger.Info("PDP tracker cleanup done") + } + + <-healthcheckDone + + if exitErr != nil { + exitErr <- runErr + } + }() + + return nil +} + +func (p *PDPTracker) cleanup(ctx context.Context) error { + return database.DoRetry(ctx, func() error { + return p.dbNoContext.WithContext(ctx).Where("id = ?", p.workerID).Delete(&model.Worker{}).Error + }) +} + +// runOnce performs a single cycle of PDP deal tracking. +// It queries wallets, fetches their PDP proof sets, and updates deal status. +func (p *PDPTracker) runOnce(ctx context.Context) error { + if p.pdpClient == nil { + return nil + } + + db := p.dbNoContext.WithContext(ctx) + + // Get all wallets to track + var wallets []model.Wallet + err := db.Find(&wallets).Error + if err != nil { + return errors.Wrap(err, "failed to get wallets from database") + } + + now := time.Now() + var updated, inserted int64 + + for _, wallet := range wallets { + Logger.Infof("tracking PDP deals for wallet %s", wallet.ID) + + // Get proof sets for this wallet + proofSets, err := p.pdpClient.GetProofSetsForClient(ctx, wallet.Address) + if err != nil { + Logger.Warnw("failed to get proof sets for wallet", "wallet", wallet.ID, "error", err) + continue + } + + for _, ps := range proofSets { + for _, pieceCID := range ps.PieceCIDs { + parsedPieceCID, parseErr := cid.Parse(pieceCID) + if parseErr != nil { + Logger.Warnw("invalid piece CID from PDP proof set", "pieceCID", pieceCID, "proofSetID", ps.ProofSetID, "error", parseErr) + continue + } + modelPieceCID := model.CID(parsedPieceCID) + + // Check if we already have this deal tracked + var existingDeal model.Deal + err := db.Where("proof_set_id = ? AND piece_cid = ? AND deal_type = ?", + ps.ProofSetID, modelPieceCID, model.DealTypePDP).First(&existingDeal).Error + + if err == nil { + // Deal exists, check if status changed + needsUpdate := false + updates := map[string]any{} + + if existingDeal.ProofSetLive == nil || *existingDeal.ProofSetLive != ps.IsLive { + updates["proof_set_live"] = ps.IsLive + needsUpdate = true + } + if existingDeal.NextChallengeEpoch == nil || *existingDeal.NextChallengeEpoch != ps.NextChallengeEpoch { + updates["next_challenge_epoch"] = ps.NextChallengeEpoch + needsUpdate = true + } + + // Update state based on proof set status + newState := p.getPDPDealState(ps) + if existingDeal.State != newState { + updates["state"] = newState + needsUpdate = true + } + + if needsUpdate { + updates["last_verified_at"] = now + err = database.DoRetry(ctx, func() error { + return db.Model(&model.Deal{}).Where("id = ?", existingDeal.ID).Updates(updates).Error + }) + if err != nil { + Logger.Errorw("failed to update PDP deal", "dealID", existingDeal.ID, "error", err) + continue + } + Logger.Infow("PDP deal updated", "dealID", existingDeal.ID, "proofSetID", ps.ProofSetID) + updated++ + } + } else if errors.Is(err, gorm.ErrRecordNotFound) { + // New PDP deal, insert it + newState := p.getPDPDealState(ps) + newDeal := model.Deal{ + DealType: model.DealTypePDP, + State: newState, + ClientID: wallet.ID, + Provider: ps.ProviderAddress, + PieceCID: modelPieceCID, + ProofSetID: ptr.Of(ps.ProofSetID), + ProofSetLive: ptr.Of(ps.IsLive), + NextChallengeEpoch: ptr.Of(ps.NextChallengeEpoch), + LastVerifiedAt: ptr.Of(now), + } + + err = database.DoRetry(ctx, func() error { + return db.Create(&newDeal).Error + }) + if err != nil { + Logger.Errorw("failed to insert PDP deal", "proofSetID", ps.ProofSetID, "error", err) + continue + } + Logger.Infow("PDP deal inserted", "proofSetID", ps.ProofSetID, "state", newState) + inserted++ + } else { + Logger.Errorw("failed to query existing PDP deal", "error", err) + } + } + } + } + + Logger.Infof("PDP tracker: updated %d deals, inserted %d deals", updated, inserted) + return nil +} + +// getPDPDealState determines the deal state based on proof set status +func (p *PDPTracker) getPDPDealState(ps ProofSetInfo) model.DealState { + if ps.IsLive { + return model.DealActive + } + // If not live, it might be proposed (waiting for first challenge) or expired + // This logic may need refinement based on actual PDP contract semantics + return model.DealPublished +} diff --git a/service/pdptracker/pdptracker_test.go b/service/pdptracker/pdptracker_test.go new file mode 100644 index 00000000..3dd332b7 --- /dev/null +++ b/service/pdptracker/pdptracker_test.go @@ -0,0 +1,125 @@ +package pdptracker + +import ( + "context" + "testing" + "time" + + "github.com/data-preservation-programs/singularity/model" + "github.com/data-preservation-programs/singularity/util/testutil" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +type mockPDPClient struct { + proofSets map[string][]ProofSetInfo +} + +func (m *mockPDPClient) GetProofSetsForClient(_ context.Context, clientAddress string) ([]ProofSetInfo, error) { + return m.proofSets[clientAddress], nil +} + +func (m *mockPDPClient) GetProofSetInfo(_ context.Context, _ uint64) (*ProofSetInfo, error) { + return nil, nil +} + +func (m *mockPDPClient) IsProofSetLive(_ context.Context, _ uint64) (bool, error) { + return false, nil +} + +func (m *mockPDPClient) GetNextChallengeEpoch(_ context.Context, _ uint64) (int32, error) { + return 0, nil +} + +func TestPDPTracker_Name(t *testing.T) { + tracker := NewPDPTracker(nil, time.Minute, "", nil, true) + require.Equal(t, "PDPTracker", tracker.Name()) +} + +func TestPDPTracker_RunOnce_UpsertByParsedPieceCID(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + err := db.Create(&model.Wallet{ + ID: "f0100", + Address: "f4wallet", + }).Error + require.NoError(t, err) + + const pieceCID = "baga6ea4seaqao7s73y24kcutaosvacpdjgfe5pw76ooefnyqw4ynr3d2y6x2mpq" + client := &mockPDPClient{ + proofSets: map[string][]ProofSetInfo{ + "f4wallet": { + { + ProofSetID: 7, + ClientAddress: "f4wallet", + ProviderAddress: "f01234", + IsLive: true, + NextChallengeEpoch: 10, + PieceCIDs: []string{pieceCID}, + }, + }, + }, + } + + tracker := NewPDPTracker(db, time.Minute, "", client, true) + require.NoError(t, tracker.runOnce(ctx)) + + var first model.Deal + err = db.Where("deal_type = ?", model.DealTypePDP).First(&first).Error + require.NoError(t, err) + require.Equal(t, model.DealTypePDP, first.DealType) + require.Equal(t, pieceCID, first.PieceCID.String()) + require.NotNil(t, first.ProofSetID) + require.EqualValues(t, 7, *first.ProofSetID) + require.NotNil(t, first.ProofSetLive) + require.True(t, *first.ProofSetLive) + require.Equal(t, model.DealActive, first.State) + require.NotNil(t, first.LastVerifiedAt) + + client.proofSets["f4wallet"][0].IsLive = false + client.proofSets["f4wallet"][0].NextChallengeEpoch = 11 + require.NoError(t, tracker.runOnce(ctx)) + + var deals []model.Deal + err = db.Where("deal_type = ?", model.DealTypePDP).Find(&deals).Error + require.NoError(t, err) + require.Len(t, deals, 1) + require.NotNil(t, deals[0].ProofSetLive) + require.False(t, *deals[0].ProofSetLive) + require.NotNil(t, deals[0].NextChallengeEpoch) + require.EqualValues(t, 11, *deals[0].NextChallengeEpoch) + require.Equal(t, model.DealPublished, deals[0].State) + require.NotNil(t, deals[0].LastVerifiedAt) + }) +} + +func TestPDPTracker_RunOnce_SkipsInvalidPieceCID(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + err := db.Create(&model.Wallet{ + ID: "f0100", + Address: "f4wallet", + }).Error + require.NoError(t, err) + + client := &mockPDPClient{ + proofSets: map[string][]ProofSetInfo{ + "f4wallet": { + { + ProofSetID: 7, + ClientAddress: "f4wallet", + ProviderAddress: "f01234", + IsLive: true, + NextChallengeEpoch: 10, + PieceCIDs: []string{"not-a-cid"}, + }, + }, + }, + } + tracker := NewPDPTracker(db, time.Minute, "", client, true) + require.NoError(t, tracker.runOnce(ctx)) + + var count int64 + err = db.Model(&model.Deal{}).Where("deal_type = ?", model.DealTypePDP).Count(&count).Error + require.NoError(t, err) + require.EqualValues(t, 0, count) + }) +}