From df5f70f2d0d23e4e0cd1869e4423ce41d514271a Mon Sep 17 00:00:00 2001 From: Rekseto Date: Tue, 10 Mar 2026 16:58:49 +0000 Subject: [PATCH 01/42] mod/gateway: WIP --- mod/gateway/client/bind.go | 24 +++++ mod/gateway/client/client.go | 36 +++++++ mod/gateway/client/connect.go | 24 +++++ mod/gateway/client/list.go | 25 +++++ mod/gateway/errors.go | 6 ++ mod/gateway/module.go | 9 ++ mod/gateway/socket.go | 25 +++++ mod/gateway/src/bind.go | 66 ++++++++++++ mod/gateway/src/bind_gateway.go | 20 ++++ mod/gateway/src/binder_conn_pool.go | 32 ++++++ mod/gateway/src/config.go | 20 +++- mod/gateway/src/conn.go | 42 -------- mod/gateway/src/connect.go | 52 ++++++++++ mod/gateway/src/deps.go | 3 +- mod/gateway/src/dialer.go | 41 -------- mod/gateway/src/loader.go | 10 +- mod/gateway/src/module.go | 141 ++++++-------------------- mod/gateway/src/op_list.go | 28 +++++ mod/gateway/src/op_node_bind.go | 30 ++++++ mod/gateway/src/op_node_connect.go | 29 ++++++ mod/gateway/src/receiver_conn_pool.go | 78 ++++++++++++++ mod/gateway/src/route_service.go | 79 --------------- mod/gateway/src/server.go | 106 +++++++++++++++++++ mod/gateway/src/subscribe_service.go | 46 --------- mod/gateway/src/subscriber.go | 83 --------------- mod/gateway/visibility.go | 10 ++ mod/tcp/module.go | 3 + mod/tcp/src/loader.go | 2 + mod/tcp/src/module.go | 28 +++++ mod/tcp/src/server.go | 82 +++++++++++---- 30 files changed, 751 insertions(+), 429 deletions(-) create mode 100644 mod/gateway/client/bind.go create mode 100644 mod/gateway/client/client.go create mode 100644 mod/gateway/client/connect.go create mode 100644 mod/gateway/client/list.go create mode 100644 mod/gateway/errors.go create mode 100644 mod/gateway/module.go create mode 100644 mod/gateway/socket.go create mode 100644 mod/gateway/src/bind.go create mode 100644 mod/gateway/src/bind_gateway.go create mode 100644 mod/gateway/src/binder_conn_pool.go delete mode 100644 mod/gateway/src/conn.go create mode 100644 mod/gateway/src/connect.go delete mode 100644 mod/gateway/src/dialer.go create mode 100644 mod/gateway/src/op_list.go create mode 100644 mod/gateway/src/op_node_bind.go create mode 100644 mod/gateway/src/op_node_connect.go create mode 100644 mod/gateway/src/receiver_conn_pool.go delete mode 100644 mod/gateway/src/route_service.go create mode 100644 mod/gateway/src/server.go delete mode 100644 mod/gateway/src/subscribe_service.go delete mode 100644 mod/gateway/src/subscriber.go create mode 100644 mod/gateway/visibility.go diff --git a/mod/gateway/client/bind.go b/mod/gateway/client/bind.go new file mode 100644 index 000000000..3a484aa74 --- /dev/null +++ b/mod/gateway/client/bind.go @@ -0,0 +1,24 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/query" + gw "github.com/cryptopunkscc/astrald/mod/gateway" +) + +func (c *Client) Bind(ctx *astral.Context, visibility gw.Visibility) (*gw.Socket, error) { + ch, err := c.queryCh(ctx, gw.MethodBind, query.Args{"visibility": string(visibility)}) + if err != nil { + return nil, err + } + defer ch.Close() + + var socket *gw.Socket + err = ch.Switch( + channel.Expect(&socket), + channel.PassErrors, + ) + + return socket, err +} diff --git a/mod/gateway/client/client.go b/mod/gateway/client/client.go new file mode 100644 index 000000000..756aa8820 --- /dev/null +++ b/mod/gateway/client/client.go @@ -0,0 +1,36 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/astrald" +) + +type Client struct { + astral *astrald.Client + targetID *astral.Identity +} + +var defaultClient *Client + +func New(targetID *astral.Identity, a *astrald.Client) *Client { + if a == nil { + a = astrald.Default() + } + return &Client{astral: a, targetID: targetID} +} + +func Default() *Client { + if defaultClient == nil { + defaultClient = New(nil, nil) + } + return defaultClient +} + +func SetDefault(client *Client) { + defaultClient = client +} + +func (c *Client) queryCh(ctx *astral.Context, method string, args any, cfg ...channel.ConfigFunc) (*channel.Channel, error) { + return c.astral.WithTarget(c.targetID).QueryChannel(ctx, method, args, cfg...) +} diff --git a/mod/gateway/client/connect.go b/mod/gateway/client/connect.go new file mode 100644 index 000000000..e190a30a7 --- /dev/null +++ b/mod/gateway/client/connect.go @@ -0,0 +1,24 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/query" + gw "github.com/cryptopunkscc/astrald/mod/gateway" +) + +func (c *Client) Connect(ctx *astral.Context, target *astral.Identity) (*gw.Socket, error) { + ch, err := c.queryCh(ctx, gw.MethodConnect, query.Args{"target": target.String()}) + if err != nil { + return nil, err + } + defer ch.Close() + + var socket *gw.Socket + err = ch.Switch( + channel.Expect(&socket), + channel.PassErrors, + ) + + return socket, err +} diff --git a/mod/gateway/client/list.go b/mod/gateway/client/list.go new file mode 100644 index 000000000..945948ae6 --- /dev/null +++ b/mod/gateway/client/list.go @@ -0,0 +1,25 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/query" + gw "github.com/cryptopunkscc/astrald/mod/gateway" +) + +func (c *Client) List(ctx *astral.Context) ([]*astral.Identity, error) { + ch, err := c.queryCh(ctx, gw.MethodList, query.Args{}) + if err != nil { + return nil, err + } + defer ch.Close() + + var list []*astral.Identity + err = ch.Switch( + channel.Collect(&list), + channel.StopOnEOS, + channel.PassErrors, + ) + + return list, err +} diff --git a/mod/gateway/errors.go b/mod/gateway/errors.go new file mode 100644 index 000000000..b109c8825 --- /dev/null +++ b/mod/gateway/errors.go @@ -0,0 +1,6 @@ +package gateway + +import "errors" + +var ErrUnauthorized = errors.New("unauthorized") +var ErrTargetNotReachable = errors.New("target not reachable") diff --git a/mod/gateway/module.go b/mod/gateway/module.go new file mode 100644 index 000000000..593b87e27 --- /dev/null +++ b/mod/gateway/module.go @@ -0,0 +1,9 @@ +package gateway + +const ModuleName = "gateway" + +const ( + MethodBind = "gateway.node.bind" + MethodConnect = "gateway.node.connect" + MethodList = "gateway.node.list" +) diff --git a/mod/gateway/socket.go b/mod/gateway/socket.go new file mode 100644 index 000000000..9a6afa371 --- /dev/null +++ b/mod/gateway/socket.go @@ -0,0 +1,25 @@ +package gateway + +import ( + "io" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" +) + +// Socket describes a raw connection point at the gateway. The recipient opens +// a raw exonet connection to the Endpoint and sends Nonce as the first bytes +// to identify itself to the gateway. +type Socket struct { + Endpoint exonet.Endpoint + Nonce astral.Nonce +} + +func (Socket) ObjectType() string { return "mod.gateway.socket" } + +func (s Socket) WriteTo(w io.Writer) (int64, error) { return astral.Objectify(&s).WriteTo(w) } +func (s *Socket) ReadFrom(r io.Reader) (int64, error) { return astral.Objectify(s).ReadFrom(r) } + +func init() { + astral.Add(&Socket{}) +} diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go new file mode 100644 index 000000000..816c062b4 --- /dev/null +++ b/mod/gateway/src/bind.go @@ -0,0 +1,66 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +type Binder struct { + Identity *astral.Identity + Visibility gateway.Visibility + Nonce astral.Nonce + ConnPool *binderConnPool +} + +func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibility gateway.Visibility, network string) (*gateway.Socket, error) { + if !mod.canGateway(identity) { + return nil, gateway.ErrUnauthorized + } + + endpoint, err := mod.getGatewayEndpoint(ctx, network) + if err != nil { + return nil, err + } + + nonce := astral.NewNonce() + + binder := &Binder{ + Identity: identity, + Visibility: visibility, + Nonce: nonce, + ConnPool: newBinderConnPool(mod), + } + + if old, ok := mod.binderByIdentity(identity); ok { + mod.binders.Remove(old) + } + + mod.binders.Add(binder) + + return &gateway.Socket{ + Nonce: nonce, + Endpoint: endpoint, + }, nil +} + +func (mod *Module) canGateway(identity *astral.Identity) bool { + return mod.config.ActAsGateway +} + +func (mod *Module) binderByNonce(nonce astral.Nonce) (*Binder, bool) { + for _, b := range mod.binders.Clone() { + if b.Nonce == nonce { + return b, true + } + } + return nil, false +} + +func (mod *Module) binderByIdentity(identity *astral.Identity) (*Binder, bool) { + for _, b := range mod.binders.Clone() { + if b.Identity.IsEqual(identity) { + return b, true + } + } + return nil, false +} diff --git a/mod/gateway/src/bind_gateway.go b/mod/gateway/src/bind_gateway.go new file mode 100644 index 000000000..74c497080 --- /dev/null +++ b/mod/gateway/src/bind_gateway.go @@ -0,0 +1,20 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/lib/astrald" + "github.com/cryptopunkscc/astrald/mod/gateway" + gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" +) + +func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity, visibility gateway.Visibility) { + client := gatewayClient.New(gatewayID, astrald.Default()) + + socket, err := client.Bind(ctx, visibility) + if err != nil { + mod.log.Error("bind to %v: %v", gatewayID, err) + return + } + + newReceiverConnPool(mod, socket).Run(ctx) +} diff --git a/mod/gateway/src/binder_conn_pool.go b/mod/gateway/src/binder_conn_pool.go new file mode 100644 index 000000000..a5a65bb8b --- /dev/null +++ b/mod/gateway/src/binder_conn_pool.go @@ -0,0 +1,32 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/sig" +) + +type binderConnPool struct { + *Module + + conns sig.Set[exonet.Conn] +} + +func newBinderConnPool(module *Module) *binderConnPool { + return &binderConnPool{Module: module} +} + +func (p *binderConnPool) add(conn exonet.Conn) { + p.conns.Add(conn) +} + +func (p *binderConnPool) take() (exonet.Conn, bool) { + items := p.conns.Clone() + if len(items) == 0 { + return nil, false + } + conn := items[0] + if err := p.conns.Remove(conn); err != nil { + return nil, false + } + return conn, true +} diff --git a/mod/gateway/src/config.go b/mod/gateway/src/config.go index 5e51fee44..045a3d7a0 100644 --- a/mod/gateway/src/config.go +++ b/mod/gateway/src/config.go @@ -1,13 +1,29 @@ package gateway +import "github.com/cryptopunkscc/astrald/mod/gateway" + const defaultGateway = "node1f3AwbE1gJAgAqEx98FMipokcaE9ZapIphzDUkAceE7Pmw8ghmFV19QKCATeC7uyoLszQA" +const ( + defaultInitConns = 1 + defaultMaxConns = 8 +) + type Config struct { - Subscribe []string `yaml:"subscribe"` + ActAsGateway bool `yaml:"act_as_gateway"` + + Sockets map[string]uint16 `yaml:"sockets"` + Visibility gateway.Visibility `yaml:"visibility"` + Gateways []string `yaml:"gateways"` + InitConns int32 `yaml:"init_conns"` + MaxConns int32 `yaml:"max_conns"` } var defaultConfig = Config{ - Subscribe: []string{ + Visibility: gateway.VisibilityPublic, + Gateways: []string{ defaultGateway, }, + InitConns: defaultInitConns, + MaxConns: defaultMaxConns, } diff --git a/mod/gateway/src/conn.go b/mod/gateway/src/conn.go deleted file mode 100644 index 8eda9e9f3..000000000 --- a/mod/gateway/src/conn.go +++ /dev/null @@ -1,42 +0,0 @@ -package gateway - -import ( - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" -) - -var _ exonet.Conn = &Conn{} - -type Conn struct { - astral.Conn - localEndpoint *gateway.Endpoint - remoteEndpoint *gateway.Endpoint - outbound bool -} - -func newConn(conn astral.Conn, localEndpoint *gateway.Endpoint, remoteEndpoint *gateway.Endpoint, outbound bool) *Conn { - c := &Conn{ - Conn: conn, - localEndpoint: localEndpoint, - remoteEndpoint: remoteEndpoint, - outbound: outbound, - } - return c -} - -func (conn Conn) LocalEndpoint() exonet.Endpoint { - return conn.localEndpoint -} - -func (conn Conn) RemoteEndpoint() exonet.Endpoint { - return conn.remoteEndpoint -} - -func (conn Conn) Outbound() bool { - return conn.outbound -} - -func (Conn) Network() string { - return NetworkName -} diff --git a/mod/gateway/src/connect.go b/mod/gateway/src/connect.go new file mode 100644 index 000000000..a4430c588 --- /dev/null +++ b/mod/gateway/src/connect.go @@ -0,0 +1,52 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +type Connecting struct { + Identity *astral.Identity + Target *astral.Identity + Nonce astral.Nonce +} + +func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, network string) (socket gateway.Socket, err error) { + if !mod.canGateway(caller) { + return socket, gateway.ErrUnauthorized + } + + endpoint, err := mod.getGatewayEndpoint(mod.ctx, network) + if err != nil { + return gateway.Socket{}, err + } + + nonce := astral.NewNonce() + + _, ok := mod.binderByIdentity(target) + if !ok { + return socket, gateway.ErrTargetNotReachable + } + + connecting := &Connecting{ + Identity: caller, + Target: target, + Nonce: nonce, + } + + mod.connecting.Add(connecting) + + return gateway.Socket{ + Nonce: nonce, + Endpoint: endpoint, + }, nil +} + +func (mod *Module) connectingByNonce(nonce astral.Nonce) (*Connecting, bool) { + for _, c := range mod.connecting.Clone() { + if c.Nonce == nonce { + return c, true + } + } + return nil, false +} diff --git a/mod/gateway/src/deps.go b/mod/gateway/src/deps.go index 23cf6f795..6efbf1123 100644 --- a/mod/gateway/src/deps.go +++ b/mod/gateway/src/deps.go @@ -11,9 +11,10 @@ func (mod *Module) LoadDependencies(*astral.Context) (err error) { return } - mod.Exonet.SetDialer("gw", mod.dialer) + // mod.Exonet.SetDialer("gw", mod.dialer) mod.Exonet.SetUnpacker("gw", mod) mod.Exonet.SetParser("gw", mod) + mod.ops.AddStructPrefix(mod, "Op") return } diff --git a/mod/gateway/src/dialer.go b/mod/gateway/src/dialer.go deleted file mode 100644 index 1d1dcac00..000000000 --- a/mod/gateway/src/dialer.go +++ /dev/null @@ -1,41 +0,0 @@ -package gateway - -import ( - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/lib/query" - "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" -) - -type Dialer struct { - node astral.Node -} - -func NewDialer(node astral.Node) *Dialer { - return &Dialer{node: node} -} - -func (dialer *Dialer) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.Conn, error) { - e, err := Unpack(endpoint.Pack()) - if err != nil { - return nil, err - } - - if e.GatewayID.IsEqual(dialer.node.Identity()) { - return nil, ErrInvalidGateway - } - - var q = astral.NewQuery(dialer.node.Identity(), e.GatewayID, RouteServiceName+"."+e.TargetID.String()) - - conn, err := query.Route(ctx, dialer.node, q) - if err != nil { - return nil, err - } - - return newConn( - conn, - gateway.NewEndpoint(dialer.node.Identity(), dialer.node.Identity()), - e, - true, - ), err -} diff --git a/mod/gateway/src/loader.go b/mod/gateway/src/loader.go index c367d565e..e07170879 100644 --- a/mod/gateway/src/loader.go +++ b/mod/gateway/src/loader.go @@ -14,12 +14,10 @@ type Loader struct{} func (Loader) Load(node astral.Node, assets assets.Assets, log *log.Logger) (core.Module, error) { mod := &Module{ - node: node, - log: log, - PathRouter: routers.NewPathRouter(node.Identity(), false), - config: defaultConfig, - dialer: NewDialer(node), - subscribers: make(map[string]*Subscriber), + node: node, + log: log, + PathRouter: routers.NewPathRouter(node.Identity(), false), + config: defaultConfig, } _ = assets.LoadYAML(ModuleName, &mod.config) diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index 0b52c3ffc..72cd4fc62 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -1,17 +1,16 @@ package gateway import ( - "strings" - "sync" - "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/astral/log" + "github.com/cryptopunkscc/astrald/lib/ops" "github.com/cryptopunkscc/astrald/lib/routers" "github.com/cryptopunkscc/astrald/mod/dir" "github.com/cryptopunkscc/astrald/mod/exonet" - gateway2 "github.com/cryptopunkscc/astrald/mod/gateway" + ipmod "github.com/cryptopunkscc/astrald/mod/ip" "github.com/cryptopunkscc/astrald/mod/nodes" - "github.com/cryptopunkscc/astrald/tasks" + tcpmod "github.com/cryptopunkscc/astrald/mod/tcp" + "github.com/cryptopunkscc/astrald/sig" ) const NetworkName = "gw" @@ -20,137 +19,63 @@ type Deps struct { Dir dir.Module Exonet exonet.Module Nodes nodes.Module + TCP tcpmod.Module + IP ipmod.Module } type Module struct { Deps *routers.PathRouter - config Config - node astral.Node - log *log.Logger - ctx *astral.Context - dialer *Dialer - subscribers map[string]*Subscriber - mu sync.Mutex -} -func (mod *Module) Run(ctx *astral.Context) error { - mod.ctx = ctx.IncludeZone(astral.ZoneNetwork) + ops ops.Set + config Config + node astral.Node + log *log.Logger + ctx *astral.Context - mod.subscribeToGateways() + binders sig.Set[*Binder] + connecting sig.Set[*Connecting] - return tasks.Group( - &SubscribeService{Module: mod}, - &RouteService{Module: mod, router: mod.node}, - ).Run(ctx) + listenEndpoints sig.Map[string, exonet.Endpoint] } -func (mod *Module) subscribeToGateways() { - for _, gateName := range mod.config.Subscribe { - var gateID *astral.Identity - - if after, found := strings.CutPrefix(gateName, "node1"); found && len(after) > 32 { - var info nodes.NodeInfo - - err := info.UnmarshalText([]byte(after)) - if err != nil { - mod.log.Error("parse node info: %v", err) - continue - } - - // try to set alias - err = mod.Dir.SetAlias(info.Identity, string(info.Alias)) - if err != nil { - mod.log.Error("set alias: %v", err) - } - - // save endpoints - for _, ep := range info.Endpoints { - err = mod.Nodes.AddEndpoint(info.Identity, nodes.NewEndpointWithTTL(ep)) - if err != nil { - mod.log.Error("add endpoint: %v", err) - continue - } - } - - // subscribe - err = mod.Subscribe(info.Identity) - if err != nil { - mod.log.Error("subscribe: %v", err) - } - continue - } - - gateID, err := mod.Dir.ResolveIdentity(gateName) - if err != nil { - mod.log.Error("resolve identity %v: %v", gateName, err) - continue - } - - err = mod.Subscribe(gateID) - if err != nil { - mod.log.Error("subscribe: %v", err) - } - } +func (mod *Module) GetOpSet() *ops.Set { + return &mod.ops } -func (mod *Module) Subscribe(gateway *astral.Identity) error { - mod.mu.Lock() - defer mod.mu.Unlock() - - switch { - case gateway.IsZero(): - return ErrInvalidGateway - case gateway.IsEqual(mod.node.Identity()): - return ErrInvalidGateway - } - - var hex = gateway.String() - - if _, found := mod.subscribers[hex]; found { - return ErrAlreadySubscribed - } +func (mod *Module) Run(ctx *astral.Context) error { + mod.ctx = ctx.IncludeZone(astral.ZoneNetwork) - var s = NewSubscriber(gateway, mod.node, mod.log) - mod.subscribers[hex] = s + mod.startServers(mod.ctx) - go func() { - err := s.Run(mod.ctx) + for _, gatewayStr := range mod.config.Gateways { + identity, err := mod.Dir.ResolveIdentity(gatewayStr) if err != nil { - mod.log.Errorv(1, "gateway %v subscriber ended with error: %v", gateway, err) + mod.log.Error("resolve gateway %v: %v", gatewayStr, err) + continue } - mod.mu.Lock() - defer mod.mu.Unlock() - delete(mod.subscribers, hex) - }() - - return nil -} - -func (mod *Module) Unsubscribe(gateway *astral.Identity) error { - mod.mu.Lock() - defer mod.mu.Unlock() - - s, found := mod.subscribers[gateway.String()] - if !found { - return ErrNotSubscribed + go mod.bindToGateway(mod.ctx, identity, mod.config.Visibility) } - s.Cancel() + <-ctx.Done() return nil } func (mod *Module) Endpoints() []exonet.Endpoint { var list = make([]exonet.Endpoint, 0) - for _, s := range mod.subscribers { - list = append(list, gateway2.NewEndpoint(s.Gateway(), mod.node.Identity())) - } - return list } +func (mod *Module) getGatewayEndpoint(ctx *astral.Context, network string) (endpoint exonet.Endpoint, err error) { + endpoint, ok := mod.listenEndpoints.Get(network) + if !ok { + // fixme: return public error (no gateway endpoint available) + return + } + return endpoint, nil +} func (mod *Module) String() string { return ModuleName } diff --git a/mod/gateway/src/op_list.go b/mod/gateway/src/op_list.go new file mode 100644 index 000000000..fdb214c5d --- /dev/null +++ b/mod/gateway/src/op_list.go @@ -0,0 +1,28 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/ops" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +type opListArgs struct { + Out string `query:"optional"` +} + +func (mod *Module) OpList(ctx *astral.Context, q *ops.Query, args opListArgs) error { + ch := q.AcceptChannel(channel.WithOutputFormat(args.Out)) + defer ch.Close() + + for _, binder := range mod.binders.Clone() { + if binder.Visibility != gateway.VisibilityPublic { + continue + } + if err := ch.Send(binder.Identity); err != nil { + return err + } + } + + return ch.Send(&astral.EOS{}) +} diff --git a/mod/gateway/src/op_node_bind.go b/mod/gateway/src/op_node_bind.go new file mode 100644 index 000000000..e8984c908 --- /dev/null +++ b/mod/gateway/src/op_node_bind.go @@ -0,0 +1,30 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/ops" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +type opNodeBindArgs struct { + Visibility gateway.Visibility + In string `query:"optional"` + Out string `query:"optional"` +} + +func (mod *Module) OpNodeBind( + ctx *astral.Context, + q *ops.Query, + args opNodeBindArgs, +) (err error) { + ch := channel.New(q.Accept(), channel.WithFormats(args.In, args.Out)) + defer ch.Close() + + socket, err := mod.bind(ctx, q.Caller(), args.Visibility, "tcp") + if err != nil { + return ch.Send(astral.NewError(err.Error())) + } + + return ch.Send(socket) +} diff --git a/mod/gateway/src/op_node_connect.go b/mod/gateway/src/op_node_connect.go new file mode 100644 index 000000000..93ed7b48f --- /dev/null +++ b/mod/gateway/src/op_node_connect.go @@ -0,0 +1,29 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/ops" +) + +type opNodeConnectArgs struct { + Target *astral.Identity + In string `query:"optional"` + Out string `query:"optional"` +} + +func (mod *Module) OpNodeConnect( + ctx *astral.Context, + q *ops.Query, + args opNodeConnectArgs, +) (err error) { + ch := channel.New(q.Accept(), channel.WithFormats(args.In, args.Out)) + defer ch.Close() + + socket, err := mod.connectTo(q.Caller(), args.Target, "tcp") + if err != nil { + return ch.Send(astral.NewError(err.Error())) + } + + return ch.Send(&socket) +} diff --git a/mod/gateway/src/receiver_conn_pool.go b/mod/gateway/src/receiver_conn_pool.go new file mode 100644 index 000000000..331aca4ce --- /dev/null +++ b/mod/gateway/src/receiver_conn_pool.go @@ -0,0 +1,78 @@ +package gateway + +import ( + "sync/atomic" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +type receiverConnPool struct { + *Module + socket *gateway.Socket + count atomic.Int32 +} + +func newReceiverConnPool(module *Module, socket *gateway.Socket) *receiverConnPool { + return &receiverConnPool{Module: module, socket: socket} +} + +func (pool *receiverConnPool) Run(ctx *astral.Context) { + for range pool.config.InitConns { + pool.spawn(ctx) + } + + <-ctx.Done() +} + +func (pool *receiverConnPool) spawn(ctx *astral.Context) { + if pool.count.Add(1) > pool.config.MaxConns { + pool.count.Add(-1) + pool.log.Error("max connections reached (%v), cannot spawn new slot", pool.config.MaxConns) + return + } + + go func() { + defer pool.count.Add(-1) + pool.hold(ctx) + }() +} + +func (pool *receiverConnPool) hold(ctx *astral.Context) { + conn, err := pool.Exonet.Dial(ctx, pool.socket.Endpoint) + if err != nil { + return + } + + // Authenticate with the gateway + if _, err = pool.socket.Nonce.WriteTo(conn); err != nil { + pool.log.Errorv(1, "nonce write to %v failed: %v", conn.RemoteEndpoint(), err) + return + } + + // Wrap conn: on first incoming byte, spawn a replacement slot; pass all bytes through untouched + slot := &slotConn{ + Conn: conn, + onFirst: func() { pool.spawn(ctx) }, + } + + if err = pool.Nodes.EstablishInboundLink(ctx, slot); err != nil { + pool.log.Errorv(1, "inbound link from %v failed: %v", conn.RemoteEndpoint(), err) + } +} + +// slotConn wraps an exonet.Conn and calls onFirst exactly once on the first incoming bytes. +type slotConn struct { + exonet.Conn + triggered atomic.Bool + onFirst func() +} + +func (c *slotConn) Read(p []byte) (int, error) { + n, err := c.Conn.Read(p) + if n > 0 && c.triggered.CompareAndSwap(false, true) { + c.onFirst() + } + return n, err +} diff --git a/mod/gateway/src/route_service.go b/mod/gateway/src/route_service.go deleted file mode 100644 index f2136d8f6..000000000 --- a/mod/gateway/src/route_service.go +++ /dev/null @@ -1,79 +0,0 @@ -package gateway - -import ( - "context" - "io" - "strings" - "time" - - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/lib/query" - "github.com/cryptopunkscc/astrald/mod/gateway" -) - -const RouteServiceName = ".gateway" -const acceptTimeout = 15 * time.Second - -type RouteService struct { - *Module - router astral.Router -} - -func (srv *RouteService) Run(ctx *astral.Context) error { - err := srv.AddRoute(RouteServiceName+".*", srv) - if err != nil { - return err - } - defer srv.RemoveRoute(RouteServiceName + ".*") - - <-ctx.Done() - return nil -} - -func (srv *RouteService) RouteQuery(ctx *astral.Context, q *astral.Query, w io.WriteCloser) (io.WriteCloser, error) { - var targetKey string - - switch { - case strings.HasPrefix(q.Query, RouteServiceName+"."): - targetKey, _ = strings.CutPrefix(q.Query, RouteServiceName+".") - - default: - return query.Reject() - } - - // check if the target is us - if targetKey == srv.node.Identity().String() { - return query.Accept(q, w, func(conn astral.Conn) { - gwConn := newConn( - conn, - gateway.NewEndpoint(q.Target, q.Target), - gateway.NewEndpoint(q.Caller, q.Target), - false, - ) - - actx, cancel := context.WithTimeout(context.Background(), acceptTimeout) - defer cancel() - - err := srv.Nodes.EstablishInboundLink(actx, gwConn) - if err != nil { - return - } - }) - } - - targetIdentity, err := astral.ParseIdentity(targetKey) - if err != nil { - return query.Reject() - } - - nextQuery := &astral.Query{ - Nonce: astral.NewNonce(), - Caller: srv.node.Identity(), - Target: targetIdentity, - Query: q.Query, - } - - srv.log.Logv(2, "forwarding %v to %v", q.Caller, targetIdentity) - - return srv.router.RouteQuery(ctx, nextQuery, w) -} diff --git a/mod/gateway/src/server.go b/mod/gateway/src/server.go new file mode 100644 index 000000000..217fa7af3 --- /dev/null +++ b/mod/gateway/src/server.go @@ -0,0 +1,106 @@ +package gateway + +import ( + "context" + "io" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" + tcpmod "github.com/cryptopunkscc/astrald/mod/tcp" +) + +func (mod *Module) startServers(ctx *astral.Context) { + for network, port := range mod.config.Sockets { + switch network { + case "tcp": + endpoint, err := mod.startTCPServer(ctx, port) + if err != nil { + mod.log.Error("start gateway tcp server on port %v: %v", port, err) + continue + } + mod.listenEndpoints.Set(network, endpoint) + default: + mod.log.Error("unsupported gateway socket network: %v", network) + } + } +} + +func (mod *Module) startTCPServer(ctx *astral.Context, port uint16) (*tcpmod.Endpoint, error) { + server := mod.TCP.NewServer(astral.Uint16(port), mod.acceptConn) + + endpoint := &tcpmod.Endpoint{Port: astral.Uint16(port)} + ips, _ := mod.IP.LocalIPs() + for _, ip := range ips { + if !ip.IsLoopback() { + endpoint.IP = ip + break + } + } + if endpoint.IP == nil && len(ips) > 0 { + endpoint.IP = ips[0] + } + + go func() { + if err := server.Run(ctx); err != nil { + mod.log.Error("gateway tcp server: %v", err) + } + }() + + return endpoint, nil +} + +func (mod *Module) acceptConn(_ context.Context, conn exonet.Conn) (bool, error) { + var nonce astral.Nonce + if _, err := nonce.ReadFrom(conn); err != nil { + mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) + conn.Close() + return false, nil + } + + if binder, ok := mod.binderByNonce(nonce); ok { + mod.log.Infov(1, "slot conn from %v via %v", binder.Identity, conn.RemoteEndpoint()) + binder.ConnPool.add(conn) + return false, nil + } + + if connecting, ok := mod.connectingByNonce(nonce); ok { + mod.connecting.Remove(connecting) + + binder, ok := mod.binderByIdentity(connecting.Target) + if !ok { + mod.log.Errorv(1, "no binder for %v", connecting.Target) + conn.Close() + return false, nil + } + + binderConnection, ok := binder.ConnPool.take() + if !ok { + mod.log.Errorv(1, "no available binderConnection for %v", connecting.Target) + conn.Close() + return false, nil + } + + mod.log.Infov(1, "connecting %v to %v", connecting.Identity, connecting.Target) + + go func() { + defer binderConnection.Close() + defer conn.Close() + pipe(binderConnection, conn) + }() + return false, nil + } + + mod.log.Errorv(1, "unknown nonce %v from %v", nonce, conn.RemoteEndpoint()) + conn.Close() + return false, nil +} + +func pipe(a, b io.ReadWriteCloser) { + done := make(chan struct{}) + go func() { + defer close(done) + io.Copy(a, b) + }() + io.Copy(b, a) + <-done +} diff --git a/mod/gateway/src/subscribe_service.go b/mod/gateway/src/subscribe_service.go deleted file mode 100644 index 2c44d7278..000000000 --- a/mod/gateway/src/subscribe_service.go +++ /dev/null @@ -1,46 +0,0 @@ -package gateway - -import ( - "encoding/json" - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/lib/query" - "io" - "time" -) - -const SubscribeServiceName = ".gateway.subscribe" -const SubscribeServiceType = "mod.gateway.subscribe" -const defaultSubscriptionDuration = 24 * time.Hour - -type SubscribeService struct { - *Module -} - -type Subscription struct { - Status string - ExpiresAt time.Time `json:"expires_at,omitempty"` -} - -func (srv *SubscribeService) Run(ctx *astral.Context) error { - var err = srv.AddRoute(SubscribeServiceName, srv) - if err != nil { - return err - } - defer srv.RemoveRoute(SubscribeServiceName) - - <-ctx.Done() - return nil -} - -func (srv *SubscribeService) RouteQuery(ctx *astral.Context, q *astral.Query, w io.WriteCloser) (io.WriteCloser, error) { - return query.Accept(q, w, func(conn astral.Conn) { - defer conn.Close() - - s := &Subscription{ - Status: "ok", - ExpiresAt: time.Now().Add(defaultSubscriptionDuration), - } - - json.NewEncoder(conn).Encode(s) - }) -} diff --git a/mod/gateway/src/subscriber.go b/mod/gateway/src/subscriber.go deleted file mode 100644 index 4f12950c1..000000000 --- a/mod/gateway/src/subscriber.go +++ /dev/null @@ -1,83 +0,0 @@ -package gateway - -import ( - "context" - "encoding/json" - "errors" - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/astral/log" - "github.com/cryptopunkscc/astrald/lib/query" - "time" -) - -const minimumSubscriptionDuration = 15 * time.Minute -const subscribeRetryInterval = 60 * time.Second - -type Subscriber struct { - node astral.Node - log *log.Logger - gateway *astral.Identity - cancel context.CancelFunc -} - -func (s *Subscriber) Gateway() *astral.Identity { - return s.gateway -} - -func NewSubscriber(gateway *astral.Identity, node astral.Node, log *log.Logger) *Subscriber { - return &Subscriber{node: node, log: log, gateway: gateway} -} - -func (s *Subscriber) Run(ctx *astral.Context) error { - ctx, s.cancel = ctx.WithCancel() - defer s.cancel() - - var expiresAt time.Time - for { - conn, err := query.Route(ctx, s.node, astral.NewQuery(s.node.Identity(), s.gateway, SubscribeServiceName)) - if err != nil { - select { - case <-ctx.Done(): - return nil - case <-time.After(subscribeRetryInterval): - } - continue - } - - var info Subscription - err = json.NewDecoder(conn).Decode(&info) - conn.Close() - - if err != nil { - select { - case <-ctx.Done(): - return nil - case <-time.After(subscribeRetryInterval): - } - continue - } - - if info.Status != "ok" { - return errors.New("subscription rejected") - } - - expiresAt = info.ExpiresAt - if time.Until(expiresAt) < minimumSubscriptionDuration { - return errors.New("subscription too short") - } - - s.log.Infov(2, "subscribed to %v until %v", s.gateway, expiresAt) - - select { - case <-ctx.Done(): - return nil - case <-time.After(time.Until(expiresAt) - time.Minute): - } - } -} - -func (s *Subscriber) Cancel() { - if s.cancel != nil { - s.cancel() - } -} diff --git a/mod/gateway/visibility.go b/mod/gateway/visibility.go new file mode 100644 index 000000000..8dbecf5dc --- /dev/null +++ b/mod/gateway/visibility.go @@ -0,0 +1,10 @@ +package gateway + +import "github.com/cryptopunkscc/astrald/astral" + +type Visibility = astral.String8 + +const ( + VisibilityPublic Visibility = "public" + VisibilityPrivate Visibility = "private" +) diff --git a/mod/tcp/module.go b/mod/tcp/module.go index 315b2202d..8924ea18d 100644 --- a/mod/tcp/module.go +++ b/mod/tcp/module.go @@ -1,6 +1,7 @@ package tcp import ( + "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" ) @@ -11,4 +12,6 @@ type Module interface { exonet.Unpacker exonet.Parser ListenPort() int + + NewServer(port astral.Uint16, onAccept exonet.EphemeralHandler) exonet.EphemeralListener } diff --git a/mod/tcp/src/loader.go b/mod/tcp/src/loader.go index cc2368278..b1d0d59df 100644 --- a/mod/tcp/src/loader.go +++ b/mod/tcp/src/loader.go @@ -22,6 +22,8 @@ func (Loader) Load(node astral.Node, assets assets.Assets, l *log.Logger) (core. _ = assets.LoadYAML(tcp.ModuleName, &mod.config) + mod.ops.AddStructPrefix(mod, "Op") + for _, addr := range mod.config.Endpoints { addr, _ = strings.CutPrefix(addr, fmt.Sprintf("%s:", tcp.ModuleName)) diff --git a/mod/tcp/src/module.go b/mod/tcp/src/module.go index 130e2a7e3..c7cb8e81a 100644 --- a/mod/tcp/src/module.go +++ b/mod/tcp/src/module.go @@ -1,10 +1,13 @@ package tcp import ( + "context" + "sync" "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/astral/log" + "github.com/cryptopunkscc/astrald/lib/ops" "github.com/cryptopunkscc/astrald/mod/exonet" "github.com/cryptopunkscc/astrald/mod/nodes" "github.com/cryptopunkscc/astrald/mod/tcp" @@ -23,6 +26,10 @@ type Module struct { log *log.Logger ctx *astral.Context configEndpoints []exonet.Endpoint + ops ops.Set + + mu sync.Mutex + ephemeralListeners sig.Map[astral.Uint16, exonet.EphemeralListener] server sig.Switch } @@ -32,6 +39,23 @@ type Settings struct { Dial *tree.Value[*astral.Bool] `tree:"dial"` } +func (mod *Module) GetOpSet() *ops.Set { + return &mod.ops +} + +func (mod *Module) String() string { + return tcp.ModuleName +} + +func (mod *Module) acceptAll(ctx context.Context, conn exonet.Conn) (shouldStop bool, err error) { + err = mod.Nodes.EstablishInboundLink(ctx, conn) + if err != nil { + return false, err + } + + return false, nil +} + func (mod *Module) Run(ctx *astral.Context) (err error) { mod.ctx = ctx @@ -88,3 +112,7 @@ func (mod *Module) syncConfig(ctx *astral.Context) error { return nil } + +func (mod *Module) NewServer(port astral.Uint16, onAccept exonet.EphemeralHandler) exonet.EphemeralListener { + return NewServer(mod, port, onAccept) +} diff --git a/mod/tcp/src/server.go b/mod/tcp/src/server.go index 5e7af3142..24f76dede 100644 --- a/mod/tcp/src/server.go +++ b/mod/tcp/src/server.go @@ -2,62 +2,102 @@ package tcp import ( "context" + "fmt" "net" - "strconv" + "sync/atomic" "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" "github.com/cryptopunkscc/astrald/mod/tcp" ) +var _ exonet.EphemeralListener = &Server{} + type Server struct { *Module + listenPort astral.Uint16 + listener net.Listener + onAccept exonet.EphemeralHandler + closed atomic.Bool + closedCh chan struct{} } -func NewServer(module *Module) *Server { - return &Server{Module: module} +func NewServer(module *Module, listenPort astral.Uint16, onAccept exonet.EphemeralHandler) *Server { + return &Server{ + Module: module, + listenPort: listenPort, + onAccept: onAccept, + closedCh: make(chan struct{}), + } } -func (srv *Server) Run(ctx context.Context) error { - // start the listener - var addrStr = ":" + strconv.Itoa(srv.config.ListenPort) +func (s *Server) Run(ctx *astral.Context) error { + addr := fmt.Sprintf(":%d", s.listenPort) - listener, err := net.Listen("tcp", addrStr) + listener, err := net.Listen("tcp", addr) if err != nil { - srv.log.Errorv(0, "failed to start server: %v", err) - return err + return fmt.Errorf("tcp server/run: failed to listen on %v: %w", addr, err) } - endpoint, _ := tcp.ParseEndpoint(listener.Addr().String()) + s.listener = listener - srv.log.Info("started server at %v", endpoint) - defer srv.log.Info("stopped server at %v", endpoint) + endpoint, _ := tcp.ParseEndpoint(listener.Addr().String()) + s.log.Info("started server at %v", endpoint) go func() { - <-ctx.Done() - listener.Close() + select { + case <-ctx.Done(): + s.Close() + case <-s.Done(): + } }() - // accept connections for { rawConn, err := listener.Accept() if err != nil { - return err - } + if s.closed.Load() || ctx.Err() != nil { + s.log.Info("stopped server at %v", endpoint) + return nil + } - var conn = tcp.WrapConn(rawConn, false) + return fmt.Errorf("tcp server/run: accept failed: %w", err) + } + conn := tcp.WrapConn(rawConn, false) go func() { - err := srv.Nodes.EstablishInboundLink(ctx, conn) + shouldClose, err := s.onAccept(ctx, conn) if err != nil { - srv.log.Errorv(1, "handshake failed from %v: %v", conn.RemoteEndpoint(), err) + s.log.Errorv(1, "tcp server/onAccept error from %v: %v", conn.RemoteEndpoint(), err) return } + + if shouldClose { + s.Close() + } }() } } +func (s *Server) Done() <-chan struct{} { + return s.closedCh +} + +func (s *Server) Close() error { + if !s.closed.CompareAndSwap(false, true) { + return nil + } + + if s.listener != nil { + return s.listener.Close() + } + + close(s.closedCh) + return nil +} + func (mod *Module) startServer(ctx context.Context) { - srv := NewServer(mod) + listenPort := astral.Uint16(mod.config.ListenPort) + srv := NewServer(mod, listenPort, mod.acceptAll) if err := srv.Run(astral.NewContext(ctx)); err != nil { mod.log.Errorv(1, "server error: %v", err) } From 1f0e767c484d84a2e8602ff3845275f79558e579 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 00:22:32 +0000 Subject: [PATCH 02/42] mod/gateway: WIP --- mod/gateway/src/accept.go | 115 +++++++++++++++++++++ mod/gateway/src/bind.go | 45 ++------ mod/gateway/src/binder_conn_pool.go | 32 ------ mod/gateway/src/client.go | 109 +++++++++++++++++++ mod/gateway/src/config.go | 20 ++-- mod/gateway/src/conn.go | 15 +++ mod/gateway/src/connect.go | 30 ++---- mod/gateway/src/deps.go | 2 +- mod/gateway/src/dial.go | 44 ++++++++ mod/gateway/src/module.go | 28 +++-- mod/gateway/src/op_list.go | 6 +- mod/gateway/src/server.go | 106 ------------------- mod/tcp/client/client.go | 27 +++++ mod/tcp/client/close_ephemeral_listener.go | 25 +++++ mod/tcp/client/new_ephemeral_listener.go | 25 +++++ mod/tcp/module.go | 8 +- mod/tcp/src/ephemeral_listener.go | 44 ++++++++ mod/tcp/src/module.go | 10 +- mod/tcp/src/op_close_ephemeral_listener.go | 25 +++++ mod/tcp/src/op_new_ephemeral_listener.go | 25 +++++ 20 files changed, 510 insertions(+), 231 deletions(-) create mode 100644 mod/gateway/src/accept.go delete mode 100644 mod/gateway/src/binder_conn_pool.go create mode 100644 mod/gateway/src/client.go create mode 100644 mod/gateway/src/conn.go create mode 100644 mod/gateway/src/dial.go delete mode 100644 mod/gateway/src/server.go create mode 100644 mod/tcp/client/client.go create mode 100644 mod/tcp/client/close_ephemeral_listener.go create mode 100644 mod/tcp/client/new_ephemeral_listener.go create mode 100644 mod/tcp/src/ephemeral_listener.go create mode 100644 mod/tcp/src/op_close_ephemeral_listener.go create mode 100644 mod/tcp/src/op_new_ephemeral_listener.go diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go new file mode 100644 index 000000000..8ca3dc857 --- /dev/null +++ b/mod/gateway/src/accept.go @@ -0,0 +1,115 @@ +package gateway + +import ( + "context" + + "io" + "strings" + "sync" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/tcp" +) + +func (mod *Module) startServers(ctx *astral.Context) { + for _, addr := range mod.config.Gateway.Listen { + parts := strings.SplitN(addr, ":", 2) + if len(parts) != 2 { + mod.log.Error("invalid listen address: %v", addr) + continue + } + network, address := parts[0], parts[1] + + endpoint, err := mod.Exonet.Parse(network, address) + if err != nil { + mod.log.Error("parse listen address %v: %v", addr, err) + continue + } + + switch network { + case "tcp": + tcpEndpoint, ok := endpoint.(*tcp.Endpoint) + if !ok { + mod.log.Error("invalid listen address: %v", addr) + continue + } + + if err := mod.TCP.CreateEphemeralListener(ctx, tcpEndpoint.Port, mod.acceptSocketConn); err != nil { + mod.log.Error("create ephemeral listener on %v: %v", addr, err) + continue + } + + mod.listenEndpoints.Set("tcp", tcpEndpoint) + default: + mod.log.Error("unsupported gateway socket network: %v", network) + } + } +} + +// acceptSocketConn accepts connection on socket that gateway told client to connect to. +func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, error) { + var nonce astral.Nonce + if _, err := nonce.ReadFrom(conn); err != nil { + mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) + conn.Close() + return false, nil + } + + client, ok := mod.clientByNonce(nonce) + if !ok { + mod.log.Errorv(1, "unknown nonce %v from %v", nonce, conn.RemoteEndpoint()) + conn.Close() + return false, nil + } + + if client.isBinder() { + mod.log.Infov(1, "added idle conn to %v", client.Identity) + client.add(conn) + return false, nil + } + + // connecting + mod.connecting.Remove(client) + + binderConn := client.takePipeTo() + if binderConn == nil { + mod.log.Errorv(1, "no reserved conn for %v", client.Target) + conn.Close() + return false, nil + } + + connectorConn := &clientConn{ + Conn: conn, + network: conn.RemoteEndpoint().Network(), + } + client.conns.Add(connectorConn) + + if binder, ok := mod.binderByIdentity(client.Target); ok { + binder.markPiped(binderConn, connectorConn) + } + client.markPiped(connectorConn, binderConn) + + mod.log.Infov(1, "connecting %v to %v", client.Identity, client.Target) + go pipe(binderConn, connectorConn) + return false, nil +} + +func pipe(a, b io.ReadWriteCloser) { + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + io.Copy(a, b) + a.Close() + }() + + go func() { + defer wg.Done() + io.Copy(b, a) + b.Close() + }() + + wg.Wait() +} diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go index 816c062b4..c5e443268 100644 --- a/mod/gateway/src/bind.go +++ b/mod/gateway/src/bind.go @@ -5,13 +5,6 @@ import ( "github.com/cryptopunkscc/astrald/mod/gateway" ) -type Binder struct { - Identity *astral.Identity - Visibility gateway.Visibility - Nonce astral.Nonce - ConnPool *binderConnPool -} - func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibility gateway.Visibility, network string) (*gateway.Socket, error) { if !mod.canGateway(identity) { return nil, gateway.ErrUnauthorized @@ -24,43 +17,23 @@ func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibili nonce := astral.NewNonce() - binder := &Binder{ + client := &client{ Identity: identity, - Visibility: visibility, Nonce: nonce, - ConnPool: newBinderConnPool(mod), + Visibility: visibility, + Target: nil, // its a binder } - if old, ok := mod.binderByIdentity(identity); ok { - mod.binders.Remove(old) + oldClient, ok := mod.binders.Replace(identity.String(), client) + if ok { + err = oldClient.Close() + if err != nil { + mod.log.Error("failed to close oldClient client: %v", err) + } } - mod.binders.Add(binder) - return &gateway.Socket{ Nonce: nonce, Endpoint: endpoint, }, nil } - -func (mod *Module) canGateway(identity *astral.Identity) bool { - return mod.config.ActAsGateway -} - -func (mod *Module) binderByNonce(nonce astral.Nonce) (*Binder, bool) { - for _, b := range mod.binders.Clone() { - if b.Nonce == nonce { - return b, true - } - } - return nil, false -} - -func (mod *Module) binderByIdentity(identity *astral.Identity) (*Binder, bool) { - for _, b := range mod.binders.Clone() { - if b.Identity.IsEqual(identity) { - return b, true - } - } - return nil, false -} diff --git a/mod/gateway/src/binder_conn_pool.go b/mod/gateway/src/binder_conn_pool.go deleted file mode 100644 index a5a65bb8b..000000000 --- a/mod/gateway/src/binder_conn_pool.go +++ /dev/null @@ -1,32 +0,0 @@ -package gateway - -import ( - "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/sig" -) - -type binderConnPool struct { - *Module - - conns sig.Set[exonet.Conn] -} - -func newBinderConnPool(module *Module) *binderConnPool { - return &binderConnPool{Module: module} -} - -func (p *binderConnPool) add(conn exonet.Conn) { - p.conns.Add(conn) -} - -func (p *binderConnPool) take() (exonet.Conn, bool) { - items := p.conns.Clone() - if len(items) == 0 { - return nil, false - } - conn := items[0] - if err := p.conns.Remove(conn); err != nil { - return nil, false - } - return conn, true -} diff --git a/mod/gateway/src/client.go b/mod/gateway/src/client.go new file mode 100644 index 000000000..cbfbd454f --- /dev/null +++ b/mod/gateway/src/client.go @@ -0,0 +1,109 @@ +package gateway + +import ( + "errors" + "sync" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" + "github.com/cryptopunkscc/astrald/sig" +) + +type connState uint8 + +const ( + connStateIdle connState = iota + connStateReserved connState = iota + connStatePiped connState = iota +) + +type clientConn struct { + exonet.Conn + network string + state connState + pipedTo *clientConn // non-nil when connStatePiped +} + +type client struct { + mu sync.Mutex + + Identity *astral.Identity + Nonce astral.Nonce + Visibility gateway.Visibility + Target *astral.Identity // nil for binders, set for connecting + // + conns sig.Set[*clientConn] + pipeTo *clientConn // reserved binder conn for connecting clients +} + +func (c *client) isBinder() bool { + return c.Target == nil +} + +func (c *client) add(conn exonet.Conn) { + c.conns.Add(&clientConn{ + Conn: conn, + network: conn.RemoteEndpoint().Network(), + state: connStateIdle, + }) +} + +func (c *client) take() (*clientConn, bool) { + c.mu.Lock() + defer c.mu.Unlock() + for _, cc := range c.conns.Clone() { + if cc.state == connStateIdle { + cc.state = connStateReserved + return cc, true + } + } + return nil, false +} + +func (c *client) markPiped(cc, other *clientConn) { + c.mu.Lock() + defer c.mu.Unlock() + cc.state = connStatePiped + cc.pipedTo = other +} + +func (c *client) takePipeTo() *clientConn { + c.mu.Lock() + defer c.mu.Unlock() + cc := c.pipeTo + c.pipeTo = nil + return cc +} + +func (c *client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + + var errs []error + + for _, cc := range c.conns.Clone() { + errs = append(errs, cc.Close()) + } + + return errors.Join(errs...) +} + +func (mod *Module) binderByIdentity(identity *astral.Identity) (*client, bool) { + return mod.binders.Get(identity.String()) +} + +func (mod *Module) clientByNonce(nonce astral.Nonce) (*client, bool) { + for _, c := range mod.binders.Values() { + if c.Nonce == nonce { + return c, true + } + } + + for _, c := range mod.connecting.Clone() { + if c.Nonce == nonce { + return c, true + } + } + return nil, false +} diff --git a/mod/gateway/src/config.go b/mod/gateway/src/config.go index 045a3d7a0..d3388528b 100644 --- a/mod/gateway/src/config.go +++ b/mod/gateway/src/config.go @@ -4,26 +4,20 @@ import "github.com/cryptopunkscc/astrald/mod/gateway" const defaultGateway = "node1f3AwbE1gJAgAqEx98FMipokcaE9ZapIphzDUkAceE7Pmw8ghmFV19QKCATeC7uyoLszQA" -const ( - defaultInitConns = 1 - defaultMaxConns = 8 -) +type GatewayConfig struct { + Enabled bool `yaml:"enabled"` + Listen []string `yaml:"listen"` +} type Config struct { - ActAsGateway bool `yaml:"act_as_gateway"` - - Sockets map[string]uint16 `yaml:"sockets"` + Gateway GatewayConfig `yaml:"gateway"` Visibility gateway.Visibility `yaml:"visibility"` - Gateways []string `yaml:"gateways"` InitConns int32 `yaml:"init_conns"` MaxConns int32 `yaml:"max_conns"` } var defaultConfig = Config{ Visibility: gateway.VisibilityPublic, - Gateways: []string{ - defaultGateway, - }, - InitConns: defaultInitConns, - MaxConns: defaultMaxConns, + InitConns: 1, + MaxConns: 8, } diff --git a/mod/gateway/src/conn.go b/mod/gateway/src/conn.go new file mode 100644 index 000000000..bc14ad8c3 --- /dev/null +++ b/mod/gateway/src/conn.go @@ -0,0 +1,15 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +type gwConn struct { + exonet.Conn + remote *gateway.Endpoint +} + +func (c *gwConn) RemoteEndpoint() exonet.Endpoint { + return c.remote +} diff --git a/mod/gateway/src/connect.go b/mod/gateway/src/connect.go index a4430c588..e123765f8 100644 --- a/mod/gateway/src/connect.go +++ b/mod/gateway/src/connect.go @@ -5,12 +5,6 @@ import ( "github.com/cryptopunkscc/astrald/mod/gateway" ) -type Connecting struct { - Identity *astral.Identity - Target *astral.Identity - Nonce astral.Nonce -} - func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, network string) (socket gateway.Socket, err error) { if !mod.canGateway(caller) { return socket, gateway.ErrUnauthorized @@ -21,32 +15,28 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n return gateway.Socket{}, err } - nonce := astral.NewNonce() + binder, ok := mod.binderByIdentity(target) + if !ok { + return socket, gateway.ErrTargetNotReachable + } - _, ok := mod.binderByIdentity(target) + reserved, ok := binder.take() if !ok { return socket, gateway.ErrTargetNotReachable } - connecting := &Connecting{ + nonce := astral.NewNonce() + client := &client{ Identity: caller, - Target: target, Nonce: nonce, + Target: target, + pipeTo: reserved, } - mod.connecting.Add(connecting) + mod.connecting.Add(client) return gateway.Socket{ Nonce: nonce, Endpoint: endpoint, }, nil } - -func (mod *Module) connectingByNonce(nonce astral.Nonce) (*Connecting, bool) { - for _, c := range mod.connecting.Clone() { - if c.Nonce == nonce { - return c, true - } - } - return nil, false -} diff --git a/mod/gateway/src/deps.go b/mod/gateway/src/deps.go index 6efbf1123..960f3dbcb 100644 --- a/mod/gateway/src/deps.go +++ b/mod/gateway/src/deps.go @@ -11,7 +11,7 @@ func (mod *Module) LoadDependencies(*astral.Context) (err error) { return } - // mod.Exonet.SetDialer("gw", mod.dialer) + mod.Exonet.SetDialer("gw", mod) mod.Exonet.SetUnpacker("gw", mod) mod.Exonet.SetParser("gw", mod) mod.ops.AddStructPrefix(mod, "Op") diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go new file mode 100644 index 000000000..c7806f068 --- /dev/null +++ b/mod/gateway/src/dial.go @@ -0,0 +1,44 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/lib/astrald" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" + gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" +) + +var _ exonet.Dialer = &Module{} + +func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.Conn, error) { + if endpoint.Network() != NetworkName { + return nil, exonet.ErrUnsupportedNetwork + } + + gwEndpoint, ok := endpoint.(*gateway.Endpoint) + if !ok { + return nil, exonet.ErrUnsupportedNetwork + } + + client := gatewayClient.New(gwEndpoint.GatewayID, astrald.Default()) + + socket, err := client.Connect(ctx, gwEndpoint.TargetID) + if err != nil { + return nil, err + } + + conn, err := mod.Exonet.Dial(ctx, socket.Endpoint) + if err != nil { + return nil, err + } + + if _, err := socket.Nonce.WriteTo(conn); err != nil { + conn.Close() + return nil, err + } + + return &gwConn{ + Conn: conn, + remote: gwEndpoint, + }, nil +} diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index 72cd4fc62..6fc454cda 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -33,8 +33,8 @@ type Module struct { log *log.Logger ctx *astral.Context - binders sig.Set[*Binder] - connecting sig.Set[*Connecting] + binders sig.Map[string, *client] + connecting sig.Set[*client] listenEndpoints sig.Map[string, exonet.Endpoint] } @@ -46,19 +46,19 @@ func (mod *Module) GetOpSet() *ops.Set { func (mod *Module) Run(ctx *astral.Context) error { mod.ctx = ctx.IncludeZone(astral.ZoneNetwork) - mod.startServers(mod.ctx) + if mod.config.Gateway.Enabled { + mod.startServers(mod.ctx) + } - for _, gatewayStr := range mod.config.Gateways { - identity, err := mod.Dir.ResolveIdentity(gatewayStr) - if err != nil { - mod.log.Error("resolve gateway %v: %v", gatewayStr, err) - continue - } + <-ctx.Done() - go mod.bindToGateway(mod.ctx, identity, mod.config.Visibility) + for _, c := range mod.binders.Values() { + c.Close() + } + for _, c := range mod.connecting.Clone() { + c.Close() } - <-ctx.Done() return nil } @@ -74,8 +74,14 @@ func (mod *Module) getGatewayEndpoint(ctx *astral.Context, network string) (endp // fixme: return public error (no gateway endpoint available) return } + return endpoint, nil } + +func (mod *Module) canGateway(identity *astral.Identity) bool { + return mod.config.Gateway.Enabled +} + func (mod *Module) String() string { return ModuleName } diff --git a/mod/gateway/src/op_list.go b/mod/gateway/src/op_list.go index fdb214c5d..2e3285801 100644 --- a/mod/gateway/src/op_list.go +++ b/mod/gateway/src/op_list.go @@ -15,11 +15,11 @@ func (mod *Module) OpList(ctx *astral.Context, q *ops.Query, args opListArgs) er ch := q.AcceptChannel(channel.WithOutputFormat(args.Out)) defer ch.Close() - for _, binder := range mod.binders.Clone() { - if binder.Visibility != gateway.VisibilityPublic { + for _, client := range mod.binders.Values() { + if client.Visibility != gateway.VisibilityPublic { continue } - if err := ch.Send(binder.Identity); err != nil { + if err := ch.Send(client.Identity); err != nil { return err } } diff --git a/mod/gateway/src/server.go b/mod/gateway/src/server.go deleted file mode 100644 index 217fa7af3..000000000 --- a/mod/gateway/src/server.go +++ /dev/null @@ -1,106 +0,0 @@ -package gateway - -import ( - "context" - "io" - - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/mod/exonet" - tcpmod "github.com/cryptopunkscc/astrald/mod/tcp" -) - -func (mod *Module) startServers(ctx *astral.Context) { - for network, port := range mod.config.Sockets { - switch network { - case "tcp": - endpoint, err := mod.startTCPServer(ctx, port) - if err != nil { - mod.log.Error("start gateway tcp server on port %v: %v", port, err) - continue - } - mod.listenEndpoints.Set(network, endpoint) - default: - mod.log.Error("unsupported gateway socket network: %v", network) - } - } -} - -func (mod *Module) startTCPServer(ctx *astral.Context, port uint16) (*tcpmod.Endpoint, error) { - server := mod.TCP.NewServer(astral.Uint16(port), mod.acceptConn) - - endpoint := &tcpmod.Endpoint{Port: astral.Uint16(port)} - ips, _ := mod.IP.LocalIPs() - for _, ip := range ips { - if !ip.IsLoopback() { - endpoint.IP = ip - break - } - } - if endpoint.IP == nil && len(ips) > 0 { - endpoint.IP = ips[0] - } - - go func() { - if err := server.Run(ctx); err != nil { - mod.log.Error("gateway tcp server: %v", err) - } - }() - - return endpoint, nil -} - -func (mod *Module) acceptConn(_ context.Context, conn exonet.Conn) (bool, error) { - var nonce astral.Nonce - if _, err := nonce.ReadFrom(conn); err != nil { - mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) - conn.Close() - return false, nil - } - - if binder, ok := mod.binderByNonce(nonce); ok { - mod.log.Infov(1, "slot conn from %v via %v", binder.Identity, conn.RemoteEndpoint()) - binder.ConnPool.add(conn) - return false, nil - } - - if connecting, ok := mod.connectingByNonce(nonce); ok { - mod.connecting.Remove(connecting) - - binder, ok := mod.binderByIdentity(connecting.Target) - if !ok { - mod.log.Errorv(1, "no binder for %v", connecting.Target) - conn.Close() - return false, nil - } - - binderConnection, ok := binder.ConnPool.take() - if !ok { - mod.log.Errorv(1, "no available binderConnection for %v", connecting.Target) - conn.Close() - return false, nil - } - - mod.log.Infov(1, "connecting %v to %v", connecting.Identity, connecting.Target) - - go func() { - defer binderConnection.Close() - defer conn.Close() - pipe(binderConnection, conn) - }() - return false, nil - } - - mod.log.Errorv(1, "unknown nonce %v from %v", nonce, conn.RemoteEndpoint()) - conn.Close() - return false, nil -} - -func pipe(a, b io.ReadWriteCloser) { - done := make(chan struct{}) - go func() { - defer close(done) - io.Copy(a, b) - }() - io.Copy(b, a) - <-done -} diff --git a/mod/tcp/client/client.go b/mod/tcp/client/client.go new file mode 100644 index 000000000..96a78e3d3 --- /dev/null +++ b/mod/tcp/client/client.go @@ -0,0 +1,27 @@ +package tcp + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/astrald" +) + +type Client struct { + astral *astrald.Client + targetID *astral.Identity +} + +func New(targetID *astral.Identity, a *astrald.Client) *Client { + if a == nil { + a = astrald.Default() + } + return &Client{astral: a, targetID: targetID} +} + +func (client *Client) WithTarget(target *astral.Identity) *Client { + return &Client{astral: client.astral, targetID: target} +} + +func (client *Client) queryCh(ctx *astral.Context, method string, args any, cfg ...channel.ConfigFunc) (*channel.Channel, error) { + return client.astral.WithTarget(client.targetID).QueryChannel(ctx, method, args, cfg...) +} diff --git a/mod/tcp/client/close_ephemeral_listener.go b/mod/tcp/client/close_ephemeral_listener.go new file mode 100644 index 000000000..d6a5e036d --- /dev/null +++ b/mod/tcp/client/close_ephemeral_listener.go @@ -0,0 +1,25 @@ +package tcp + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/query" + "github.com/cryptopunkscc/astrald/mod/tcp" +) + +func (client *Client) CloseEphemeralListener(ctx *astral.Context, port astral.Uint16) error { + ch, err := client.queryCh(ctx, tcp.MethodCloseEphemeralListener, query.Args{ + "port": port, + }) + if err != nil { + return err + } + defer ch.Close() + + return ch.Switch( + channel.ExpectAck, + func(msg *astral.ErrorMessage) error { + return msg + }, + ) +} diff --git a/mod/tcp/client/new_ephemeral_listener.go b/mod/tcp/client/new_ephemeral_listener.go new file mode 100644 index 000000000..108d684de --- /dev/null +++ b/mod/tcp/client/new_ephemeral_listener.go @@ -0,0 +1,25 @@ +package tcp + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/query" + "github.com/cryptopunkscc/astrald/mod/tcp" +) + +func (client *Client) CreateEphemeralListener(ctx *astral.Context, port astral.Uint16) error { + ch, err := client.queryCh(ctx, tcp.MethodNewEphemeralListener, query.Args{ + "port": port, + }) + if err != nil { + return err + } + defer ch.Close() + + return ch.Switch( + channel.ExpectAck, + func(msg *astral.ErrorMessage) error { + return msg + }, + ) +} diff --git a/mod/tcp/module.go b/mod/tcp/module.go index 8924ea18d..5d649612c 100644 --- a/mod/tcp/module.go +++ b/mod/tcp/module.go @@ -7,11 +7,15 @@ import ( const ModuleName = "tcp" +const ( + MethodNewEphemeralListener = "tcp.new_ephemeral_listener" + MethodCloseEphemeralListener = "tcp.close_ephemeral_listener" +) + type Module interface { exonet.Dialer exonet.Unpacker exonet.Parser ListenPort() int - - NewServer(port astral.Uint16, onAccept exonet.EphemeralHandler) exonet.EphemeralListener + CreateEphemeralListener(ctx *astral.Context, port astral.Uint16, handler exonet.EphemeralHandler) error } diff --git a/mod/tcp/src/ephemeral_listener.go b/mod/tcp/src/ephemeral_listener.go new file mode 100644 index 000000000..ac90cd0b0 --- /dev/null +++ b/mod/tcp/src/ephemeral_listener.go @@ -0,0 +1,44 @@ +package tcp + +import ( + "fmt" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/tcp" +) + +func (mod *Module) CreateEphemeralListener(ctx *astral.Context, port astral.Uint16, handler exonet.EphemeralHandler) error { + mod.mu.Lock() + defer mod.mu.Unlock() + + if _, ok := mod.ephemeralListeners.Get(port); ok { + return fmt.Errorf("%w: port %v", tcp.ErrEphemeralListenerExists, port) + } + + srv := NewServer(mod, port, handler) + mod.ephemeralListeners.Set(port, srv) + + go func() { + err := srv.Run(ctx) + if err != nil { + mod.log.Error("ephemeral listener error: %v", err) + } + + mod.ephemeralListeners.Delete(port) + }() + + return nil +} + +func (mod *Module) CloseEphemeralListener(port astral.Uint16) error { + listener, ok := mod.ephemeralListeners.Get(port) + if !ok { + return tcp.ErrEphemeralListenerNotExist + } + + listener.Close() + mod.ephemeralListeners.Delete(port) + + return nil +} diff --git a/mod/tcp/src/module.go b/mod/tcp/src/module.go index c7cb8e81a..b2a9a831f 100644 --- a/mod/tcp/src/module.go +++ b/mod/tcp/src/module.go @@ -28,10 +28,10 @@ type Module struct { configEndpoints []exonet.Endpoint ops ops.Set - mu sync.Mutex - ephemeralListeners sig.Map[astral.Uint16, exonet.EphemeralListener] + mu sync.Mutex - server sig.Switch + server sig.Switch + ephemeralListeners sig.Map[astral.Uint16, exonet.EphemeralListener] } type Settings struct { @@ -112,7 +112,3 @@ func (mod *Module) syncConfig(ctx *astral.Context) error { return nil } - -func (mod *Module) NewServer(port astral.Uint16, onAccept exonet.EphemeralHandler) exonet.EphemeralListener { - return NewServer(mod, port, onAccept) -} diff --git a/mod/tcp/src/op_close_ephemeral_listener.go b/mod/tcp/src/op_close_ephemeral_listener.go new file mode 100644 index 000000000..1ed7179e2 --- /dev/null +++ b/mod/tcp/src/op_close_ephemeral_listener.go @@ -0,0 +1,25 @@ +package tcp + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/ops" +) + +type opCloseEphemeralListenerArgs struct { + Port astral.Uint16 + In string `query:"optional"` + Out string `query:"optional"` +} + +func (mod *Module) OpCloseEphemeralListener(ctx *astral.Context, q *ops.Query, args opCloseEphemeralListenerArgs) (err error) { + ch := channel.New(q.Accept(), channel.WithFormats(args.In, args.Out)) + defer ch.Close() + + err = mod.CloseEphemeralListener(args.Port) + if err != nil { + return ch.Send(astral.NewError(err.Error())) + } + + return ch.Send(&astral.Ack{}) +} diff --git a/mod/tcp/src/op_new_ephemeral_listener.go b/mod/tcp/src/op_new_ephemeral_listener.go new file mode 100644 index 000000000..bb649ffc4 --- /dev/null +++ b/mod/tcp/src/op_new_ephemeral_listener.go @@ -0,0 +1,25 @@ +package tcp + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/channel" + "github.com/cryptopunkscc/astrald/lib/ops" +) + +type opNewEphemeralListenerArgs struct { + Port astral.Uint16 + In string `query:"optional"` + Out string `query:"optional"` +} + +func (mod *Module) OpNewEphemeralListener(ctx *astral.Context, q *ops.Query, args opNewEphemeralListenerArgs) (err error) { + ch := channel.New(q.Accept(), channel.WithFormats(args.In, args.Out)) + defer ch.Close() + + err = mod.CreateEphemeralListener(ctx, args.Port, mod.acceptAll) + if err != nil { + return ch.Send(astral.NewError(err.Error())) + } + + return ch.Send(&astral.Ack{}) +} From 6ae241f2b67bc7065b5f83665dd0b92c865704a3 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 16:04:23 +0000 Subject: [PATCH 03/42] mod/gateway: WIP --- mod/gateway/README.md | 16 +++++ mod/gateway/src/bind_gateway.go | 20 ------ mod/gateway/src/binding.go | 92 +++++++++++++++++++++++++++ mod/gateway/src/loader.go | 7 +- mod/gateway/src/module.go | 3 +- mod/gateway/src/receiver_conn_pool.go | 78 ----------------------- 6 files changed, 113 insertions(+), 103 deletions(-) create mode 100644 mod/gateway/README.md delete mode 100644 mod/gateway/src/bind_gateway.go create mode 100644 mod/gateway/src/binding.go delete mode 100644 mod/gateway/src/receiver_conn_pool.go diff --git a/mod/gateway/README.md b/mod/gateway/README.md new file mode 100644 index 000000000..be77fd38a --- /dev/null +++ b/mod/gateway/README.md @@ -0,0 +1,16 @@ +# gateway + +## Configuration + +`gateway.yaml`: + +```yaml +gateway: + enabled: true + listen: + - tcp::6000 + +visibility: public +init_conns: 1 +max_conns: 8 +``` diff --git a/mod/gateway/src/bind_gateway.go b/mod/gateway/src/bind_gateway.go deleted file mode 100644 index 74c497080..000000000 --- a/mod/gateway/src/bind_gateway.go +++ /dev/null @@ -1,20 +0,0 @@ -package gateway - -import ( - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/lib/astrald" - "github.com/cryptopunkscc/astrald/mod/gateway" - gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" -) - -func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity, visibility gateway.Visibility) { - client := gatewayClient.New(gatewayID, astrald.Default()) - - socket, err := client.Bind(ctx, visibility) - if err != nil { - mod.log.Error("bind to %v: %v", gatewayID, err) - return - } - - newReceiverConnPool(mod, socket).Run(ctx) -} diff --git a/mod/gateway/src/binding.go b/mod/gateway/src/binding.go new file mode 100644 index 000000000..3f3b261d8 --- /dev/null +++ b/mod/gateway/src/binding.go @@ -0,0 +1,92 @@ +package gateway + +import ( + "sync/atomic" + + "github.com/cryptopunkscc/astrald/astral" + libastrald "github.com/cryptopunkscc/astrald/lib/astrald" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" + gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" +) + +func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity, visibility gateway.Visibility) { + client := gatewayClient.New(gatewayID, libastrald.Default()) + + socket, err := client.Bind(ctx, visibility) + if err != nil { + mod.log.Error("bind to %v: %v", gatewayID, err) + return + } + + newGatewayBinding(mod, socket).Run(ctx) +} + +type gatewayBinding struct { + *Module + socket *gateway.Socket + count atomic.Int32 +} + +func newGatewayBinding(module *Module, socket *gateway.Socket) *gatewayBinding { + return &gatewayBinding{Module: module, socket: socket} +} + +func (b *gatewayBinding) Run(ctx *astral.Context) { + for range b.config.InitConns { + b.spawn(ctx) + } + + <-ctx.Done() +} + +func (b *gatewayBinding) spawn(ctx *astral.Context) { + if b.count.Add(1) > b.config.MaxConns { + b.count.Add(-1) + b.log.Error("max connections reached (%v), cannot spawn new slot", b.config.MaxConns) + return + } + + go func() { + defer b.count.Add(-1) + b.hold(ctx) + }() +} + +func (b *gatewayBinding) hold(ctx *astral.Context) { + conn, err := b.Exonet.Dial(ctx, b.socket.Endpoint) + if err != nil { + return + } + + // Authenticate with the gateway + if _, err = b.socket.Nonce.WriteTo(conn); err != nil { + b.log.Errorv(1, "nonce write to %v failed: %v", conn.RemoteEndpoint(), err) + return + } + + // Wrap conn: on first incoming byte, spawn a replacement slot; pass all bytes through untouched + slot := &triggerConn{ + Conn: conn, + onFirst: func() { b.spawn(ctx) }, + } + + if err = b.Nodes.EstablishInboundLink(ctx, slot); err != nil { + b.log.Errorv(1, "inbound link from %v failed: %v", conn.RemoteEndpoint(), err) + } +} + +// triggerConn wraps an exonet.Conn and calls onFirst exactly once on the first incoming bytes. +type triggerConn struct { + exonet.Conn + triggered atomic.Bool + onFirst func() +} + +func (c *triggerConn) Read(p []byte) (int, error) { + n, err := c.Conn.Read(p) + if n > 0 && c.triggered.CompareAndSwap(false, true) { + c.onFirst() + } + return n, err +} diff --git a/mod/gateway/src/loader.go b/mod/gateway/src/loader.go index e07170879..c7bea3e59 100644 --- a/mod/gateway/src/loader.go +++ b/mod/gateway/src/loader.go @@ -6,10 +6,9 @@ import ( "github.com/cryptopunkscc/astrald/core" "github.com/cryptopunkscc/astrald/core/assets" "github.com/cryptopunkscc/astrald/lib/routers" + "github.com/cryptopunkscc/astrald/mod/gateway" ) -const ModuleName = "gateway" - type Loader struct{} func (Loader) Load(node astral.Node, assets assets.Assets, log *log.Logger) (core.Module, error) { @@ -20,13 +19,13 @@ func (Loader) Load(node astral.Node, assets assets.Assets, log *log.Logger) (cor config: defaultConfig, } - _ = assets.LoadYAML(ModuleName, &mod.config) + _ = assets.LoadYAML(gateway.ModuleName, &mod.config) return mod, nil } func init() { - if err := core.RegisterModule(ModuleName, Loader{}); err != nil { + if err := core.RegisterModule(gateway.ModuleName, Loader{}); err != nil { panic(err) } } diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index 6fc454cda..b8fb608d4 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -7,6 +7,7 @@ import ( "github.com/cryptopunkscc/astrald/lib/routers" "github.com/cryptopunkscc/astrald/mod/dir" "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" ipmod "github.com/cryptopunkscc/astrald/mod/ip" "github.com/cryptopunkscc/astrald/mod/nodes" tcpmod "github.com/cryptopunkscc/astrald/mod/tcp" @@ -83,5 +84,5 @@ func (mod *Module) canGateway(identity *astral.Identity) bool { } func (mod *Module) String() string { - return ModuleName + return gateway.ModuleName } diff --git a/mod/gateway/src/receiver_conn_pool.go b/mod/gateway/src/receiver_conn_pool.go deleted file mode 100644 index 331aca4ce..000000000 --- a/mod/gateway/src/receiver_conn_pool.go +++ /dev/null @@ -1,78 +0,0 @@ -package gateway - -import ( - "sync/atomic" - - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" -) - -type receiverConnPool struct { - *Module - socket *gateway.Socket - count atomic.Int32 -} - -func newReceiverConnPool(module *Module, socket *gateway.Socket) *receiverConnPool { - return &receiverConnPool{Module: module, socket: socket} -} - -func (pool *receiverConnPool) Run(ctx *astral.Context) { - for range pool.config.InitConns { - pool.spawn(ctx) - } - - <-ctx.Done() -} - -func (pool *receiverConnPool) spawn(ctx *astral.Context) { - if pool.count.Add(1) > pool.config.MaxConns { - pool.count.Add(-1) - pool.log.Error("max connections reached (%v), cannot spawn new slot", pool.config.MaxConns) - return - } - - go func() { - defer pool.count.Add(-1) - pool.hold(ctx) - }() -} - -func (pool *receiverConnPool) hold(ctx *astral.Context) { - conn, err := pool.Exonet.Dial(ctx, pool.socket.Endpoint) - if err != nil { - return - } - - // Authenticate with the gateway - if _, err = pool.socket.Nonce.WriteTo(conn); err != nil { - pool.log.Errorv(1, "nonce write to %v failed: %v", conn.RemoteEndpoint(), err) - return - } - - // Wrap conn: on first incoming byte, spawn a replacement slot; pass all bytes through untouched - slot := &slotConn{ - Conn: conn, - onFirst: func() { pool.spawn(ctx) }, - } - - if err = pool.Nodes.EstablishInboundLink(ctx, slot); err != nil { - pool.log.Errorv(1, "inbound link from %v failed: %v", conn.RemoteEndpoint(), err) - } -} - -// slotConn wraps an exonet.Conn and calls onFirst exactly once on the first incoming bytes. -type slotConn struct { - exonet.Conn - triggered atomic.Bool - onFirst func() -} - -func (c *slotConn) Read(p []byte) (int, error) { - n, err := c.Conn.Read(p) - if n > 0 && c.triggered.CompareAndSwap(false, true) { - c.onFirst() - } - return n, err -} From 47a074d15f72854614b2f4805a3784661bc9aab4 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 16:05:42 +0000 Subject: [PATCH 04/42] mod/tcp: add errors related to ephemeral --- mod/tcp/errors.go | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 mod/tcp/errors.go diff --git a/mod/tcp/errors.go b/mod/tcp/errors.go new file mode 100644 index 000000000..3ad3a4e80 --- /dev/null +++ b/mod/tcp/errors.go @@ -0,0 +1,6 @@ +package tcp + +import "errors" + +var ErrEphemeralListenerExists = errors.New("ephemeral listener already exists") +var ErrEphemeralListenerNotExist = errors.New("ephemeral listener not exists") From 962ea94a7ce1632b3146ecc30b5e2718fcc8a776 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 16:15:40 +0000 Subject: [PATCH 05/42] mod/gateway: add log --- mod/gateway/src/accept.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 8ca3dc857..732795e6d 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -35,6 +35,7 @@ func (mod *Module) startServers(ctx *astral.Context) { continue } + mod.log.Logv(1, "start listening on %v", tcpEndpoint) if err := mod.TCP.CreateEphemeralListener(ctx, tcpEndpoint.Port, mod.acceptSocketConn); err != nil { mod.log.Error("create ephemeral listener on %v: %v", addr, err) continue @@ -88,6 +89,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, if binder, ok := mod.binderByIdentity(client.Target); ok { binder.markPiped(binderConn, connectorConn) } + client.markPiped(connectorConn, binderConn) mod.log.Infov(1, "connecting %v to %v", client.Identity, client.Target) From 5e7829af2249e4932c62de3272aef5fa98134311 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 16:30:10 +0000 Subject: [PATCH 06/42] mod/gateway: binding to gateways from config --- mod/gateway/src/binding.go | 9 +++++---- mod/gateway/src/config.go | 9 +++++++-- mod/gateway/src/module.go | 4 ++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/mod/gateway/src/binding.go b/mod/gateway/src/binding.go index 3f3b261d8..8d6d97d09 100644 --- a/mod/gateway/src/binding.go +++ b/mod/gateway/src/binding.go @@ -10,16 +10,17 @@ import ( gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" ) -func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity, visibility gateway.Visibility) { +func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity, visibility gateway.Visibility) (err error) { client := gatewayClient.New(gatewayID, libastrald.Default()) socket, err := client.Bind(ctx, visibility) if err != nil { - mod.log.Error("bind to %v: %v", gatewayID, err) - return + return err } - newGatewayBinding(mod, socket).Run(ctx) + go newGatewayBinding(mod, socket).Run(ctx) + + return nil } type gatewayBinding struct { diff --git a/mod/gateway/src/config.go b/mod/gateway/src/config.go index d3388528b..ee607da0d 100644 --- a/mod/gateway/src/config.go +++ b/mod/gateway/src/config.go @@ -1,6 +1,9 @@ package gateway -import "github.com/cryptopunkscc/astrald/mod/gateway" +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/gateway" +) const defaultGateway = "node1f3AwbE1gJAgAqEx98FMipokcaE9ZapIphzDUkAceE7Pmw8ghmFV19QKCATeC7uyoLszQA" @@ -10,7 +13,9 @@ type GatewayConfig struct { } type Config struct { - Gateway GatewayConfig `yaml:"gateway"` + Gateway GatewayConfig `yaml:"gateway"` + Gateways []*astral.Identity `yaml:"gateways"` + Visibility gateway.Visibility `yaml:"visibility"` InitConns int32 `yaml:"init_conns"` MaxConns int32 `yaml:"max_conns"` diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index b8fb608d4..944bc4943 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -60,6 +60,10 @@ func (mod *Module) Run(ctx *astral.Context) error { c.Close() } + for _, gw := range mod.config.Gateways { + mod.bindToGateway(ctx, gw, mod.config.Visibility) + } + return nil } From 415279e4751189d2f628f55de22a0d0f0e4dfdf5 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 16:46:24 +0000 Subject: [PATCH 07/42] mod/gateway: fix suffix --- mod/gateway/module.go | 6 +++--- mod/gateway/src/binding.go | 11 ++++++++++- mod/gateway/src/module.go | 11 +++++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/mod/gateway/module.go b/mod/gateway/module.go index 593b87e27..7c54ecfca 100644 --- a/mod/gateway/module.go +++ b/mod/gateway/module.go @@ -3,7 +3,7 @@ package gateway const ModuleName = "gateway" const ( - MethodBind = "gateway.node.bind" - MethodConnect = "gateway.node.connect" - MethodList = "gateway.node.list" + MethodBind = "gateway.node_bind" + MethodConnect = "gateway.node_connect" + MethodList = "gateway.node_list" ) diff --git a/mod/gateway/src/binding.go b/mod/gateway/src/binding.go index 8d6d97d09..f96d90f86 100644 --- a/mod/gateway/src/binding.go +++ b/mod/gateway/src/binding.go @@ -1,6 +1,7 @@ package gateway import ( + "fmt" "sync/atomic" "github.com/cryptopunkscc/astrald/astral" @@ -11,13 +12,18 @@ import ( ) func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity, visibility gateway.Visibility) (err error) { + mod.log.Logv(1, "binding to gateway %v", gatewayID) client := gatewayClient.New(gatewayID, libastrald.Default()) - socket, err := client.Bind(ctx, visibility) + socket, err := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), visibility) if err != nil { + fmt.Println("BIND FAILED: ", err) return err } + fmt.Println("GOT SOCKET: ", socket) + // todo: if we lose connection to gateway (e.g reboot) we should try to rebind + go newGatewayBinding(mod, socket).Run(ctx) return nil @@ -60,12 +66,15 @@ func (b *gatewayBinding) hold(ctx *astral.Context) { return } + fmt.Println("DIALED GW") // Authenticate with the gateway if _, err = b.socket.Nonce.WriteTo(conn); err != nil { b.log.Errorv(1, "nonce write to %v failed: %v", conn.RemoteEndpoint(), err) return } + fmt.Println("WROTE NONCE") + // Wrap conn: on first incoming byte, spawn a replacement slot; pass all bytes through untouched slot := &triggerConn{ Conn: conn, diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index 944bc4943..d905afd3a 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -51,6 +51,13 @@ func (mod *Module) Run(ctx *astral.Context) error { mod.startServers(mod.ctx) } + for _, gw := range mod.config.Gateways { + err := mod.bindToGateway(ctx, gw, mod.config.Visibility) + if err != nil { + mod.log.Error("failed to bind to gateway: %v", err) + } + } + <-ctx.Done() for _, c := range mod.binders.Values() { @@ -60,10 +67,6 @@ func (mod *Module) Run(ctx *astral.Context) error { c.Close() } - for _, gw := range mod.config.Gateways { - mod.bindToGateway(ctx, gw, mod.config.Visibility) - } - return nil } From edd1a65b9be8f39afed5eeec55edb0c2782c7ff0 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 17:03:44 +0000 Subject: [PATCH 08/42] mod/gateway: add logging --- mod/gateway/src/accept.go | 2 ++ mod/gateway/src/binding.go | 33 +++++++++++++++++---------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 732795e6d..f7c6d563f 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -50,6 +50,8 @@ func (mod *Module) startServers(ctx *astral.Context) { // acceptSocketConn accepts connection on socket that gateway told client to connect to. func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, error) { + mod.log.Infov(1, "accepting connection from %v", conn.RemoteEndpoint()) + var nonce astral.Nonce if _, err := nonce.ReadFrom(conn); err != nil { mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) diff --git a/mod/gateway/src/binding.go b/mod/gateway/src/binding.go index f96d90f86..c670774ab 100644 --- a/mod/gateway/src/binding.go +++ b/mod/gateway/src/binding.go @@ -17,11 +17,9 @@ func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity socket, err := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), visibility) if err != nil { - fmt.Println("BIND FAILED: ", err) return err } - fmt.Println("GOT SOCKET: ", socket) // todo: if we lose connection to gateway (e.g reboot) we should try to rebind go newGatewayBinding(mod, socket).Run(ctx) @@ -41,49 +39,52 @@ func newGatewayBinding(module *Module, socket *gateway.Socket) *gatewayBinding { func (b *gatewayBinding) Run(ctx *astral.Context) { for range b.config.InitConns { - b.spawn(ctx) + b.spawnConnection(ctx) } <-ctx.Done() } -func (b *gatewayBinding) spawn(ctx *astral.Context) { +func (b *gatewayBinding) spawnConnection(ctx *astral.Context) { if b.count.Add(1) > b.config.MaxConns { b.count.Add(-1) - b.log.Error("max connections reached (%v), cannot spawn new slot", b.config.MaxConns) + b.log.Error("max connections reached (%v), cannot spawnConnection new slot", b.config.MaxConns) return } go func() { defer b.count.Add(-1) - b.hold(ctx) + err := b.hold(ctx) + if err != nil { + b.log.Error("handling incoming connection failed: %v", err) + } }() } -func (b *gatewayBinding) hold(ctx *astral.Context) { +func (b *gatewayBinding) hold(ctx *astral.Context) error { + b.log.Logv(1, "connecting to gateway socket %v", b.socket.Endpoint) conn, err := b.Exonet.Dial(ctx, b.socket.Endpoint) if err != nil { - return + return err } - fmt.Println("DIALED GW") // Authenticate with the gateway if _, err = b.socket.Nonce.WriteTo(conn); err != nil { - b.log.Errorv(1, "nonce write to %v failed: %v", conn.RemoteEndpoint(), err) - return + return fmt.Errorf("nonce write to %v failed: %v", conn.RemoteEndpoint(), err) } - fmt.Println("WROTE NONCE") - - // Wrap conn: on first incoming byte, spawn a replacement slot; pass all bytes through untouched + // Wrap conn: on the first incoming byte, spawnConnection a replacement slot; pass all bytes through untouched slot := &triggerConn{ Conn: conn, - onFirst: func() { b.spawn(ctx) }, + onFirst: func() { b.spawnConnection(ctx) }, } + // fixme: i dont know if this is blocking if err = b.Nodes.EstablishInboundLink(ctx, slot); err != nil { - b.log.Errorv(1, "inbound link from %v failed: %v", conn.RemoteEndpoint(), err) + return fmt.Errorf("inbound link from %v failed: %v", conn.RemoteEndpoint(), err) } + + return nil } // triggerConn wraps an exonet.Conn and calls onFirst exactly once on the first incoming bytes. From 8ddf60549749b5e1a6baf4c1f85cf228c465d7a9 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Wed, 11 Mar 2026 17:31:48 +0000 Subject: [PATCH 09/42] mod/gateway: add ZoneNetwork when dialing --- mod/gateway/src/dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go index c7806f068..b6bf817e0 100644 --- a/mod/gateway/src/dial.go +++ b/mod/gateway/src/dial.go @@ -22,7 +22,7 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C client := gatewayClient.New(gwEndpoint.GatewayID, astrald.Default()) - socket, err := client.Connect(ctx, gwEndpoint.TargetID) + socket, err := client.Connect(ctx.IncludeZone(astral.ZoneNetwork), gwEndpoint.TargetID) if err != nil { return nil, err } From 82c0dec39039388273903e25f4f988f445eeb5f5 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 01:21:00 +0000 Subject: [PATCH 10/42] mod/gateway: WIP --- mod/gateway/errors.go | 1 + mod/gateway/src/accept.go | 11 ++++++++--- mod/gateway/src/bind.go | 16 ++++++++++++---- mod/gateway/src/binding.go | 9 +++++---- mod/gateway/src/connect.go | 22 ++++++++++++++++++++++ mod/gateway/src/dial.go | 6 ++++++ 6 files changed, 54 insertions(+), 11 deletions(-) diff --git a/mod/gateway/errors.go b/mod/gateway/errors.go index b109c8825..6bf69cd47 100644 --- a/mod/gateway/errors.go +++ b/mod/gateway/errors.go @@ -4,3 +4,4 @@ import "errors" var ErrUnauthorized = errors.New("unauthorized") var ErrTargetNotReachable = errors.New("target not reachable") +var ErrInvalidGateway = errors.New("invalid gateway") diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index f7c6d563f..7a40d27a8 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -48,7 +48,7 @@ func (mod *Module) startServers(ctx *astral.Context) { } } -// acceptSocketConn accepts connection on socket that gateway told client to connect to. +// acceptSocketConn accepts connection on the socket that gateway told client to connect to. func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, error) { mod.log.Infov(1, "accepting connection from %v", conn.RemoteEndpoint()) @@ -86,12 +86,17 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, Conn: conn, network: conn.RemoteEndpoint().Network(), } + client.conns.Add(connectorConn) - if binder, ok := mod.binderByIdentity(client.Target); ok { - binder.markPiped(binderConn, connectorConn) + targetClient, ok := mod.binderByIdentity(client.Target) + if !ok { + // fixme: return public error () + return false, nil } + targetClient.markPiped(binderConn, connectorConn) + client.markPiped(connectorConn, binderConn) mod.log.Infov(1, "connecting %v to %v", client.Identity, client.Target) diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go index c5e443268..4d38f498a 100644 --- a/mod/gateway/src/bind.go +++ b/mod/gateway/src/bind.go @@ -21,14 +21,22 @@ func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibili Identity: identity, Nonce: nonce, Visibility: visibility, - Target: nil, // its a binder + Target: nil, } oldClient, ok := mod.binders.Replace(identity.String(), client) if ok { - err = oldClient.Close() - if err != nil { - mod.log.Error("failed to close oldClient client: %v", err) + if err = oldClient.Close(); err != nil { + mod.log.Error("failed to close old client: %v", err) + } + + targetID := oldClient.Identity.String() + for _, c := range mod.connecting.Clone() { + if c.Target.String() == targetID { + if c.takePipeTo() != nil { + mod.connecting.Remove(c) + } + } } } diff --git a/mod/gateway/src/binding.go b/mod/gateway/src/binding.go index c670774ab..92eaed9f8 100644 --- a/mod/gateway/src/binding.go +++ b/mod/gateway/src/binding.go @@ -22,6 +22,8 @@ func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity // todo: if we lose connection to gateway (e.g reboot) we should try to rebind + // note: schedule task similiar to nodes/maintain_link_task.go (which for duration of mod.ctx will try to maintain connection to gateway.Socket) + go newGatewayBinding(mod, socket).Run(ctx) return nil @@ -73,14 +75,13 @@ func (b *gatewayBinding) hold(ctx *astral.Context) error { return fmt.Errorf("nonce write to %v failed: %v", conn.RemoteEndpoint(), err) } - // Wrap conn: on the first incoming byte, spawnConnection a replacement slot; pass all bytes through untouched - slot := &triggerConn{ + // On first incoming bytes, spawn a new connection + replenishingConn := &triggerConn{ Conn: conn, onFirst: func() { b.spawnConnection(ctx) }, } - // fixme: i dont know if this is blocking - if err = b.Nodes.EstablishInboundLink(ctx, slot); err != nil { + if err = b.Nodes.EstablishInboundLink(ctx, replenishingConn); err != nil { return fmt.Errorf("inbound link from %v failed: %v", conn.RemoteEndpoint(), err) } diff --git a/mod/gateway/src/connect.go b/mod/gateway/src/connect.go index e123765f8..3fa5148ee 100644 --- a/mod/gateway/src/connect.go +++ b/mod/gateway/src/connect.go @@ -1,10 +1,14 @@ package gateway import ( + "time" + "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/gateway" ) +const connectTimeout = 30 * time.Second + func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, network string) (socket gateway.Socket, err error) { if !mod.canGateway(caller) { return socket, gateway.ErrUnauthorized @@ -22,6 +26,7 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n reserved, ok := binder.take() if !ok { + // fixme: return public err ErrCannotConnect return socket, gateway.ErrTargetNotReachable } @@ -35,6 +40,23 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n mod.connecting.Add(client) + go func() { + select { + case <-time.After(connectTimeout): + } + + binderConn := client.takePipeTo() + if binderConn == nil { + return + } + + mod.connecting.Remove(client) + err = binderConn.Close() + if err != nil { + mod.log.Error("failed to close binderConn: %v", err) + } + }() + return gateway.Socket{ Nonce: nonce, Endpoint: endpoint, diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go index b6bf817e0..023aafb4e 100644 --- a/mod/gateway/src/dial.go +++ b/mod/gateway/src/dial.go @@ -20,6 +20,10 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C return nil, exonet.ErrUnsupportedNetwork } + if gwEndpoint.GatewayID.IsEqual(mod.node.Identity()) { + return nil, gateway.ErrInvalidGateway + } + client := gatewayClient.New(gwEndpoint.GatewayID, astrald.Default()) socket, err := client.Connect(ctx.IncludeZone(astral.ZoneNetwork), gwEndpoint.TargetID) @@ -27,6 +31,8 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C return nil, err } + // todo: if we cannot obtain socket we should try to connect to gateway and route to target over link + conn, err := mod.Exonet.Dial(ctx, socket.Endpoint) if err != nil { return nil, err From 89191f21109637b6ab74c968c77d7d61a3c8522f Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 01:40:54 +0000 Subject: [PATCH 11/42] mod/gateway: WIP --- astral/router.go | 3 +- mod/gateway/module.go | 1 + mod/gateway/src/conn.go | 17 +++++++---- mod/gateway/src/dial.go | 38 ++++++++++++++++++------- mod/gateway/src/module.go | 5 ++++ mod/gateway/src/route.go | 60 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 mod/gateway/src/route.go diff --git a/astral/router.go b/astral/router.go index b83ba2f14..344215f6e 100644 --- a/astral/router.go +++ b/astral/router.go @@ -1,7 +1,6 @@ package astral import ( - "context" "io" ) @@ -9,4 +8,4 @@ type Router interface { RouteQuery(ctx *Context, q *Query, w io.WriteCloser) (io.WriteCloser, error) } -type RouteQueryFunc func(ctx context.Context, q *Query, w io.WriteCloser) (io.WriteCloser, error) +type RouteQueryFunc func(ctx *Context, q *Query, w io.WriteCloser) (io.WriteCloser, error) diff --git a/mod/gateway/module.go b/mod/gateway/module.go index 7c54ecfca..fa9929eb0 100644 --- a/mod/gateway/module.go +++ b/mod/gateway/module.go @@ -6,4 +6,5 @@ const ( MethodBind = "gateway.node_bind" MethodConnect = "gateway.node_connect" MethodList = "gateway.node_list" + MethodRoute = "gateway.route" ) diff --git a/mod/gateway/src/conn.go b/mod/gateway/src/conn.go index bc14ad8c3..2a7b0e886 100644 --- a/mod/gateway/src/conn.go +++ b/mod/gateway/src/conn.go @@ -1,15 +1,20 @@ package gateway import ( + "io" + "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" ) +var _ exonet.Conn = (*gwConn)(nil) + type gwConn struct { - exonet.Conn - remote *gateway.Endpoint + io.ReadWriteCloser + local exonet.Endpoint + remote exonet.Endpoint + outbound bool } -func (c *gwConn) RemoteEndpoint() exonet.Endpoint { - return c.remote -} +func (c *gwConn) LocalEndpoint() exonet.Endpoint { return c.local } +func (c *gwConn) RemoteEndpoint() exonet.Endpoint { return c.remote } +func (c gwConn) Outbound() bool { return c.outbound } diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go index 023aafb4e..8d5063a77 100644 --- a/mod/gateway/src/dial.go +++ b/mod/gateway/src/dial.go @@ -3,6 +3,7 @@ package gateway import ( "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/lib/astrald" + "github.com/cryptopunkscc/astrald/lib/query" "github.com/cryptopunkscc/astrald/mod/exonet" "github.com/cryptopunkscc/astrald/mod/gateway" gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" @@ -26,25 +27,40 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C client := gatewayClient.New(gwEndpoint.GatewayID, astrald.Default()) - socket, err := client.Connect(ctx.IncludeZone(astral.ZoneNetwork), gwEndpoint.TargetID) - if err != nil { - return nil, err + // Fast path: socket-based (raw TCP piped through gateway) + if socket, err := client.Connect(ctx.IncludeZone(astral.ZoneNetwork), gwEndpoint.TargetID); err == nil { + if conn, err := mod.Exonet.Dial(ctx, socket.Endpoint); err == nil { + if _, err = socket.Nonce.WriteTo(conn); err == nil { + return &gwConn{ + ReadWriteCloser: conn, + local: conn.LocalEndpoint(), + remote: gwEndpoint, + outbound: conn.Outbound(), + }, nil + } + conn.Close() + } } - // todo: if we cannot obtain socket we should try to connect to gateway and route to target over link + // Slow path: link-based (route query through existing astral links) + mod.log.Logv(1, "socket path unavailable, trying link path to %v via %v", gwEndpoint.TargetID, gwEndpoint.GatewayID) - conn, err := mod.Exonet.Dial(ctx, socket.Endpoint) - if err != nil { - return nil, err + q := &astral.Query{ + Nonce: astral.NewNonce(), + Caller: mod.node.Identity(), + Target: gwEndpoint.GatewayID, + Query: gateway.MethodRoute + "." + gwEndpoint.TargetID.String(), } - if _, err := socket.Nonce.WriteTo(conn); err != nil { - conn.Close() + conn, err := query.Route(ctx, mod.node, q) + if err != nil { return nil, err } return &gwConn{ - Conn: conn, - remote: gwEndpoint, + ReadWriteCloser: conn, + local: gateway.NewEndpoint(mod.node.Identity(), mod.node.Identity()), + remote: gwEndpoint, + outbound: true, }, nil } diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index d905afd3a..d6d219347 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -47,6 +47,11 @@ func (mod *Module) GetOpSet() *ops.Set { func (mod *Module) Run(ctx *astral.Context) error { mod.ctx = ctx.IncludeZone(astral.ZoneNetwork) + err := mod.AddRoute(gateway.MethodRoute+".*", routers.Func(mod.routeQuery)) + if err != nil { + return err + } + if mod.config.Gateway.Enabled { mod.startServers(mod.ctx) } diff --git a/mod/gateway/src/route.go b/mod/gateway/src/route.go new file mode 100644 index 000000000..e7aa7f6a5 --- /dev/null +++ b/mod/gateway/src/route.go @@ -0,0 +1,60 @@ +package gateway + +import ( + "context" + "io" + "strings" + "time" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/lib/query" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +const acceptTimeout = 30 * time.Second + +func (mod *Module) routeQuery(ctx *astral.Context, q *astral.Query, w io.WriteCloser) (io.WriteCloser, error) { + var targetKey string + switch { + case strings.HasPrefix(q.Query, gateway.MethodRoute+"."): + targetKey, _ = strings.CutPrefix(q.Query, gateway.MethodRoute+".") + default: + return query.Reject() + } + + // target is us + if targetKey == mod.node.Identity().String() { + return query.Accept(q, w, func(conn astral.Conn) { + c := &gwConn{ + ReadWriteCloser: conn, + local: gateway.NewEndpoint(q.Target, q.Target), + remote: gateway.NewEndpoint(q.Caller, q.Target), + outbound: false, + } + + actx, cancel := context.WithTimeout(context.Background(), acceptTimeout) + defer cancel() + + if err := mod.Nodes.EstablishInboundLink(actx, c); err != nil { + mod.log.Errorv(1, "inbound link from %v failed: %v", q.Caller, err) + } + }) + } + + // forward query (will automatically use existing link) + + targetIdentity, err := astral.ParseIdentity(targetKey) + if err != nil { + return query.Reject() + } + + nextQuery := &astral.Query{ + Nonce: astral.NewNonce(), + Caller: mod.node.Identity(), + Target: targetIdentity, + Query: q.Query, + } + + mod.log.Logv(2, "routing %v to %v via link", q.Caller, targetIdentity) + return mod.node.RouteQuery(ctx, nextQuery, w) +} From d4bae6375007d40a99a1ac3f7b1082e0942f2ac0 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 01:58:01 +0000 Subject: [PATCH 12/42] mod/gateway: WIP --- mod/gateway/src/binding.go | 18 ++++++------------ mod/gateway/src/config.go | 9 ++------- mod/gateway/src/dial.go | 35 +++++++++++++++++++++-------------- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/mod/gateway/src/binding.go b/mod/gateway/src/binding.go index 92eaed9f8..c3b05db80 100644 --- a/mod/gateway/src/binding.go +++ b/mod/gateway/src/binding.go @@ -21,8 +21,7 @@ func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity } // todo: if we lose connection to gateway (e.g reboot) we should try to rebind - - // note: schedule task similiar to nodes/maintain_link_task.go (which for duration of mod.ctx will try to maintain connection to gateway.Socket) + // note: maybe schedule task similiar to nodes/maintain_link_task.go (which for duration of mod.ctx will try to maintain connection to gateway.Socket) go newGatewayBinding(mod, socket).Run(ctx) @@ -40,30 +39,25 @@ func newGatewayBinding(module *Module, socket *gateway.Socket) *gatewayBinding { } func (b *gatewayBinding) Run(ctx *astral.Context) { - for range b.config.InitConns { - b.spawnConnection(ctx) - } + b.spawnConnection(ctx) <-ctx.Done() } func (b *gatewayBinding) spawnConnection(ctx *astral.Context) { - if b.count.Add(1) > b.config.MaxConns { - b.count.Add(-1) - b.log.Error("max connections reached (%v), cannot spawnConnection new slot", b.config.MaxConns) - return - } + b.log.Logv(1, "spawning connection to gateway socket %v", b.socket.Endpoint) + b.count.Add(1) go func() { defer b.count.Add(-1) - err := b.hold(ctx) + err := b.connectToGatewaySocket(ctx) if err != nil { b.log.Error("handling incoming connection failed: %v", err) } }() } -func (b *gatewayBinding) hold(ctx *astral.Context) error { +func (b *gatewayBinding) connectToGatewaySocket(ctx *astral.Context) error { b.log.Logv(1, "connecting to gateway socket %v", b.socket.Endpoint) conn, err := b.Exonet.Dial(ctx, b.socket.Endpoint) if err != nil { diff --git a/mod/gateway/src/config.go b/mod/gateway/src/config.go index ee607da0d..9bf2170fe 100644 --- a/mod/gateway/src/config.go +++ b/mod/gateway/src/config.go @@ -13,16 +13,11 @@ type GatewayConfig struct { } type Config struct { - Gateway GatewayConfig `yaml:"gateway"` - Gateways []*astral.Identity `yaml:"gateways"` - + Gateway GatewayConfig `yaml:"gateway"` Visibility gateway.Visibility `yaml:"visibility"` - InitConns int32 `yaml:"init_conns"` - MaxConns int32 `yaml:"max_conns"` + Gateways []*astral.Identity `yaml:"gateways"` } var defaultConfig = Config{ Visibility: gateway.VisibilityPublic, - InitConns: 1, - MaxConns: 8, } diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go index 8d5063a77..87bc17da2 100644 --- a/mod/gateway/src/dial.go +++ b/mod/gateway/src/dial.go @@ -26,23 +26,30 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C } client := gatewayClient.New(gwEndpoint.GatewayID, astrald.Default()) + socket, err := client.Connect(ctx.IncludeZone(astral.ZoneNetwork), gwEndpoint.TargetID) + if err != nil { + return mod.route(ctx, gwEndpoint) + } + + conn, err := mod.Exonet.Dial(ctx, socket.Endpoint) + if err != nil { + return mod.route(ctx, gwEndpoint) + } - // Fast path: socket-based (raw TCP piped through gateway) - if socket, err := client.Connect(ctx.IncludeZone(astral.ZoneNetwork), gwEndpoint.TargetID); err == nil { - if conn, err := mod.Exonet.Dial(ctx, socket.Endpoint); err == nil { - if _, err = socket.Nonce.WriteTo(conn); err == nil { - return &gwConn{ - ReadWriteCloser: conn, - local: conn.LocalEndpoint(), - remote: gwEndpoint, - outbound: conn.Outbound(), - }, nil - } - conn.Close() - } + if _, err := socket.Nonce.WriteTo(conn); err != nil { + conn.Close() + return mod.route(ctx, gwEndpoint) } - // Slow path: link-based (route query through existing astral links) + return &gwConn{ + ReadWriteCloser: conn, + local: conn.LocalEndpoint(), + remote: gwEndpoint, + outbound: conn.Outbound(), + }, nil +} + +func (mod *Module) route(ctx *astral.Context, gwEndpoint *gateway.Endpoint) (exonet.Conn, error) { mod.log.Logv(1, "socket path unavailable, trying link path to %v via %v", gwEndpoint.TargetID, gwEndpoint.GatewayID) q := &astral.Query{ From 42025eba7d0c90de5ab794f8e10d868ac77d0cc2 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 02:06:44 +0000 Subject: [PATCH 13/42] mod/gateway: proper closing conns --- mod/gateway/src/accept.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 7a40d27a8..5e205d5aa 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -5,7 +5,6 @@ import ( "io" "strings" - "sync" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" @@ -56,14 +55,14 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, if _, err := nonce.ReadFrom(conn); err != nil { mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) conn.Close() - return false, nil + return true, nil } client, ok := mod.clientByNonce(nonce) if !ok { mod.log.Errorv(1, "unknown nonce %v from %v", nonce, conn.RemoteEndpoint()) conn.Close() - return false, nil + return true, nil } if client.isBinder() { @@ -74,12 +73,11 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, // connecting mod.connecting.Remove(client) - binderConn := client.takePipeTo() if binderConn == nil { mod.log.Errorv(1, "no reserved conn for %v", client.Target) conn.Close() - return false, nil + return true, nil } connectorConn := &clientConn{ @@ -91,12 +89,10 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, targetClient, ok := mod.binderByIdentity(client.Target) if !ok { - // fixme: return public error () - return false, nil + return true, nil } targetClient.markPiped(binderConn, connectorConn) - client.markPiped(connectorConn, binderConn) mod.log.Infov(1, "connecting %v to %v", client.Identity, client.Target) @@ -105,20 +101,20 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, } func pipe(a, b io.ReadWriteCloser) { - var wg sync.WaitGroup - wg.Add(2) + done := make(chan struct{}, 2) + defer close(done) go func() { - defer wg.Done() io.Copy(a, b) - a.Close() + done <- struct{}{} }() go func() { - defer wg.Done() io.Copy(b, a) - b.Close() + done <- struct{}{} }() - wg.Wait() + <-done + a.Close() + b.Close() } From 532f048431de46149d7b4fdb957347c486e7fd53 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 13:57:11 +0000 Subject: [PATCH 14/42] mod/gateway: WIP --- mod/gateway/maintain_binding_task.go | 7 ++ mod/gateway/src/binding.go | 98 ------------------- mod/gateway/src/maintain_binding_task.go | 118 +++++++++++++++++++++++ mod/gateway/src/module.go | 18 ++-- 4 files changed, 134 insertions(+), 107 deletions(-) create mode 100644 mod/gateway/maintain_binding_task.go delete mode 100644 mod/gateway/src/binding.go create mode 100644 mod/gateway/src/maintain_binding_task.go diff --git a/mod/gateway/maintain_binding_task.go b/mod/gateway/maintain_binding_task.go new file mode 100644 index 000000000..7877bdab3 --- /dev/null +++ b/mod/gateway/maintain_binding_task.go @@ -0,0 +1,7 @@ +package gateway + +import "github.com/cryptopunkscc/astrald/mod/scheduler" + +type MaintainBindingTask interface { + scheduler.Task +} diff --git a/mod/gateway/src/binding.go b/mod/gateway/src/binding.go deleted file mode 100644 index c3b05db80..000000000 --- a/mod/gateway/src/binding.go +++ /dev/null @@ -1,98 +0,0 @@ -package gateway - -import ( - "fmt" - "sync/atomic" - - "github.com/cryptopunkscc/astrald/astral" - libastrald "github.com/cryptopunkscc/astrald/lib/astrald" - "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" - gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" -) - -func (mod *Module) bindToGateway(ctx *astral.Context, gatewayID *astral.Identity, visibility gateway.Visibility) (err error) { - mod.log.Logv(1, "binding to gateway %v", gatewayID) - client := gatewayClient.New(gatewayID, libastrald.Default()) - - socket, err := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), visibility) - if err != nil { - return err - } - - // todo: if we lose connection to gateway (e.g reboot) we should try to rebind - // note: maybe schedule task similiar to nodes/maintain_link_task.go (which for duration of mod.ctx will try to maintain connection to gateway.Socket) - - go newGatewayBinding(mod, socket).Run(ctx) - - return nil -} - -type gatewayBinding struct { - *Module - socket *gateway.Socket - count atomic.Int32 -} - -func newGatewayBinding(module *Module, socket *gateway.Socket) *gatewayBinding { - return &gatewayBinding{Module: module, socket: socket} -} - -func (b *gatewayBinding) Run(ctx *astral.Context) { - b.spawnConnection(ctx) - - <-ctx.Done() -} - -func (b *gatewayBinding) spawnConnection(ctx *astral.Context) { - b.log.Logv(1, "spawning connection to gateway socket %v", b.socket.Endpoint) - b.count.Add(1) - - go func() { - defer b.count.Add(-1) - err := b.connectToGatewaySocket(ctx) - if err != nil { - b.log.Error("handling incoming connection failed: %v", err) - } - }() -} - -func (b *gatewayBinding) connectToGatewaySocket(ctx *astral.Context) error { - b.log.Logv(1, "connecting to gateway socket %v", b.socket.Endpoint) - conn, err := b.Exonet.Dial(ctx, b.socket.Endpoint) - if err != nil { - return err - } - - // Authenticate with the gateway - if _, err = b.socket.Nonce.WriteTo(conn); err != nil { - return fmt.Errorf("nonce write to %v failed: %v", conn.RemoteEndpoint(), err) - } - - // On first incoming bytes, spawn a new connection - replenishingConn := &triggerConn{ - Conn: conn, - onFirst: func() { b.spawnConnection(ctx) }, - } - - if err = b.Nodes.EstablishInboundLink(ctx, replenishingConn); err != nil { - return fmt.Errorf("inbound link from %v failed: %v", conn.RemoteEndpoint(), err) - } - - return nil -} - -// triggerConn wraps an exonet.Conn and calls onFirst exactly once on the first incoming bytes. -type triggerConn struct { - exonet.Conn - triggered atomic.Bool - onFirst func() -} - -func (c *triggerConn) Read(p []byte) (int, error) { - n, err := c.Conn.Read(p) - if n > 0 && c.triggered.CompareAndSwap(false, true) { - c.onFirst() - } - return n, err -} diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go new file mode 100644 index 000000000..1babb0c50 --- /dev/null +++ b/mod/gateway/src/maintain_binding_task.go @@ -0,0 +1,118 @@ +package gateway + +import ( + "sync/atomic" + "time" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/lib/astrald" + "github.com/cryptopunkscc/astrald/mod/events" + "github.com/cryptopunkscc/astrald/mod/gateway" + gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" + "github.com/cryptopunkscc/astrald/mod/nodes" + "github.com/cryptopunkscc/astrald/mod/scheduler" + "github.com/cryptopunkscc/astrald/sig" +) + +var _ scheduler.Task = &MaintainBindingTask{} +var _ scheduler.EventReceiver = &MaintainBindingTask{} + +type MaintainBindingTask struct { + mod *Module + GatewayID *astral.Identity + Visibility gateway.Visibility + wake chan struct{} + actionRequired atomic.Bool +} + +func (mod *Module) NewMaintainBindingTask(gatewayID *astral.Identity, visibility gateway.Visibility) *MaintainBindingTask { + return &MaintainBindingTask{ + mod: mod, + GatewayID: gatewayID, + Visibility: visibility, + wake: make(chan struct{}, 1), + } +} + +func (task *MaintainBindingTask) String() string { + return "maintain_binding_task" +} + +func (task *MaintainBindingTask) Run(ctx *astral.Context) error { + task.mod.log.Log("starting to maintain binding to %v", task.GatewayID) + + retry, err := sig.NewRetry(time.Second, 15*time.Minute, 2) + if err != nil { + return err + } + + count := -1 + task.actionRequired.Store(true) + + for { + for !task.actionRequired.Load() { + select { + case <-ctx.Done(): + return ctx.Err() + case <-task.wake: + } + } + + switch { + case count == 0: + task.mod.log.Log("binding to %v lost, rebinding", task.GatewayID) + case count > 0 && count%5 == 0: + task.mod.log.Log("still trying to bind to %v (attempt %v)", task.GatewayID, count) + } + + client := gatewayClient.New(task.GatewayID, astrald.Default()) + socket, bindErr := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), task.Visibility) + if bindErr != nil { + count = <-retry.Retry() + continue + } + + retry.Reset() + if count > 0 { + task.mod.log.Log("rebound to %v after %v attempts", task.GatewayID, count) + } else if count < 0 { + task.mod.log.Log("bound to %v", task.GatewayID) + } + count = 0 + task.actionRequired.Store(false) + + task.maintainSocketConnections(ctx, socket) + + if ctx.Err() != nil { + return ctx.Err() + } + + // socket dead — trigger rebind + task.actionRequired.Store(true) + select { + case task.wake <- struct{}{}: + default: + } + } +} + +func (task *MaintainBindingTask) maintainSocketConnections(ctx *astral.Context, socket *gateway.Socket) { + if err := newSocketPool(task.mod, socket).Run(ctx); err == errSocketUnreachable { + task.mod.log.Log("gateway socket %v unreachable, will rebind", socket.Endpoint) + } +} + +func (task *MaintainBindingTask) ReceiveEvent(e *events.Event) { + switch typed := e.Data.(type) { + case *nodes.StreamClosedEvent: + if !typed.RemoteIdentity.IsEqual(task.GatewayID) || typed.StreamCount != 0 { + return + } + if !task.actionRequired.Swap(true) { + select { + case task.wake <- struct{}{}: + default: + } + } + } +} diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index d6d219347..2d15fd9c3 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -10,6 +10,7 @@ import ( "github.com/cryptopunkscc/astrald/mod/gateway" ipmod "github.com/cryptopunkscc/astrald/mod/ip" "github.com/cryptopunkscc/astrald/mod/nodes" + "github.com/cryptopunkscc/astrald/mod/scheduler" tcpmod "github.com/cryptopunkscc/astrald/mod/tcp" "github.com/cryptopunkscc/astrald/sig" ) @@ -17,11 +18,12 @@ import ( const NetworkName = "gw" type Deps struct { - Dir dir.Module - Exonet exonet.Module - Nodes nodes.Module - TCP tcpmod.Module - IP ipmod.Module + Dir dir.Module + Exonet exonet.Module + Nodes nodes.Module + Scheduler scheduler.Module + TCP tcpmod.Module + IP ipmod.Module } type Module struct { @@ -56,11 +58,9 @@ func (mod *Module) Run(ctx *astral.Context) error { mod.startServers(mod.ctx) } + <-mod.Scheduler.Ready() for _, gw := range mod.config.Gateways { - err := mod.bindToGateway(ctx, gw, mod.config.Visibility) - if err != nil { - mod.log.Error("failed to bind to gateway: %v", err) - } + mod.Scheduler.Schedule(mod.NewMaintainBindingTask(gw, mod.config.Visibility)) } <-ctx.Done() From 52caa31d2265b2d2b3255ff88d245071986aa284 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 14:27:35 +0000 Subject: [PATCH 15/42] mod/gateway: WIP --- mod/gateway/src/accept.go | 6 +- mod/gateway/src/bind.go | 4 +- mod/gateway/src/client.go | 6 +- mod/gateway/src/connect.go | 4 +- mod/gateway/src/maintain_binding_task.go | 2 +- mod/gateway/src/module.go | 6 +- mod/gateway/src/socket_pool.go | 189 +++++++++++++++++++++++ 7 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 mod/gateway/src/socket_pool.go diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 5e205d5aa..7d8dd9adc 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -71,8 +71,8 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, return false, nil } - // connecting - mod.connecting.Remove(client) + // clients + mod.clients.Remove(client) binderConn := client.takePipeTo() if binderConn == nil { mod.log.Errorv(1, "no reserved conn for %v", client.Target) @@ -95,7 +95,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, targetClient.markPiped(binderConn, connectorConn) client.markPiped(connectorConn, binderConn) - mod.log.Infov(1, "connecting %v to %v", client.Identity, client.Target) + mod.log.Infov(1, "clients %v to %v", client.Identity, client.Target) go pipe(binderConn, connectorConn) return false, nil } diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go index 4d38f498a..5d9b70955 100644 --- a/mod/gateway/src/bind.go +++ b/mod/gateway/src/bind.go @@ -31,10 +31,10 @@ func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibili } targetID := oldClient.Identity.String() - for _, c := range mod.connecting.Clone() { + for _, c := range mod.clients.Clone() { if c.Target.String() == targetID { if c.takePipeTo() != nil { - mod.connecting.Remove(c) + mod.clients.Remove(c) } } } diff --git a/mod/gateway/src/client.go b/mod/gateway/src/client.go index cbfbd454f..b8e688937 100644 --- a/mod/gateway/src/client.go +++ b/mod/gateway/src/client.go @@ -31,10 +31,10 @@ type client struct { Identity *astral.Identity Nonce astral.Nonce Visibility gateway.Visibility - Target *astral.Identity // nil for binders, set for connecting + Target *astral.Identity // nil for binders, set for clients // conns sig.Set[*clientConn] - pipeTo *clientConn // reserved binder conn for connecting clients + pipeTo *clientConn // reserved binder conn for clients clients } func (c *client) isBinder() bool { @@ -100,7 +100,7 @@ func (mod *Module) clientByNonce(nonce astral.Nonce) (*client, bool) { } } - for _, c := range mod.connecting.Clone() { + for _, c := range mod.clients.Clone() { if c.Nonce == nonce { return c, true } diff --git a/mod/gateway/src/connect.go b/mod/gateway/src/connect.go index 3fa5148ee..ba864831b 100644 --- a/mod/gateway/src/connect.go +++ b/mod/gateway/src/connect.go @@ -38,7 +38,7 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n pipeTo: reserved, } - mod.connecting.Add(client) + mod.clients.Add(client) go func() { select { @@ -50,7 +50,7 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n return } - mod.connecting.Remove(client) + mod.clients.Remove(client) err = binderConn.Close() if err != nil { mod.log.Error("failed to close binderConn: %v", err) diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go index 1babb0c50..5453d5545 100644 --- a/mod/gateway/src/maintain_binding_task.go +++ b/mod/gateway/src/maintain_binding_task.go @@ -97,7 +97,7 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { } func (task *MaintainBindingTask) maintainSocketConnections(ctx *astral.Context, socket *gateway.Socket) { - if err := newSocketPool(task.mod, socket).Run(ctx); err == errSocketUnreachable { + if err := newSocketPool(ctx, task.mod, socket).Run(); err == ErrSocketUnreachable { task.mod.log.Log("gateway socket %v unreachable, will rebind", socket.Endpoint) } } diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index 2d15fd9c3..d8211d036 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -36,8 +36,8 @@ type Module struct { log *log.Logger ctx *astral.Context - binders sig.Map[string, *client] - connecting sig.Set[*client] + binders sig.Map[string, *client] + clients sig.Set[*client] listenEndpoints sig.Map[string, exonet.Endpoint] } @@ -68,7 +68,7 @@ func (mod *Module) Run(ctx *astral.Context) error { for _, c := range mod.binders.Values() { c.Close() } - for _, c := range mod.connecting.Clone() { + for _, c := range mod.clients.Clone() { c.Close() } diff --git a/mod/gateway/src/socket_pool.go b/mod/gateway/src/socket_pool.go new file mode 100644 index 000000000..3b11243be --- /dev/null +++ b/mod/gateway/src/socket_pool.go @@ -0,0 +1,189 @@ +package gateway + +import ( + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/astral/log" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" + "github.com/cryptopunkscc/astrald/mod/nodes" + "github.com/cryptopunkscc/astrald/sig" +) + +const ( + socketPoolTargetIdle = 1 + socketPoolMaxFails = 3 +) + +var ErrSocketUnreachable = errors.New("socket unreachable") + +type SocketPool struct { + ctx *astral.Context + socket *gateway.Socket + exonet exonet.Module + nodes nodes.Module + log *log.Logger + + mu sync.Mutex + total int + idle int + + signal chan struct{} +} + +func newSocketPool(ctx *astral.Context, mod *Module, socket *gateway.Socket) *SocketPool { + return &SocketPool{ + ctx: ctx, + socket: socket, + exonet: mod.Exonet, + nodes: mod.Nodes, + log: mod.log, + signal: make(chan struct{}, 1), + } +} + +func (p *SocketPool) Run() error { + retry, _ := sig.NewRetry(time.Second, 2*time.Minute, 2) + failStreak := 0 + + p.notify() + + for { + select { + case <-p.ctx.Done(): + return p.ctx.Err() + case <-p.signal: + for p.idleCount() < socketPoolTargetIdle { + conn, err := p.acquireSocketConnection() + if err != nil { + failStreak++ + if failStreak >= socketPoolMaxFails { + p.log.Log("gateway socket %v unreachable", p.socket.Endpoint) + return ErrSocketUnreachable + } + select { + case <-p.ctx.Done(): + return p.ctx.Err() + case <-retry.Retry(): + } + continue + } + + failStreak = 0 + retry.Reset() + p.attach(conn) + } + } + } +} + +func (p *SocketPool) acquireSocketConnection() (exonet.Conn, error) { + conn, err := p.exonet.Dial(p.ctx, p.socket.Endpoint) + if err != nil { + return nil, err + } + if _, err := p.socket.Nonce.WriteTo(conn); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +func (p *SocketPool) addIdle() (total, idle int) { + p.mu.Lock() + defer p.mu.Unlock() + p.total++ + p.idle++ + return p.total, p.idle +} + +func (p *SocketPool) connTaken() (total, idle int) { + p.mu.Lock() + p.idle-- + total, idle = p.total, p.idle + p.mu.Unlock() + p.notify() + return +} + +func (p *SocketPool) onClosedConn(wasUsed bool) (total, idle int) { + p.mu.Lock() + p.total-- + if !wasUsed { + p.idle-- + } + total, idle = p.total, p.idle + p.mu.Unlock() + if !wasUsed { + p.notify() + } + return +} + +func (p *SocketPool) idleCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.idle +} + +func (p *SocketPool) attach(conn exonet.Conn) { + total, idle := p.addIdle() + p.log.Logv(1, "gateway conn up (total: %v, idle: %v)", total, idle) + + pc := &socketConn{Conn: conn} + + pc.onFirst = func() { + total, idle := p.connTaken() + p.log.Logv(1, "gateway conn taken (total: %v, idle: %v)", total, idle) + } + + pc.onClose = func() { + total, idle := p.onClosedConn(pc.used.Load()) + p.log.Logv(1, "gateway conn down (total: %v, idle: %v)", total, idle) + } + + go func() { + if err := p.nodes.EstablishInboundLink(p.ctx, pc); err != nil { + p.log.Logv(1, "inbound link from %v: %v", conn.RemoteEndpoint(), err) + } + }() +} + +func (p *SocketPool) notify() { + select { + case p.signal <- struct{}{}: + default: + } +} + +type socketConn struct { + exonet.Conn + + onFirst func() + onClose func() + + used atomic.Bool +} + +func (c *socketConn) Read(b []byte) (int, error) { + if !c.used.Swap(true) && c.onFirst != nil { + c.onFirst() + } + return c.Conn.Read(b) +} + +func (c *socketConn) Write(b []byte) (int, error) { + return c.Conn.Write(b) +} + +func (c *socketConn) Close() error { + err := c.Conn.Close() + if c.onClose != nil { + c.onClose() + } + return err +} From f70d5bb678d48f8abe93c3f0e180f2d3496eabc8 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 15:24:14 +0000 Subject: [PATCH 16/42] mod/gateway: improve socket pool --- mod/gateway/errors.go | 1 + mod/gateway/src/maintain_binding_task.go | 65 +++------------- mod/gateway/src/route.go | 1 + mod/gateway/src/socket_pool.go | 97 ++++++++++-------------- 4 files changed, 52 insertions(+), 112 deletions(-) diff --git a/mod/gateway/errors.go b/mod/gateway/errors.go index 6bf69cd47..f944a8cae 100644 --- a/mod/gateway/errors.go +++ b/mod/gateway/errors.go @@ -5,3 +5,4 @@ import "errors" var ErrUnauthorized = errors.New("unauthorized") var ErrTargetNotReachable = errors.New("target not reachable") var ErrInvalidGateway = errors.New("invalid gateway") +var ErrSocketUnreachable = errors.New("socket unreachable") diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go index 5453d5545..b6b32812e 100644 --- a/mod/gateway/src/maintain_binding_task.go +++ b/mod/gateway/src/maintain_binding_task.go @@ -1,28 +1,23 @@ package gateway import ( - "sync/atomic" + "errors" "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/lib/astrald" - "github.com/cryptopunkscc/astrald/mod/events" "github.com/cryptopunkscc/astrald/mod/gateway" gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" - "github.com/cryptopunkscc/astrald/mod/nodes" "github.com/cryptopunkscc/astrald/mod/scheduler" "github.com/cryptopunkscc/astrald/sig" ) var _ scheduler.Task = &MaintainBindingTask{} -var _ scheduler.EventReceiver = &MaintainBindingTask{} type MaintainBindingTask struct { - mod *Module - GatewayID *astral.Identity - Visibility gateway.Visibility - wake chan struct{} - actionRequired atomic.Bool + mod *Module + GatewayID *astral.Identity + Visibility gateway.Visibility } func (mod *Module) NewMaintainBindingTask(gatewayID *astral.Identity, visibility gateway.Visibility) *MaintainBindingTask { @@ -30,7 +25,6 @@ func (mod *Module) NewMaintainBindingTask(gatewayID *astral.Identity, visibility mod: mod, GatewayID: gatewayID, Visibility: visibility, - wake: make(chan struct{}, 1), } } @@ -47,17 +41,8 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { } count := -1 - task.actionRequired.Store(true) for { - for !task.actionRequired.Load() { - select { - case <-ctx.Done(): - return ctx.Err() - case <-task.wake: - } - } - switch { case count == 0: task.mod.log.Log("binding to %v lost, rebinding", task.GatewayID) @@ -68,7 +53,11 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { client := gatewayClient.New(task.GatewayID, astrald.Default()) socket, bindErr := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), task.Visibility) if bindErr != nil { - count = <-retry.Retry() + select { + case <-ctx.Done(): + return ctx.Err() + case count = <-retry.Retry(): + } continue } @@ -79,40 +68,10 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { task.mod.log.Log("bound to %v", task.GatewayID) } count = 0 - task.actionRequired.Store(false) - - task.maintainSocketConnections(ctx, socket) - if ctx.Err() != nil { - return ctx.Err() - } - - // socket dead — trigger rebind - task.actionRequired.Store(true) - select { - case task.wake <- struct{}{}: - default: - } - } -} - -func (task *MaintainBindingTask) maintainSocketConnections(ctx *astral.Context, socket *gateway.Socket) { - if err := newSocketPool(ctx, task.mod, socket).Run(); err == ErrSocketUnreachable { - task.mod.log.Log("gateway socket %v unreachable, will rebind", socket.Endpoint) - } -} - -func (task *MaintainBindingTask) ReceiveEvent(e *events.Event) { - switch typed := e.Data.(type) { - case *nodes.StreamClosedEvent: - if !typed.RemoteIdentity.IsEqual(task.GatewayID) || typed.StreamCount != 0 { - return - } - if !task.actionRequired.Swap(true) { - select { - case task.wake <- struct{}{}: - default: - } + err := task.mod.newSocketPool(ctx, socket).Run() + if errors.Is(err, gateway.ErrSocketUnreachable) { + task.mod.log.Log("gateway socket %v unreachable, will rebind", socket.Endpoint) } } } diff --git a/mod/gateway/src/route.go b/mod/gateway/src/route.go index e7aa7f6a5..ab1732fd5 100644 --- a/mod/gateway/src/route.go +++ b/mod/gateway/src/route.go @@ -32,6 +32,7 @@ func (mod *Module) routeQuery(ctx *astral.Context, q *astral.Query, w io.WriteCl outbound: false, } + // prevents slow gateway connections actx, cancel := context.WithTimeout(context.Background(), acceptTimeout) defer cancel() diff --git a/mod/gateway/src/socket_pool.go b/mod/gateway/src/socket_pool.go index 3b11243be..0f73d59e3 100644 --- a/mod/gateway/src/socket_pool.go +++ b/mod/gateway/src/socket_pool.go @@ -1,7 +1,6 @@ package gateway import ( - "errors" "sync" "sync/atomic" "time" @@ -10,7 +9,6 @@ import ( "github.com/cryptopunkscc/astrald/astral/log" "github.com/cryptopunkscc/astrald/mod/exonet" "github.com/cryptopunkscc/astrald/mod/gateway" - "github.com/cryptopunkscc/astrald/mod/nodes" "github.com/cryptopunkscc/astrald/sig" ) @@ -19,13 +17,10 @@ const ( socketPoolMaxFails = 3 ) -var ErrSocketUnreachable = errors.New("socket unreachable") - type SocketPool struct { + *Module ctx *astral.Context socket *gateway.Socket - exonet exonet.Module - nodes nodes.Module log *log.Logger mu sync.Mutex @@ -35,21 +30,18 @@ type SocketPool struct { signal chan struct{} } -func newSocketPool(ctx *astral.Context, mod *Module, socket *gateway.Socket) *SocketPool { +func (mod *Module) newSocketPool(ctx *astral.Context, socket *gateway.Socket) *SocketPool { return &SocketPool{ ctx: ctx, + Module: mod, socket: socket, - exonet: mod.Exonet, - nodes: mod.Nodes, log: mod.log, signal: make(chan struct{}, 1), } } func (p *SocketPool) Run() error { - retry, _ := sig.NewRetry(time.Second, 2*time.Minute, 2) - failStreak := 0 - + retry, _ := sig.NewRetry(time.Second, 30*time.Second, 2) p.notify() for { @@ -58,96 +50,83 @@ func (p *SocketPool) Run() error { return p.ctx.Err() case <-p.signal: for p.idleCount() < socketPoolTargetIdle { - conn, err := p.acquireSocketConnection() + conn, err := p.acquireConn() if err != nil { - failStreak++ - if failStreak >= socketPoolMaxFails { - p.log.Log("gateway socket %v unreachable", p.socket.Endpoint) - return ErrSocketUnreachable - } select { case <-p.ctx.Done(): return p.ctx.Err() - case <-retry.Retry(): + case count := <-retry.Retry(): + if count >= socketPoolMaxFails { + p.log.Log("gateway socket %v unreachable", p.socket.Endpoint) + return gateway.ErrSocketUnreachable + } } continue } - failStreak = 0 retry.Reset() - p.attach(conn) + p.handoff(conn) } } } } -func (p *SocketPool) acquireSocketConnection() (exonet.Conn, error) { - conn, err := p.exonet.Dial(p.ctx, p.socket.Endpoint) +func (p *SocketPool) acquireConn() (exonet.Conn, error) { + conn, err := p.Exonet.Dial(p.ctx, p.socket.Endpoint) if err != nil { return nil, err } + if _, err := p.socket.Nonce.WriteTo(conn); err != nil { conn.Close() return nil, err } + return conn, nil } -func (p *SocketPool) addIdle() (total, idle int) { +func (p *SocketPool) idleCount() int { p.mu.Lock() defer p.mu.Unlock() - p.total++ - p.idle++ - return p.total, p.idle + return p.idle } -func (p *SocketPool) connTaken() (total, idle int) { +func (p *SocketPool) addIdle() { p.mu.Lock() - p.idle-- - total, idle = p.total, p.idle + p.idle++ + p.total++ + total, idle := p.total, p.idle p.mu.Unlock() p.notify() - return + p.log.Logv(1, "gateway conn added (total: %v, idle: %v)", total, idle) } -func (p *SocketPool) onClosedConn(wasUsed bool) (total, idle int) { +func (p *SocketPool) onConnTaken() { p.mu.Lock() - p.total-- - if !wasUsed { - p.idle-- - } - total, idle = p.total, p.idle + p.idle-- + total, idle := p.total, p.idle p.mu.Unlock() - if !wasUsed { - p.notify() - } - return + p.notify() + p.log.Logv(1, "gateway conn taken (total: %v, idle: %v)", total, idle) } -func (p *SocketPool) idleCount() int { +func (p *SocketPool) onConnClosed() { p.mu.Lock() - defer p.mu.Unlock() - return p.idle + p.total-- + total, idle := p.total, p.idle + p.mu.Unlock() + p.notify() + p.log.Logv(1, "gateway conn down (total: %v, idle: %v)", total, idle) } -func (p *SocketPool) attach(conn exonet.Conn) { - total, idle := p.addIdle() - p.log.Logv(1, "gateway conn up (total: %v, idle: %v)", total, idle) - +func (p *SocketPool) handoff(conn exonet.Conn) { pc := &socketConn{Conn: conn} - - pc.onFirst = func() { - total, idle := p.connTaken() - p.log.Logv(1, "gateway conn taken (total: %v, idle: %v)", total, idle) - } - - pc.onClose = func() { - total, idle := p.onClosedConn(pc.used.Load()) - p.log.Logv(1, "gateway conn down (total: %v, idle: %v)", total, idle) - } + pc.onFirst = p.onConnTaken + pc.onClose = p.onConnClosed + p.addIdle() go func() { - if err := p.nodes.EstablishInboundLink(p.ctx, pc); err != nil { + if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { p.log.Logv(1, "inbound link from %v: %v", conn.RemoteEndpoint(), err) } }() From 7fb27761bc40168515a1d9fb802e20094b74d2f9 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 15:28:54 +0000 Subject: [PATCH 17/42] mod/gateway: maintain_binding_task.go --- mod/gateway/src/maintain_binding_task.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go index b6b32812e..1b6abf6c1 100644 --- a/mod/gateway/src/maintain_binding_task.go +++ b/mod/gateway/src/maintain_binding_task.go @@ -51,8 +51,8 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { } client := gatewayClient.New(task.GatewayID, astrald.Default()) - socket, bindErr := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), task.Visibility) - if bindErr != nil { + socket, err := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), task.Visibility) + if err != nil { select { case <-ctx.Done(): return ctx.Err() @@ -69,7 +69,7 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { } count = 0 - err := task.mod.newSocketPool(ctx, socket).Run() + err = task.mod.newSocketPool(ctx, socket).Run() if errors.Is(err, gateway.ErrSocketUnreachable) { task.mod.log.Log("gateway socket %v unreachable, will rebind", socket.Endpoint) } From f291817ec6ded874fd49195c934706dbc7f521c3 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 16:11:04 +0000 Subject: [PATCH 18/42] mod/gateway: improvements in SocketPool --- mod/gateway/src/maintain_binding_task.go | 9 ++-- mod/gateway/src/socket_pool.go | 53 +++++++++++++----------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go index 1b6abf6c1..a7da57543 100644 --- a/mod/gateway/src/maintain_binding_task.go +++ b/mod/gateway/src/maintain_binding_task.go @@ -1,7 +1,6 @@ package gateway import ( - "errors" "time" "github.com/cryptopunkscc/astrald/astral" @@ -40,8 +39,9 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { return err } - count := -1 + client := gatewayClient.New(task.GatewayID, astrald.Default()) + count := -1 for { switch { case count == 0: @@ -50,7 +50,6 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { task.mod.log.Log("still trying to bind to %v (attempt %v)", task.GatewayID, count) } - client := gatewayClient.New(task.GatewayID, astrald.Default()) socket, err := client.Bind(ctx.IncludeZone(astral.ZoneNetwork), task.Visibility) if err != nil { select { @@ -70,8 +69,8 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { count = 0 err = task.mod.newSocketPool(ctx, socket).Run() - if errors.Is(err, gateway.ErrSocketUnreachable) { - task.mod.log.Log("gateway socket %v unreachable, will rebind", socket.Endpoint) + if err != nil { + task.mod.log.Error("rebinding to %v due to: %v", task.GatewayID, err) } } } diff --git a/mod/gateway/src/socket_pool.go b/mod/gateway/src/socket_pool.go index 0f73d59e3..12968651a 100644 --- a/mod/gateway/src/socket_pool.go +++ b/mod/gateway/src/socket_pool.go @@ -13,7 +13,7 @@ import ( ) const ( - socketPoolTargetIdle = 1 + socketPoolTargetIdle = 2 socketPoolMaxFails = 3 ) @@ -49,7 +49,7 @@ func (p *SocketPool) Run() error { case <-p.ctx.Done(): return p.ctx.Err() case <-p.signal: - for p.idleCount() < socketPoolTargetIdle { + for toAdd := socketPoolTargetIdle - p.idleCount(); toAdd > 0; toAdd-- { conn, err := p.acquireConn() if err != nil { select { @@ -57,10 +57,10 @@ func (p *SocketPool) Run() error { return p.ctx.Err() case count := <-retry.Retry(): if count >= socketPoolMaxFails { - p.log.Log("gateway socket %v unreachable", p.socket.Endpoint) return gateway.ErrSocketUnreachable } } + toAdd++ continue } @@ -95,39 +95,40 @@ func (p *SocketPool) addIdle() { p.mu.Lock() p.idle++ p.total++ - total, idle := p.total, p.idle p.mu.Unlock() - p.notify() - p.log.Logv(1, "gateway conn added (total: %v, idle: %v)", total, idle) } func (p *SocketPool) onConnTaken() { p.mu.Lock() p.idle-- - total, idle := p.total, p.idle p.mu.Unlock() p.notify() - p.log.Logv(1, "gateway conn taken (total: %v, idle: %v)", total, idle) } -func (p *SocketPool) onConnClosed() { +func (p *SocketPool) onConnClosed(wasIdle bool) { p.mu.Lock() p.total-- - total, idle := p.total, p.idle + if wasIdle { + p.idle-- + } + p.mu.Unlock() p.notify() - p.log.Logv(1, "gateway conn down (total: %v, idle: %v)", total, idle) } func (p *SocketPool) handoff(conn exonet.Conn) { pc := &socketConn{Conn: conn} - pc.onFirst = p.onConnTaken - pc.onClose = p.onConnClosed + + // when first write is done it means we started responding to link negotiation + pc.onFirstWrite = p.onConnTaken + pc.onClose = func() { p.onConnClosed(!pc.used.Load()) } p.addIdle() go func() { - if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { + err := p.Nodes.EstablishInboundLink(p.ctx, pc) + if err != nil { p.log.Logv(1, "inbound link from %v: %v", conn.RemoteEndpoint(), err) + return } }() } @@ -142,27 +143,29 @@ func (p *SocketPool) notify() { type socketConn struct { exonet.Conn - onFirst func() - onClose func() + onFirstWrite func() + onClose func() - used atomic.Bool + closed atomic.Bool + used atomic.Bool } -func (c *socketConn) Read(b []byte) (int, error) { - if !c.used.Swap(true) && c.onFirst != nil { - c.onFirst() +func (c *socketConn) Write(b []byte) (int, error) { + if !c.used.Swap(true) && c.onFirstWrite != nil { + c.onFirstWrite() } - return c.Conn.Read(b) -} -func (c *socketConn) Write(b []byte) (int, error) { return c.Conn.Write(b) } func (c *socketConn) Close() error { err := c.Conn.Close() - if c.onClose != nil { - c.onClose() + + if !c.closed.Swap(true) { + if c.onClose != nil { + c.onClose() + } } + return err } From 837de7484a78e59c231d08bb63218972d52f58b7 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 17:13:43 +0000 Subject: [PATCH 19/42] mod/gateway: minor fixes --- mod/gateway/src/accept.go | 15 +++++----- mod/gateway/src/maintain_binding_task.go | 2 +- mod/gateway/src/socket_pool.go | 35 ++++++++++++++++-------- mod/nodes/src/peers.go | 6 ++++ 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 7d8dd9adc..c5402ed0f 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "sync" "io" "strings" @@ -101,20 +102,20 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, } func pipe(a, b io.ReadWriteCloser) { - done := make(chan struct{}, 2) - defer close(done) + var wg sync.WaitGroup + wg.Add(2) go func() { + defer wg.Done() io.Copy(a, b) - done <- struct{}{} + a.Close() }() go func() { + defer wg.Done() io.Copy(b, a) - done <- struct{}{} + b.Close() }() - <-done - a.Close() - b.Close() + wg.Wait() } diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go index a7da57543..c46eae6f8 100644 --- a/mod/gateway/src/maintain_binding_task.go +++ b/mod/gateway/src/maintain_binding_task.go @@ -68,7 +68,7 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { } count = 0 - err = task.mod.newSocketPool(ctx, socket).Run() + err = task.mod.newSocketPool(ctx, task.GatewayID, socket).Run() if err != nil { task.mod.log.Error("rebinding to %v due to: %v", task.GatewayID, err) } diff --git a/mod/gateway/src/socket_pool.go b/mod/gateway/src/socket_pool.go index 12968651a..db906b82d 100644 --- a/mod/gateway/src/socket_pool.go +++ b/mod/gateway/src/socket_pool.go @@ -19,9 +19,10 @@ const ( type SocketPool struct { *Module - ctx *astral.Context - socket *gateway.Socket - log *log.Logger + ctx *astral.Context + socket *gateway.Socket + gatewayID *astral.Identity + log *log.Logger mu sync.Mutex total int @@ -30,13 +31,14 @@ type SocketPool struct { signal chan struct{} } -func (mod *Module) newSocketPool(ctx *astral.Context, socket *gateway.Socket) *SocketPool { +func (mod *Module) newSocketPool(ctx *astral.Context, gatewayID *astral.Identity, socket *gateway.Socket) *SocketPool { return &SocketPool{ - ctx: ctx, - Module: mod, - socket: socket, - log: mod.log, - signal: make(chan struct{}, 1), + ctx: ctx, + Module: mod, + socket: socket, + gatewayID: gatewayID, + log: mod.log, + signal: make(chan struct{}, 1), } } @@ -117,7 +119,11 @@ func (p *SocketPool) onConnClosed(wasIdle bool) { } func (p *SocketPool) handoff(conn exonet.Conn) { - pc := &socketConn{Conn: conn} + pc := &socketConn{ + Conn: conn, + localEndpoint: gateway.NewEndpoint(p.node.Identity(), p.node.Identity()), + remoteEndpoint: gateway.NewEndpoint(p.gatewayID, p.node.Identity()), + } // when first write is done it means we started responding to link negotiation pc.onFirstWrite = p.onConnTaken @@ -143,13 +149,18 @@ func (p *SocketPool) notify() { type socketConn struct { exonet.Conn - onFirstWrite func() - onClose func() + localEndpoint exonet.Endpoint + remoteEndpoint exonet.Endpoint + onFirstWrite func() + onClose func() closed atomic.Bool used atomic.Bool } +func (c *socketConn) LocalEndpoint() exonet.Endpoint { return c.localEndpoint } +func (c *socketConn) RemoteEndpoint() exonet.Endpoint { return c.remoteEndpoint } + func (c *socketConn) Write(b []byte) (int, error) { if !c.used.Swap(true) && c.onFirstWrite != nil { c.onFirstWrite() diff --git a/mod/nodes/src/peers.go b/mod/nodes/src/peers.go index fddfaa92a..c0c7d6aec 100644 --- a/mod/nodes/src/peers.go +++ b/mod/nodes/src/peers.go @@ -11,6 +11,7 @@ import ( "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/lib/query" "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" "github.com/cryptopunkscc/astrald/mod/nodes" "github.com/cryptopunkscc/astrald/mod/nodes/src/frames" "github.com/cryptopunkscc/astrald/mod/nodes/src/noise" @@ -369,6 +370,11 @@ func (mod *Peers) reflectStream(s *Stream) (err error) { return } + // note: rethink maybe switch (?) + if _, ok := s.RemoteEndpoint().(*gateway.Endpoint); ok { + // dont reflect gateway endpoints + return + } // reflect the endpoint err = mod.Objects.Push(mod.ctx, s.RemoteIdentity(), &nodes.ObservedEndpointMessage{ From 3629b65b73dc59198ebec4e3ab60c0487c9e85d6 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 17:16:39 +0000 Subject: [PATCH 20/42] mod/gateway: log fix --- mod/gateway/src/accept.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index c5402ed0f..b2b0924d1 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -50,8 +50,6 @@ func (mod *Module) startServers(ctx *astral.Context) { // acceptSocketConn accepts connection on the socket that gateway told client to connect to. func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, error) { - mod.log.Infov(1, "accepting connection from %v", conn.RemoteEndpoint()) - var nonce astral.Nonce if _, err := nonce.ReadFrom(conn); err != nil { mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) @@ -66,6 +64,8 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, return true, nil } + mod.log.Infov(1, "accepting connection from %v", client.Identity) + if client.isBinder() { mod.log.Infov(1, "added idle conn to %v", client.Identity) client.add(conn) From a3691e69cba075e2892793df21ec75d61de9af67 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 17:27:12 +0000 Subject: [PATCH 21/42] mod/gateway: small fixes --- mod/gateway/src/accept.go | 20 +++++++++----------- mod/tcp/src/server.go | 5 +++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index b2b0924d1..fec5919f5 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "fmt" "sync" "io" @@ -20,7 +21,6 @@ func (mod *Module) startServers(ctx *astral.Context) { continue } network, address := parts[0], parts[1] - endpoint, err := mod.Exonet.Parse(network, address) if err != nil { mod.log.Error("parse listen address %v: %v", addr, err) @@ -49,19 +49,19 @@ func (mod *Module) startServers(ctx *astral.Context) { } // acceptSocketConn accepts connection on the socket that gateway told client to connect to. -func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, error) { +func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopListener bool, err error) { var nonce astral.Nonce if _, err := nonce.ReadFrom(conn); err != nil { mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) conn.Close() - return true, nil + return stopListener, nil } client, ok := mod.clientByNonce(nonce) if !ok { mod.log.Errorv(1, "unknown nonce %v from %v", nonce, conn.RemoteEndpoint()) conn.Close() - return true, nil + return stopListener, nil } mod.log.Infov(1, "accepting connection from %v", client.Identity) @@ -69,16 +69,14 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, if client.isBinder() { mod.log.Infov(1, "added idle conn to %v", client.Identity) client.add(conn) - return false, nil + return stopListener, nil } // clients mod.clients.Remove(client) binderConn := client.takePipeTo() if binderConn == nil { - mod.log.Errorv(1, "no reserved conn for %v", client.Target) - conn.Close() - return true, nil + return stopListener, fmt.Errorf("no reserved conn for %v", client.Target) } connectorConn := &clientConn{ @@ -90,15 +88,15 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (bool, targetClient, ok := mod.binderByIdentity(client.Target) if !ok { - return true, nil + return stopListener, nil } targetClient.markPiped(binderConn, connectorConn) client.markPiped(connectorConn, binderConn) - mod.log.Infov(1, "clients %v to %v", client.Identity, client.Target) + mod.log.Infov(1, "pipe from %v to %v created", client.Identity, client.Target) go pipe(binderConn, connectorConn) - return false, nil + return stopListener, nil } func pipe(a, b io.ReadWriteCloser) { diff --git a/mod/tcp/src/server.go b/mod/tcp/src/server.go index 24f76dede..d0189f31f 100644 --- a/mod/tcp/src/server.go +++ b/mod/tcp/src/server.go @@ -65,13 +65,14 @@ func (s *Server) Run(ctx *astral.Context) error { conn := tcp.WrapConn(rawConn, false) go func() { - shouldClose, err := s.onAccept(ctx, conn) + stopListener, err := s.onAccept(ctx, conn) if err != nil { + conn.Close() s.log.Errorv(1, "tcp server/onAccept error from %v: %v", conn.RemoteEndpoint(), err) return } - if shouldClose { + if stopListener { s.Close() } }() From f44db73ae96069f81e39bb157334a1a50ef0d664 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Thu, 12 Mar 2026 23:23:59 +0000 Subject: [PATCH 22/42] mod/gateway: adding EventReceicer to maintain_binding_task.go --- mod/gateway/src/accept.go | 3 +-- mod/gateway/src/client.go | 10 ++++---- mod/gateway/src/maintain_binding_task.go | 32 ++++++++++++++++++------ mod/tcp/src/dial.go | 3 ++- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index fec5919f5..668748f1f 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -65,7 +65,6 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi } mod.log.Infov(1, "accepting connection from %v", client.Identity) - if client.isBinder() { mod.log.Infov(1, "added idle conn to %v", client.Identity) client.add(conn) @@ -84,7 +83,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi network: conn.RemoteEndpoint().Network(), } - client.conns.Add(connectorConn) + client.connections.Add(connectorConn) targetClient, ok := mod.binderByIdentity(client.Target) if !ok { diff --git a/mod/gateway/src/client.go b/mod/gateway/src/client.go index b8e688937..58a47955d 100644 --- a/mod/gateway/src/client.go +++ b/mod/gateway/src/client.go @@ -33,8 +33,8 @@ type client struct { Visibility gateway.Visibility Target *astral.Identity // nil for binders, set for clients // - conns sig.Set[*clientConn] - pipeTo *clientConn // reserved binder conn for clients clients + connections sig.Set[*clientConn] + pipeTo *clientConn // reserved binder conn for clients clients } func (c *client) isBinder() bool { @@ -42,7 +42,7 @@ func (c *client) isBinder() bool { } func (c *client) add(conn exonet.Conn) { - c.conns.Add(&clientConn{ + c.connections.Add(&clientConn{ Conn: conn, network: conn.RemoteEndpoint().Network(), state: connStateIdle, @@ -52,7 +52,7 @@ func (c *client) add(conn exonet.Conn) { func (c *client) take() (*clientConn, bool) { c.mu.Lock() defer c.mu.Unlock() - for _, cc := range c.conns.Clone() { + for _, cc := range c.connections.Clone() { if cc.state == connStateIdle { cc.state = connStateReserved return cc, true @@ -82,7 +82,7 @@ func (c *client) Close() error { var errs []error - for _, cc := range c.conns.Clone() { + for _, cc := range c.connections.Clone() { errs = append(errs, cc.Close()) } diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go index c46eae6f8..186430c52 100644 --- a/mod/gateway/src/maintain_binding_task.go +++ b/mod/gateway/src/maintain_binding_task.go @@ -5,25 +5,33 @@ import ( "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/lib/astrald" + "github.com/cryptopunkscc/astrald/mod/events" "github.com/cryptopunkscc/astrald/mod/gateway" gatewayClient "github.com/cryptopunkscc/astrald/mod/gateway/client" + "github.com/cryptopunkscc/astrald/mod/ip" "github.com/cryptopunkscc/astrald/mod/scheduler" "github.com/cryptopunkscc/astrald/sig" ) var _ scheduler.Task = &MaintainBindingTask{} +var _ scheduler.EventReceiver = &MaintainBindingTask{} type MaintainBindingTask struct { mod *Module GatewayID *astral.Identity Visibility gateway.Visibility + retry *sig.Retry + triggerCh chan struct{} } func (mod *Module) NewMaintainBindingTask(gatewayID *astral.Identity, visibility gateway.Visibility) *MaintainBindingTask { + retry, _ := sig.NewRetry(time.Second, 15*time.Minute, 2) return &MaintainBindingTask{ mod: mod, GatewayID: gatewayID, Visibility: visibility, + retry: retry, + triggerCh: make(chan struct{}, 1), } } @@ -33,12 +41,6 @@ func (task *MaintainBindingTask) String() string { func (task *MaintainBindingTask) Run(ctx *astral.Context) error { task.mod.log.Log("starting to maintain binding to %v", task.GatewayID) - - retry, err := sig.NewRetry(time.Second, 15*time.Minute, 2) - if err != nil { - return err - } - client := gatewayClient.New(task.GatewayID, astrald.Default()) count := -1 @@ -55,12 +57,13 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { select { case <-ctx.Done(): return ctx.Err() - case count = <-retry.Retry(): + case count = <-task.retry.Retry(): + case <-task.triggerCh: } continue } - retry.Reset() + task.retry.Reset() if count > 0 { task.mod.log.Log("rebound to %v after %v attempts", task.GatewayID, count) } else if count < 0 { @@ -74,3 +77,16 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { } } } + +func (task *MaintainBindingTask) ReceiveEvent(e *events.Event) { + switch typed := e.Data.(type) { + case *ip.EventNetworkAddressChanged: + if len(typed.Added) > 0 { + task.retry.Reset() + select { + case task.triggerCh <- struct{}{}: + default: + } + } + } +} diff --git a/mod/tcp/src/dial.go b/mod/tcp/src/dial.go index 783bf7405..7682ae550 100644 --- a/mod/tcp/src/dial.go +++ b/mod/tcp/src/dial.go @@ -2,6 +2,7 @@ package tcp import ( _net "net" + "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" @@ -20,7 +21,7 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C return nil, exonet.ErrDisabledNetwork } - var dialer = _net.Dialer{Timeout: mod.config.DialTimeout} + var dialer = _net.Dialer{Timeout: mod.config.DialTimeout, KeepAlive: 5 * time.Second} tcpConn, err := dialer.DialContext(ctx, "tcp", endpoint.Address()) if err != nil { From 5a005038d6450aacf6347549bb5d9bd088454d67 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 00:47:01 +0000 Subject: [PATCH 23/42] mod/gateway: minor refactors --- mod/gateway/src/accept.go | 46 ++++----- mod/gateway/src/bind.go | 24 ++--- mod/gateway/src/binder.go | 88 ++++++++++++++++ mod/gateway/src/client.go | 109 -------------------- mod/gateway/src/connect.go | 31 +++--- mod/gateway/src/connector.go | 49 +++++++++ mod/gateway/src/module.go | 32 +++++- mod/gateway/src/{socket_pool.go => pool.go} | 0 8 files changed, 210 insertions(+), 169 deletions(-) create mode 100644 mod/gateway/src/binder.go delete mode 100644 mod/gateway/src/client.go create mode 100644 mod/gateway/src/connector.go rename mod/gateway/src/{socket_pool.go => pool.go} (100%) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 668748f1f..81e794c69 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -57,44 +57,44 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi return stopListener, nil } - client, ok := mod.clientByNonce(nonce) + if b, ok := mod.binderByNonce(nonce); ok { + mod.log.Infov(1, "added idle conn to binder %v", b.Identity) + b.addConn(conn) + return stopListener, nil + } + + c, ok := mod.connectorByNonce(nonce) if !ok { mod.log.Errorv(1, "unknown nonce %v from %v", nonce, conn.RemoteEndpoint()) conn.Close() return stopListener, nil } - mod.log.Infov(1, "accepting connection from %v", client.Identity) - if client.isBinder() { - mod.log.Infov(1, "added idle conn to %v", client.Identity) - client.add(conn) - return stopListener, nil + mod.connectors.Remove(c) + + reserved := c.takeReserved() + if reserved == nil { + conn.Close() + return stopListener, fmt.Errorf("no reserved conn for %v", c.Target) } - // clients - mod.clients.Remove(client) - binderConn := client.takePipeTo() - if binderConn == nil { - return stopListener, fmt.Errorf("no reserved conn for %v", client.Target) + targetBinder, ok := mod.binderByIdentity(c.Target) + if !ok { + reserved.Close() + conn.Close() + return stopListener, nil } - connectorConn := &clientConn{ + cc := &connectorConn{ Conn: conn, network: conn.RemoteEndpoint().Network(), + pipedTo: reserved, } - client.connections.Add(connectorConn) - - targetClient, ok := mod.binderByIdentity(client.Target) - if !ok { - return stopListener, nil - } - - targetClient.markPiped(binderConn, connectorConn) - client.markPiped(connectorConn, binderConn) + targetBinder.markPiped(reserved, cc) - mod.log.Infov(1, "pipe from %v to %v created", client.Identity, client.Target) - go pipe(binderConn, connectorConn) + mod.log.Infov(1, "pipe from %v to %v created", c.Identity, c.Target) + go pipe(reserved, cc) return stopListener, nil } diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go index 5d9b70955..1c4b46509 100644 --- a/mod/gateway/src/bind.go +++ b/mod/gateway/src/bind.go @@ -15,33 +15,29 @@ func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibili return nil, err } - nonce := astral.NewNonce() - - client := &client{ + b := &binder{ Identity: identity, - Nonce: nonce, + Nonce: astral.NewNonce(), Visibility: visibility, - Target: nil, } - oldClient, ok := mod.binders.Replace(identity.String(), client) + oldBinder, ok := mod.binders.Replace(identity.String(), b) if ok { - if err = oldClient.Close(); err != nil { - mod.log.Error("failed to close old client: %v", err) + if err = oldBinder.Close(); err != nil { + mod.log.Error("failed to close old binder: %v", err) } - targetID := oldClient.Identity.String() - for _, c := range mod.clients.Clone() { + targetID := oldBinder.Identity.String() + for _, c := range mod.connectors.Clone() { if c.Target.String() == targetID { - if c.takePipeTo() != nil { - mod.clients.Remove(c) - } + mod.connectors.Remove(c) + c.Close() } } } return &gateway.Socket{ - Nonce: nonce, + Nonce: b.Nonce, Endpoint: endpoint, }, nil } diff --git a/mod/gateway/src/binder.go b/mod/gateway/src/binder.go new file mode 100644 index 000000000..c42fc1e09 --- /dev/null +++ b/mod/gateway/src/binder.go @@ -0,0 +1,88 @@ +package gateway + +import ( + "errors" + "sync" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" + "github.com/cryptopunkscc/astrald/sig" +) + +type connState uint8 + +const ( + connStateIdle connState = iota + connStateReserved connState = iota + connStatePiped connState = iota +) + +// binderConn is a connection pre-opened by a binder node to the gateway, +// sitting idle until a connector claims it. +type binderConn struct { + exonet.Conn + state connState + pipedTo *connectorConn + onClose func() +} + +func (bc *binderConn) Close() error { + err := bc.Conn.Close() + if bc.onClose != nil { + bc.onClose() + } + return err +} + +// binder represents a node registered as reachable through the gateway. +// Only one binder registration per identity is allowed. +type binder struct { + mu sync.Mutex + Identity *astral.Identity + Nonce astral.Nonce + Visibility gateway.Visibility + conns sig.Set[*binderConn] +} + +func (b *binder) addConn(conn exonet.Conn) { + bc := &binderConn{ + Conn: conn, + state: connStateIdle, + } + bc.onClose = func() { b.conns.Remove(bc) } + b.conns.Add(bc) +} + +// takeConn reserves an idle binderConn for a connector. +func (b *binder) takeConn() (*binderConn, bool) { + b.mu.Lock() + defer b.mu.Unlock() + for _, bc := range b.conns.Clone() { + if bc.state == connStateIdle { + bc.state = connStateReserved + return bc, true + } + } + return nil, false +} + +func (b *binder) markPiped(bc *binderConn, cc *connectorConn) { + b.mu.Lock() + defer b.mu.Unlock() + bc.state = connStatePiped + bc.pipedTo = cc +} + +func (b *binder) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + var errs []error + for _, bc := range b.conns.Clone() { + if bc.state == connStateReserved { + continue + } + errs = append(errs, bc.Close()) + } + return errors.Join(errs...) +} diff --git a/mod/gateway/src/client.go b/mod/gateway/src/client.go deleted file mode 100644 index 58a47955d..000000000 --- a/mod/gateway/src/client.go +++ /dev/null @@ -1,109 +0,0 @@ -package gateway - -import ( - "errors" - "sync" - - "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" - "github.com/cryptopunkscc/astrald/sig" -) - -type connState uint8 - -const ( - connStateIdle connState = iota - connStateReserved connState = iota - connStatePiped connState = iota -) - -type clientConn struct { - exonet.Conn - network string - state connState - pipedTo *clientConn // non-nil when connStatePiped -} - -type client struct { - mu sync.Mutex - - Identity *astral.Identity - Nonce astral.Nonce - Visibility gateway.Visibility - Target *astral.Identity // nil for binders, set for clients - // - connections sig.Set[*clientConn] - pipeTo *clientConn // reserved binder conn for clients clients -} - -func (c *client) isBinder() bool { - return c.Target == nil -} - -func (c *client) add(conn exonet.Conn) { - c.connections.Add(&clientConn{ - Conn: conn, - network: conn.RemoteEndpoint().Network(), - state: connStateIdle, - }) -} - -func (c *client) take() (*clientConn, bool) { - c.mu.Lock() - defer c.mu.Unlock() - for _, cc := range c.connections.Clone() { - if cc.state == connStateIdle { - cc.state = connStateReserved - return cc, true - } - } - return nil, false -} - -func (c *client) markPiped(cc, other *clientConn) { - c.mu.Lock() - defer c.mu.Unlock() - cc.state = connStatePiped - cc.pipedTo = other -} - -func (c *client) takePipeTo() *clientConn { - c.mu.Lock() - defer c.mu.Unlock() - cc := c.pipeTo - c.pipeTo = nil - return cc -} - -func (c *client) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - - var errs []error - - for _, cc := range c.connections.Clone() { - errs = append(errs, cc.Close()) - } - - return errors.Join(errs...) -} - -func (mod *Module) binderByIdentity(identity *astral.Identity) (*client, bool) { - return mod.binders.Get(identity.String()) -} - -func (mod *Module) clientByNonce(nonce astral.Nonce) (*client, bool) { - for _, c := range mod.binders.Values() { - if c.Nonce == nonce { - return c, true - } - } - - for _, c := range mod.clients.Clone() { - if c.Nonce == nonce { - return c, true - } - } - return nil, false -} diff --git a/mod/gateway/src/connect.go b/mod/gateway/src/connect.go index ba864831b..930542326 100644 --- a/mod/gateway/src/connect.go +++ b/mod/gateway/src/connect.go @@ -19,46 +19,41 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n return gateway.Socket{}, err } - binder, ok := mod.binderByIdentity(target) + b, ok := mod.binderByIdentity(target) if !ok { return socket, gateway.ErrTargetNotReachable } - reserved, ok := binder.take() + reserved, ok := b.takeConn() if !ok { - // fixme: return public err ErrCannotConnect return socket, gateway.ErrTargetNotReachable } - nonce := astral.NewNonce() - client := &client{ + c := &connector{ Identity: caller, - Nonce: nonce, + Nonce: astral.NewNonce(), Target: target, - pipeTo: reserved, + reserved: reserved, } - mod.clients.Add(client) + mod.connectors.Add(c) go func() { - select { - case <-time.After(connectTimeout): - } + <-time.After(connectTimeout) - binderConn := client.takePipeTo() - if binderConn == nil { + bc := c.takeReserved() + if bc == nil { return } - mod.clients.Remove(client) - err = binderConn.Close() - if err != nil { - mod.log.Error("failed to close binderConn: %v", err) + mod.connectors.Remove(c) + if err := bc.Close(); err != nil { + mod.log.Error("failed to close reserved conn: %v", err) } }() return gateway.Socket{ - Nonce: nonce, + Nonce: c.Nonce, Endpoint: endpoint, }, nil } diff --git a/mod/gateway/src/connector.go b/mod/gateway/src/connector.go new file mode 100644 index 000000000..68554b938 --- /dev/null +++ b/mod/gateway/src/connector.go @@ -0,0 +1,49 @@ +package gateway + +import ( + "errors" + "sync" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/exonet" +) + +// connectorConn is the connection opened by a connector node to the gateway, +// to be piped to a reserved binderConn. +type connectorConn struct { + exonet.Conn + network string + pipedTo *binderConn +} + +// connector represents a pending connection request from a node that wants +// to reach a binder through the gateway. Multiple connectors per identity +// are allowed. +type connector struct { + mu sync.Mutex + Identity *astral.Identity + Nonce astral.Nonce + Target *astral.Identity + reserved *binderConn +} + +// takeReserved atomically takes the reserved binderConn, returning nil if +// already taken (connection already established or timed out). +func (c *connector) takeReserved() *binderConn { + c.mu.Lock() + defer c.mu.Unlock() + bc := c.reserved + c.reserved = nil + return bc +} + +func (c *connector) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + var errs []error + if c.reserved != nil { + errs = append(errs, c.reserved.Close()) + c.reserved = nil + } + return errors.Join(errs...) +} diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index d8211d036..84ca5988e 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -36,8 +36,8 @@ type Module struct { log *log.Logger ctx *astral.Context - binders sig.Map[string, *client] - clients sig.Set[*client] + binders sig.Map[string, *binder] + connectors sig.Set[*connector] listenEndpoints sig.Map[string, exonet.Endpoint] } @@ -65,10 +65,10 @@ func (mod *Module) Run(ctx *astral.Context) error { <-ctx.Done() - for _, c := range mod.binders.Values() { - c.Close() + for _, b := range mod.binders.Values() { + b.Close() } - for _, c := range mod.clients.Clone() { + for _, c := range mod.connectors.Clone() { c.Close() } @@ -91,6 +91,28 @@ func (mod *Module) getGatewayEndpoint(ctx *astral.Context, network string) (endp return endpoint, nil } +func (mod *Module) binderByIdentity(identity *astral.Identity) (*binder, bool) { + return mod.binders.Get(identity.String()) +} + +func (mod *Module) binderByNonce(nonce astral.Nonce) (*binder, bool) { + for _, b := range mod.binders.Values() { + if b.Nonce == nonce { + return b, true + } + } + return nil, false +} + +func (mod *Module) connectorByNonce(nonce astral.Nonce) (*connector, bool) { + for _, c := range mod.connectors.Clone() { + if c.Nonce == nonce { + return c, true + } + } + return nil, false +} + func (mod *Module) canGateway(identity *astral.Identity) bool { return mod.config.Gateway.Enabled } diff --git a/mod/gateway/src/socket_pool.go b/mod/gateway/src/pool.go similarity index 100% rename from mod/gateway/src/socket_pool.go rename to mod/gateway/src/pool.go From c23aabc5a44b902f8f5fa1749f7086058b76c5a1 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 00:58:14 +0000 Subject: [PATCH 24/42] mod/gateway: add comments --- mod/gateway/src/pool.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index db906b82d..eb7285ac7 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -146,6 +146,7 @@ func (p *SocketPool) notify() { } } +// socketConn is considered a connection only after the first write is done. type socketConn struct { exonet.Conn From ad0f831eeeb38164bd537b6c0f7b9455e42dfbcb Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 10:26:38 +0000 Subject: [PATCH 25/42] mod/gateway: adding GatewayID to socket --- mod/gateway/socket.go | 5 +++-- mod/gateway/src/bind.go | 5 +++-- mod/gateway/src/pool.go | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mod/gateway/socket.go b/mod/gateway/socket.go index 9a6afa371..f6dda583c 100644 --- a/mod/gateway/socket.go +++ b/mod/gateway/socket.go @@ -11,8 +11,9 @@ import ( // a raw exonet connection to the Endpoint and sends Nonce as the first bytes // to identify itself to the gateway. type Socket struct { - Endpoint exonet.Endpoint - Nonce astral.Nonce + GatewayID *astral.Identity + Endpoint exonet.Endpoint + Nonce astral.Nonce } func (Socket) ObjectType() string { return "mod.gateway.socket" } diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go index 1c4b46509..348f59efd 100644 --- a/mod/gateway/src/bind.go +++ b/mod/gateway/src/bind.go @@ -37,7 +37,8 @@ func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibili } return &gateway.Socket{ - Nonce: b.Nonce, - Endpoint: endpoint, + GatewayID: mod.node.Identity(), + Nonce: b.Nonce, + Endpoint: endpoint, }, nil } diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index eb7285ac7..cf1201410 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -74,6 +74,7 @@ func (p *SocketPool) Run() error { } func (p *SocketPool) acquireConn() (exonet.Conn, error) { + p.log.Logv(1, "acquiring socket connection to %v through %v", p.socket.Endpoint) conn, err := p.Exonet.Dial(p.ctx, p.socket.Endpoint) if err != nil { return nil, err @@ -133,7 +134,7 @@ func (p *SocketPool) handoff(conn exonet.Conn) { go func() { err := p.Nodes.EstablishInboundLink(p.ctx, pc) if err != nil { - p.log.Logv(1, "inbound link from %v: %v", conn.RemoteEndpoint(), err) + pc.Close() return } }() From 647a8d6a87cfe7cd138a58b0e9ecf4251d85d23b Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 10:41:41 +0000 Subject: [PATCH 26/42] mod/gateway: better logs --- mod/gateway/socket.go | 5 ++--- mod/gateway/src/accept.go | 2 ++ mod/gateway/src/bind.go | 5 ++--- mod/gateway/src/pool.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mod/gateway/socket.go b/mod/gateway/socket.go index f6dda583c..9a6afa371 100644 --- a/mod/gateway/socket.go +++ b/mod/gateway/socket.go @@ -11,9 +11,8 @@ import ( // a raw exonet connection to the Endpoint and sends Nonce as the first bytes // to identify itself to the gateway. type Socket struct { - GatewayID *astral.Identity - Endpoint exonet.Endpoint - Nonce astral.Nonce + Endpoint exonet.Endpoint + Nonce astral.Nonce } func (Socket) ObjectType() string { return "mod.gateway.socket" } diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 81e794c69..8695b486f 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -50,6 +50,8 @@ func (mod *Module) startServers(ctx *astral.Context) { // acceptSocketConn accepts connection on the socket that gateway told client to connect to. func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopListener bool, err error) { + mod.log.Logv(2, "accepting socket connection from %v", conn.RemoteEndpoint()) + var nonce astral.Nonce if _, err := nonce.ReadFrom(conn); err != nil { mod.log.Errorv(1, "read nonce from %v: %v", conn.RemoteEndpoint(), err) diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go index 348f59efd..1c4b46509 100644 --- a/mod/gateway/src/bind.go +++ b/mod/gateway/src/bind.go @@ -37,8 +37,7 @@ func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibili } return &gateway.Socket{ - GatewayID: mod.node.Identity(), - Nonce: b.Nonce, - Endpoint: endpoint, + Nonce: b.Nonce, + Endpoint: endpoint, }, nil } diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index cf1201410..002dd514b 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -74,7 +74,7 @@ func (p *SocketPool) Run() error { } func (p *SocketPool) acquireConn() (exonet.Conn, error) { - p.log.Logv(1, "acquiring socket connection to %v through %v", p.socket.Endpoint) + p.log.Logv(2, "acquiring socket connection to %v through %v", p.socket.Endpoint, p.gatewayID) conn, err := p.Exonet.Dial(p.ctx, p.socket.Endpoint) if err != nil { return nil, err From e78b89ba47bf7cb283af011995eb033da75191cc Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 10:44:46 +0000 Subject: [PATCH 27/42] mod/gateway: log improvements --- mod/gateway/src/accept.go | 4 ++-- mod/gateway/src/connector.go | 10 +++++----- mod/gateway/src/pool.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 8695b486f..4a2de65da 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -60,7 +60,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi } if b, ok := mod.binderByNonce(nonce); ok { - mod.log.Infov(1, "added idle conn to binder %v", b.Identity) + mod.log.Infov(2, "added idle conn to binder %v", b.Identity) b.addConn(conn) return stopListener, nil } @@ -95,7 +95,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi targetBinder.markPiped(reserved, cc) - mod.log.Infov(1, "pipe from %v to %v created", c.Identity, c.Target) + mod.log.Infov(2, "pipe from %v to %v created", c.Identity, c.Target) go pipe(reserved, cc) return stopListener, nil } diff --git a/mod/gateway/src/connector.go b/mod/gateway/src/connector.go index 68554b938..0458bbd2d 100644 --- a/mod/gateway/src/connector.go +++ b/mod/gateway/src/connector.go @@ -1,7 +1,6 @@ package gateway import ( - "errors" "sync" "github.com/cryptopunkscc/astrald/astral" @@ -32,6 +31,7 @@ type connector struct { func (c *connector) takeReserved() *binderConn { c.mu.Lock() defer c.mu.Unlock() + bc := c.reserved c.reserved = nil return bc @@ -40,10 +40,10 @@ func (c *connector) takeReserved() *binderConn { func (c *connector) Close() error { c.mu.Lock() defer c.mu.Unlock() - var errs []error + if c.reserved != nil { - errs = append(errs, c.reserved.Close()) - c.reserved = nil + return c.reserved.Close() } - return errors.Join(errs...) + + return nil } diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index 002dd514b..55fecbf17 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -74,7 +74,7 @@ func (p *SocketPool) Run() error { } func (p *SocketPool) acquireConn() (exonet.Conn, error) { - p.log.Logv(2, "acquiring socket connection to %v through %v", p.socket.Endpoint, p.gatewayID) + p.log.Logv(2, "acquiring socket connection to %v through %v", p.gatewayID, p.socket.Endpoint) conn, err := p.Exonet.Dial(p.ctx, p.socket.Endpoint) if err != nil { return nil, err From 95be9f1604f9afc102f2b24c95398424a813fc08 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 12:20:23 +0000 Subject: [PATCH 28/42] mod/gateway: add ZoneNetwork for fallback --- mod/gateway/src/dial.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go index 87bc17da2..be3f98f02 100644 --- a/mod/gateway/src/dial.go +++ b/mod/gateway/src/dial.go @@ -25,8 +25,10 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C return nil, gateway.ErrInvalidGateway } + ctx = ctx.IncludeZone(astral.ZoneNetwork) + client := gatewayClient.New(gwEndpoint.GatewayID, astrald.Default()) - socket, err := client.Connect(ctx.IncludeZone(astral.ZoneNetwork), gwEndpoint.TargetID) + socket, err := client.Connect(ctx, gwEndpoint.TargetID) if err != nil { return mod.route(ctx, gwEndpoint) } From 32096649609212e047a7f122c0277d72e96e9da1 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 12:39:22 +0000 Subject: [PATCH 29/42] mod/gateway: add ZoneNetwork --- mod/gateway/src/accept.go | 1 + mod/gateway/src/route.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 4a2de65da..8834aa10b 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -95,6 +95,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi targetBinder.markPiped(reserved, cc) + // note: Maybe in the future we can add lightweight singalling before piping streams but its improvement not a need mod.log.Infov(2, "pipe from %v to %v created", c.Identity, c.Target) go pipe(reserved, cc) return stopListener, nil diff --git a/mod/gateway/src/route.go b/mod/gateway/src/route.go index ab1732fd5..8a7bdda87 100644 --- a/mod/gateway/src/route.go +++ b/mod/gateway/src/route.go @@ -14,6 +14,8 @@ import ( const acceptTimeout = 30 * time.Second func (mod *Module) routeQuery(ctx *astral.Context, q *astral.Query, w io.WriteCloser) (io.WriteCloser, error) { + ctx = ctx.IncludeZone(astral.ZoneNetwork) + var targetKey string switch { case strings.HasPrefix(q.Query, gateway.MethodRoute+"."): From 5cabe2b45d63f08a2a9b9961d8cc12a07a4d61d2 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 14:02:28 +0000 Subject: [PATCH 30/42] mod/gateway: adding endpoints resolvers & service discoverer --- mod/gateway/module.go | 3 +++ mod/gateway/src/deps.go | 1 + mod/gateway/src/endpoint_resolvers.go | 26 ++++++++++++++++++ mod/gateway/src/module.go | 14 ++++++++-- mod/gateway/src/service_discoverer.go | 39 +++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 mod/gateway/src/endpoint_resolvers.go create mode 100644 mod/gateway/src/service_discoverer.go diff --git a/mod/gateway/module.go b/mod/gateway/module.go index fa9929eb0..80bfa6787 100644 --- a/mod/gateway/module.go +++ b/mod/gateway/module.go @@ -8,3 +8,6 @@ const ( MethodList = "gateway.node_list" MethodRoute = "gateway.route" ) + +type Module interface { +} diff --git a/mod/gateway/src/deps.go b/mod/gateway/src/deps.go index 960f3dbcb..c8023cd89 100644 --- a/mod/gateway/src/deps.go +++ b/mod/gateway/src/deps.go @@ -15,6 +15,7 @@ func (mod *Module) LoadDependencies(*astral.Context) (err error) { mod.Exonet.SetUnpacker("gw", mod) mod.Exonet.SetParser("gw", mod) mod.ops.AddStructPrefix(mod, "Op") + mod.Services.AddDiscoverer(mod) return } diff --git a/mod/gateway/src/endpoint_resolvers.go b/mod/gateway/src/endpoint_resolvers.go new file mode 100644 index 000000000..735e6336c --- /dev/null +++ b/mod/gateway/src/endpoint_resolvers.go @@ -0,0 +1,26 @@ +package gateway + +import ( + "time" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/gateway" + "github.com/cryptopunkscc/astrald/mod/nodes" + "github.com/cryptopunkscc/astrald/sig" +) + +var _ nodes.EndpointResolver = &Module{} + +func (mod *Module) ResolveEndpoints(context *astral.Context, nodeID *astral.Identity) (<-chan *nodes.EndpointWithTTL, error) { + if !nodeID.IsEqual(mod.node.Identity()) { + // note: resolve endpoints for our "binders" if we are gateway + return sig.ArrayToChan([]*nodes.EndpointWithTTL{}), nil + } + + var endpoints []*nodes.EndpointWithTTL + for _, gw := range mod.gateways.Clone() { + endpoints = append(endpoints, nodes.NewEndpointWithTTL(gateway.NewEndpoint(gw, mod.node.Identity()), 7*30*24*time.Hour)) + } + + return sig.ArrayToChan(endpoints), nil +} diff --git a/mod/gateway/src/module.go b/mod/gateway/src/module.go index 84ca5988e..056bbaeea 100644 --- a/mod/gateway/src/module.go +++ b/mod/gateway/src/module.go @@ -11,6 +11,7 @@ import ( ipmod "github.com/cryptopunkscc/astrald/mod/ip" "github.com/cryptopunkscc/astrald/mod/nodes" "github.com/cryptopunkscc/astrald/mod/scheduler" + "github.com/cryptopunkscc/astrald/mod/services" tcpmod "github.com/cryptopunkscc/astrald/mod/tcp" "github.com/cryptopunkscc/astrald/sig" ) @@ -22,6 +23,7 @@ type Deps struct { Exonet exonet.Module Nodes nodes.Module Scheduler scheduler.Module + Services services.Module TCP tcpmod.Module IP ipmod.Module } @@ -36,12 +38,15 @@ type Module struct { log *log.Logger ctx *astral.Context + gateways sig.Set[*astral.Identity] binders sig.Map[string, *binder] connectors sig.Set[*connector] listenEndpoints sig.Map[string, exonet.Endpoint] } +var _ gateway.Module = &Module{} + func (mod *Module) GetOpSet() *ops.Set { return &mod.ops } @@ -59,12 +64,12 @@ func (mod *Module) Run(ctx *astral.Context) error { } <-mod.Scheduler.Ready() + for _, gw := range mod.config.Gateways { - mod.Scheduler.Schedule(mod.NewMaintainBindingTask(gw, mod.config.Visibility)) + mod.addPersistentGateway(gw) } <-ctx.Done() - for _, b := range mod.binders.Values() { b.Close() } @@ -117,6 +122,11 @@ func (mod *Module) canGateway(identity *astral.Identity) bool { return mod.config.Gateway.Enabled } +func (mod *Module) addPersistentGateway(gatewayID *astral.Identity) { + mod.gateways.Add(gatewayID) + mod.Scheduler.Schedule(mod.NewMaintainBindingTask(gatewayID, mod.config.Visibility)) +} + func (mod *Module) String() string { return gateway.ModuleName } diff --git a/mod/gateway/src/service_discoverer.go b/mod/gateway/src/service_discoverer.go new file mode 100644 index 000000000..8fef1c18c --- /dev/null +++ b/mod/gateway/src/service_discoverer.go @@ -0,0 +1,39 @@ +package gateway + +import ( + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/gateway" + "github.com/cryptopunkscc/astrald/mod/services" +) + +var _ services.Discoverer = &Module{} + +func (mod *Module) DiscoverServices( + ctx *astral.Context, + caller *astral.Identity, + follow bool, +) (<-chan *services.Update, error) { + var ch = make(chan *services.Update, 2) + + if mod.config.Gateway.Enabled { + ch <- &services.Update{ + Available: true, + Name: gateway.ModuleName, + ProviderID: mod.node.Identity(), + } + } + + if !follow { + close(ch) + return ch, nil + } + + ch <- nil + + go func() { + <-ctx.Done() + close(ch) + }() + + return ch, nil +} From 3b2b7ded9ea7010d7a6129c538aec118843ffab3 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Fri, 13 Mar 2026 14:28:45 +0000 Subject: [PATCH 31/42] mod/gateway: change comment --- mod/gateway/src/endpoint_resolvers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/gateway/src/endpoint_resolvers.go b/mod/gateway/src/endpoint_resolvers.go index 735e6336c..9ca841f93 100644 --- a/mod/gateway/src/endpoint_resolvers.go +++ b/mod/gateway/src/endpoint_resolvers.go @@ -13,7 +13,7 @@ var _ nodes.EndpointResolver = &Module{} func (mod *Module) ResolveEndpoints(context *astral.Context, nodeID *astral.Identity) (<-chan *nodes.EndpointWithTTL, error) { if !nodeID.IsEqual(mod.node.Identity()) { - // note: resolve endpoints for our "binders" if we are gateway + // note: we might resolve endpoints if we act as their gateway return sig.ArrayToChan([]*nodes.EndpointWithTTL{}), nil } From e6e6b8a74a4dc82183e64f92a16adbaa23650f68 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sat, 14 Mar 2026 04:28:42 +0000 Subject: [PATCH 32/42] mod/gateway: discovering dead connections --- mod/gateway/src/accept.go | 107 ++++++++++++++++++++++++++++++-------- mod/gateway/src/pool.go | 8 ++- mod/tcp/src/server.go | 6 +++ 3 files changed, 96 insertions(+), 25 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 8834aa10b..d475c5f10 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -3,10 +3,9 @@ package gateway import ( "context" "fmt" - "sync" - "io" "strings" + "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" @@ -87,35 +86,97 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi return stopListener, nil } + alive := mod.probeBinderConn(targetBinder, reserved) + if alive == nil { + mod.log.Errorv(1, "no alive conn for %v", c.Target) + conn.Close() + return stopListener, nil + } + cc := &connectorConn{ Conn: conn, network: conn.RemoteEndpoint().Network(), - pipedTo: reserved, + pipedTo: alive, } - targetBinder.markPiped(reserved, cc) - - // note: Maybe in the future we can add lightweight singalling before piping streams but its improvement not a need + targetBinder.markPiped(alive, cc) mod.log.Infov(2, "pipe from %v to %v created", c.Identity, c.Target) - go pipe(reserved, cc) + go pipe(alive, cc) return stopListener, nil } +const ( + socketSignalByte = byte(1) + socketProbeTimeout = 1 * time.Second +) + +// probeBinderConn signals each binderConn in the pool to verify it is alive, +// discarding dead connections until one succeeds or the pool is exhausted. +func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { + candidate := first + for { + if candidate == nil { + var ok bool + candidate, ok = b.takeConn() + if !ok { + return nil + } + } + + if d, ok := candidate.Conn.(deadliner); ok { + d.SetWriteDeadline(time.Now().Add(socketProbeTimeout)) + } + _, err := candidate.Write([]byte{socketSignalByte}) + if d, ok := candidate.Conn.(deadliner); ok { + d.SetWriteDeadline(time.Time{}) + } + if err == nil { + return candidate + } + + candidate.Close() + candidate = nil + } +} + +type deadliner interface { + SetReadDeadline(time.Time) error + SetWriteDeadline(time.Time) error +} + func pipe(a, b io.ReadWriteCloser) { - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - io.Copy(a, b) - a.Close() - }() - - go func() { - defer wg.Done() - io.Copy(b, a) - b.Close() - }() - - wg.Wait() + const idle = 30 * time.Second + + done := make(chan struct{}, 2) + + copy := func(dst, src io.ReadWriteCloser) { + buf := make([]byte, 32*1024) + srcD, srcOk := src.(deadliner) + dstD, dstOk := dst.(deadliner) + for { + if srcOk { + srcD.SetReadDeadline(time.Now().Add(idle)) + } + n, err := src.Read(buf) + if n > 0 { + if dstOk { + dstD.SetWriteDeadline(time.Now().Add(idle)) + } + if _, werr := dst.Write(buf[:n]); werr != nil { + break + } + } + if err != nil { + break + } + } + done <- struct{}{} + } + + go copy(a, b) + go copy(b, a) + + <-done + a.Close() + b.Close() } diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index 55fecbf17..cc8e7cd7b 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -132,11 +132,15 @@ func (p *SocketPool) handoff(conn exonet.Conn) { p.addIdle() go func() { - err := p.Nodes.EstablishInboundLink(p.ctx, pc) - if err != nil { + // wait for the gateway's start signal before beginning link negotiation + var sig [1]byte + if _, err := pc.Read(sig[:]); err != nil { pc.Close() return } + if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { + pc.Close() + } }() } diff --git a/mod/tcp/src/server.go b/mod/tcp/src/server.go index d0189f31f..379e11208 100644 --- a/mod/tcp/src/server.go +++ b/mod/tcp/src/server.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "sync/atomic" + "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" @@ -63,6 +64,11 @@ func (s *Server) Run(ctx *astral.Context) error { return fmt.Errorf("tcp server/run: accept failed: %w", err) } + if tc, ok := rawConn.(*net.TCPConn); ok { + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(30 * time.Second) + } + conn := tcp.WrapConn(rawConn, false) go func() { stopListener, err := s.onAccept(ctx, conn) From 2ada93690e6c549f21aa04eea007dafc209f2914 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sat, 14 Mar 2026 04:30:48 +0000 Subject: [PATCH 33/42] mod/gateway: add comment --- mod/gateway/src/accept.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index d475c5f10..065957e64 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -150,6 +150,7 @@ func pipe(a, b io.ReadWriteCloser) { done := make(chan struct{}, 2) copy := func(dst, src io.ReadWriteCloser) { + // note: sync.Pool could reduce per-connection allocations under high concurrency (pattern used by nginx, envoy, treafik) buf := make([]byte, 32*1024) srcD, srcOk := src.(deadliner) dstD, dstOk := dst.(deadliner) From fdedad9dd60e6f7f1770550a69550e017a18ef17 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sat, 14 Mar 2026 05:12:11 +0000 Subject: [PATCH 34/42] mod/gateway: simplification --- mod/gateway/src/accept.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 065957e64..b5f457f8e 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -106,15 +106,17 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi } const ( - socketSignalByte = byte(1) - socketProbeTimeout = 1 * time.Second + socketSignalByte = byte(1) + socketProbeMaxAttempts = 3 + socketProbeTimeout = 1 * time.Second ) -// probeBinderConn signals each binderConn in the pool to verify it is alive, -// discarding dead connections until one succeeds or the pool is exhausted. +// probeBinderConn signals binderConns to verify they are alive. +// It will try at most socketProbeMaxAttempts connections before giving up. func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { candidate := first - for { + + for attempts := 0; attempts < socketProbeMaxAttempts; attempts++ { if candidate == nil { var ok bool candidate, ok = b.takeConn() @@ -123,20 +125,16 @@ func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { } } - if d, ok := candidate.Conn.(deadliner); ok { - d.SetWriteDeadline(time.Now().Add(socketProbeTimeout)) - } - _, err := candidate.Write([]byte{socketSignalByte}) - if d, ok := candidate.Conn.(deadliner); ok { - d.SetWriteDeadline(time.Time{}) - } - if err == nil { + if _, err := candidate.Write([]byte{socketSignalByte}); err == nil { return candidate } candidate.Close() candidate = nil } + + mod.log.Errorv(1, "binder %v probe exhausted", b.Identity) + return nil } type deadliner interface { From 22b969cf90c38f87944100e260b0bd20c10e526c Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sat, 14 Mar 2026 15:22:49 +0000 Subject: [PATCH 35/42] mod/gateway: add singalling on gw --- mod/gateway/src/accept.go | 5 +++-- mod/gateway/src/pool.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index b5f457f8e..741a690e4 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -9,6 +9,7 @@ import ( "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" "github.com/cryptopunkscc/astrald/mod/tcp" ) @@ -106,7 +107,6 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi } const ( - socketSignalByte = byte(1) socketProbeMaxAttempts = 3 socketProbeTimeout = 1 * time.Second ) @@ -125,7 +125,8 @@ func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { } } - if _, err := candidate.Write([]byte{socketSignalByte}); err == nil { + ping := gateway.PingFrame{Ping: true, Stop: true} + if _, err := ping.WriteTo(candidate); err == nil { return candidate } diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index cc8e7cd7b..a520f342b 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -133,8 +133,8 @@ func (p *SocketPool) handoff(conn exonet.Conn) { go func() { // wait for the gateway's start signal before beginning link negotiation - var sig [1]byte - if _, err := pc.Read(sig[:]); err != nil { + var ping gateway.PingFrame + if _, err := ping.ReadFrom(pc); err != nil { pc.Close() return } From f0e158718b73ea69639078341b5aa4aaf605aabf Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sat, 14 Mar 2026 15:24:39 +0000 Subject: [PATCH 36/42] mod/gateway: pingFrame --- mod/gateway/ping_frame.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 mod/gateway/ping_frame.go diff --git a/mod/gateway/ping_frame.go b/mod/gateway/ping_frame.go new file mode 100644 index 000000000..f1846f486 --- /dev/null +++ b/mod/gateway/ping_frame.go @@ -0,0 +1,24 @@ +package gateway + +import ( + "io" + + "github.com/cryptopunkscc/astrald/astral" +) + +// PingFrame is exchanged between the gateway and a binder's idle socket conn. +// Ping=true is a ping (gateway→binder); Ping=false is a pong (binder→gateway). +// Stop=true signals the binder to stop the ping loop and proceed to link establishment. +type PingFrame struct { + Ping astral.Bool + Stop astral.Bool +} + +func (PingFrame) ObjectType() string { return "mod.gateway.ping_frame" } + +func (p PingFrame) WriteTo(w io.Writer) (int64, error) { return astral.Objectify(&p).WriteTo(w) } +func (p *PingFrame) ReadFrom(r io.Reader) (int64, error) { return astral.Objectify(p).ReadFrom(r) } + +func init() { + astral.Add(&PingFrame{}) +} From 8cc1e6aa59624d435250397780765338a6847002 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sat, 14 Mar 2026 19:59:01 +0000 Subject: [PATCH 37/42] mod/gateway: WIP --- mod/gateway/ping_frame.go | 26 ++-------- mod/gateway/src/accept.go | 101 +++++++++++++++++++++++++++++++++++--- mod/gateway/src/binder.go | 14 ++++-- mod/gateway/src/pool.go | 49 +++++++++++++++--- 4 files changed, 148 insertions(+), 42 deletions(-) diff --git a/mod/gateway/ping_frame.go b/mod/gateway/ping_frame.go index f1846f486..2a61b091e 100644 --- a/mod/gateway/ping_frame.go +++ b/mod/gateway/ping_frame.go @@ -1,24 +1,8 @@ package gateway -import ( - "io" - - "github.com/cryptopunkscc/astrald/astral" +const ( + BytePing = byte(0x00) + BytePong = byte(0x01) + ByteSignalGo = byte(0x02) + ByteSignalReady = byte(0x03) ) - -// PingFrame is exchanged between the gateway and a binder's idle socket conn. -// Ping=true is a ping (gateway→binder); Ping=false is a pong (binder→gateway). -// Stop=true signals the binder to stop the ping loop and proceed to link establishment. -type PingFrame struct { - Ping astral.Bool - Stop astral.Bool -} - -func (PingFrame) ObjectType() string { return "mod.gateway.ping_frame" } - -func (p PingFrame) WriteTo(w io.Writer) (int64, error) { return astral.Objectify(&p).WriteTo(w) } -func (p *PingFrame) ReadFrom(r io.Reader) (int64, error) { return astral.Objectify(p).ReadFrom(r) } - -func init() { - astral.Add(&PingFrame{}) -} diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 741a690e4..2596a09b1 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -61,7 +61,8 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi if b, ok := mod.binderByNonce(nonce); ok { mod.log.Infov(2, "added idle conn to binder %v", b.Identity) - b.addConn(conn) + bc := b.addConn(conn) + go mod.keepalive(bc) return stopListener, nil } @@ -107,11 +108,75 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi } const ( + socketPingInterval = 2 * time.Second + socketPingTimeout = 3 * time.Second + socketDeadTimeout = 10 * time.Second socketProbeMaxAttempts = 3 - socketProbeTimeout = 1 * time.Second + socketProbeTimeout = 5 * time.Second ) -// probeBinderConn signals binderConns to verify they are alive. +// keepalive runs on the gateway side for each idle binder conn. +// It reads binder pings and responds with pong. When a connector arrives +// via goCh it sends ByteSignalGo and waits for ByteSignalReady. +func (mod *Module) keepalive(bc *binderConn) { + defer close(bc.done) + defer bc.Close() + + for { + if d, ok := bc.Conn.(deadliner); ok { + d.SetReadDeadline(time.Now().Add(socketDeadTimeout)) + } + var b [1]byte + if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { + return + } + if d, ok := bc.Conn.(deadliner); ok { + d.SetReadDeadline(time.Time{}) + } + + switch b[0] { + case gateway.BytePing: + select { + case respCh := <-bc.goCh: + mod.sendSignal(bc, respCh) + return + default: + if _, err := bc.Conn.Write([]byte{gateway.BytePong}); err != nil { + return + } + } + default: + return + } + } +} + +// sendSignal sends ByteSignalGo to the binder and waits for ByteSignalReady. +// Called from keepalive when a connector signals via goCh. +func (mod *Module) sendSignal(bc *binderConn, respCh chan error) { + if _, err := bc.Conn.Write([]byte{gateway.ByteSignalGo}); err != nil { + respCh <- err + return + } + if d, ok := bc.Conn.(deadliner); ok { + d.SetReadDeadline(time.Now().Add(socketProbeTimeout)) + } + var b [1]byte + if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { + respCh <- err + return + } + if d, ok := bc.Conn.(deadliner); ok { + d.SetReadDeadline(time.Time{}) + } + if b[0] != gateway.ByteSignalReady { + respCh <- fmt.Errorf("expected signalReady, got 0x%02x", b[0]) + return + } + respCh <- nil +} + +// probeBinderConn signals a binderConn via its ping loop to verify liveness. // It will try at most socketProbeMaxAttempts connections before giving up. func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { candidate := first @@ -125,13 +190,33 @@ func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { } } - ping := gateway.PingFrame{Ping: true, Stop: true} - if _, err := ping.WriteTo(candidate); err == nil { - return candidate + respCh := make(chan error, 1) + select { + case candidate.goCh <- respCh: + case <-time.After(socketProbeTimeout): + // goCh buffer full — another connector is already signaling this conn + candidate.Close() + candidate = nil + continue } - candidate.Close() - candidate = nil + select { + case err := <-respCh: + if err != nil { + // sendSignal failed; keepalive already exited via defer, conn is closed + candidate.Close() + candidate = nil + continue + } + return candidate + case <-candidate.done: + // keepalive exited without responding — conn was dead + candidate = nil + continue + case <-time.After(socketProbeTimeout + socketPingInterval): + candidate.Close() + candidate = nil + } } mod.log.Errorv(1, "binder %v probe exhausted", b.Identity) diff --git a/mod/gateway/src/binder.go b/mod/gateway/src/binder.go index c42fc1e09..08ac79ccb 100644 --- a/mod/gateway/src/binder.go +++ b/mod/gateway/src/binder.go @@ -3,6 +3,7 @@ package gateway import ( "errors" "sync" + "sync/atomic" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" @@ -25,11 +26,14 @@ type binderConn struct { state connState pipedTo *connectorConn onClose func() + goCh chan chan error // connector signals the ping loop to send a Signal frame + done chan struct{} // closed when keepalive exits + closed atomic.Bool } func (bc *binderConn) Close() error { err := bc.Conn.Close() - if bc.onClose != nil { + if !bc.closed.Swap(true) && bc.onClose != nil { bc.onClose() } return err @@ -45,13 +49,16 @@ type binder struct { conns sig.Set[*binderConn] } -func (b *binder) addConn(conn exonet.Conn) { +func (b *binder) addConn(conn exonet.Conn) *binderConn { bc := &binderConn{ Conn: conn, state: connStateIdle, + goCh: make(chan chan error, 1), + done: make(chan struct{}), } bc.onClose = func() { b.conns.Remove(bc) } b.conns.Add(bc) + return bc } // takeConn reserves an idle binderConn for a connector. @@ -79,9 +86,6 @@ func (b *binder) Close() error { defer b.mu.Unlock() var errs []error for _, bc := range b.conns.Clone() { - if bc.state == connStateReserved { - continue - } errs = append(errs, bc.Close()) } return errors.Join(errs...) diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index a520f342b..6cf088374 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -1,6 +1,7 @@ package gateway import ( + "io" "sync" "sync/atomic" "time" @@ -132,14 +133,46 @@ func (p *SocketPool) handoff(conn exonet.Conn) { p.addIdle() go func() { - // wait for the gateway's start signal before beginning link negotiation - var ping gateway.PingFrame - if _, err := ping.ReadFrom(pc); err != nil { - pc.Close() - return - } - if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { - pc.Close() + // Ping loop: send BytePing, await BytePong or ByteSignalGo. + // Pong responses are written to raw conn so onConnTaken fires only on ByteSignalReady. + for { + if _, err := conn.Write([]byte{gateway.BytePing}); err != nil { + pc.Close() + return + } + if d, ok := conn.(deadliner); ok { + d.SetReadDeadline(time.Now().Add(socketPingTimeout)) + } + var b [1]byte + if _, err := io.ReadFull(conn, b[:]); err != nil { + pc.Close() + return + } + if d, ok := conn.(deadliner); ok { + d.SetReadDeadline(time.Time{}) + } + switch b[0] { + case gateway.BytePong: + select { + case <-time.After(socketPingInterval): + case <-p.ctx.Done(): + pc.Close() + return + } + case gateway.ByteSignalGo: + // write ByteSignalReady through pc to trigger onConnTaken + if _, err := pc.Write([]byte{gateway.ByteSignalReady}); err != nil { + pc.Close() + return + } + if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { + pc.Close() + } + return + default: + pc.Close() + return + } } }() } From 569f2730f8905be402a6109b634d2e247d52ddb1 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sat, 14 Mar 2026 20:10:58 +0000 Subject: [PATCH 38/42] mod/gateway: WIP --- mod/gateway/src/accept.go | 8 +++++++- mod/gateway/src/pool.go | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 2596a09b1..e248d388e 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -120,7 +120,12 @@ const ( // via goCh it sends ByteSignalGo and waits for ByteSignalReady. func (mod *Module) keepalive(bc *binderConn) { defer close(bc.done) - defer bc.Close() + piped := false + defer func() { + if !piped { + bc.Close() + } + }() for { if d, ok := bc.Conn.(deadliner); ok { @@ -138,6 +143,7 @@ func (mod *Module) keepalive(bc *binderConn) { case gateway.BytePing: select { case respCh := <-bc.goCh: + piped = true // pipe or probeBinderConn takes ownership; don't close on exit mod.sendSignal(bc, respCh) return default: diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index 6cf088374..b1f259af9 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -165,6 +165,8 @@ func (p *SocketPool) handoff(conn exonet.Conn) { pc.Close() return } + + // if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { pc.Close() } From c5c7af9f0e585679ae3933b1fbe17d654a696739 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sun, 15 Mar 2026 16:36:11 +0000 Subject: [PATCH 39/42] mod/gateway: cleanups WIP --- mod/gateway/ping_frame.go | 7 + mod/gateway/src/accept.go | 197 ++--------------------- mod/gateway/src/bind.go | 14 +- mod/gateway/src/binder.go | 154 ++++++++++++++++-- mod/gateway/src/connect.go | 16 +- mod/gateway/src/maintain_binding_task.go | 2 +- mod/gateway/src/op_node_bind.go | 2 +- mod/gateway/src/pool.go | 110 ++++++------- 8 files changed, 223 insertions(+), 279 deletions(-) diff --git a/mod/gateway/ping_frame.go b/mod/gateway/ping_frame.go index 2a61b091e..aeecaaeb9 100644 --- a/mod/gateway/ping_frame.go +++ b/mod/gateway/ping_frame.go @@ -1,8 +1,15 @@ package gateway +import "io" + const ( BytePing = byte(0x00) BytePong = byte(0x01) ByteSignalGo = byte(0x02) ByteSignalReady = byte(0x03) ) + +func WritePing(w io.Writer) error { _, err := w.Write([]byte{BytePing}); return err } +func WritePong(w io.Writer) error { _, err := w.Write([]byte{BytePong}); return err } +func WriteSignalGo(w io.Writer) error { _, err := w.Write([]byte{ByteSignalGo}); return err } +func WriteSignalReady(w io.Writer) error { _, err := w.Write([]byte{ByteSignalReady}); return err } diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index e248d388e..12bfb0607 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -3,52 +3,20 @@ package gateway import ( "context" "fmt" - "io" - "strings" "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" - "github.com/cryptopunkscc/astrald/mod/tcp" ) -func (mod *Module) startServers(ctx *astral.Context) { - for _, addr := range mod.config.Gateway.Listen { - parts := strings.SplitN(addr, ":", 2) - if len(parts) != 2 { - mod.log.Error("invalid listen address: %v", addr) - continue - } - network, address := parts[0], parts[1] - endpoint, err := mod.Exonet.Parse(network, address) - if err != nil { - mod.log.Error("parse listen address %v: %v", addr, err) - continue - } - - switch network { - case "tcp": - tcpEndpoint, ok := endpoint.(*tcp.Endpoint) - if !ok { - mod.log.Error("invalid listen address: %v", addr) - continue - } - - mod.log.Logv(1, "start listening on %v", tcpEndpoint) - if err := mod.TCP.CreateEphemeralListener(ctx, tcpEndpoint.Port, mod.acceptSocketConn); err != nil { - mod.log.Error("create ephemeral listener on %v: %v", addr, err) - continue - } - - mod.listenEndpoints.Set("tcp", tcpEndpoint) - default: - mod.log.Error("unsupported gateway socket network: %v", network) - } - } -} +const ( + socketDeadTimeout = 10 * time.Second + socketProbeMaxAttempts = 3 + socketProbeTimeout = 5 * time.Second +) -// acceptSocketConn accepts connection on the socket that gateway told client to connect to. +// acceptSocketConn dispatches an incoming socket connection to either the binder +// or connector path based on the nonce it presents. func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopListener bool, err error) { mod.log.Logv(2, "accepting socket connection from %v", conn.RemoteEndpoint()) @@ -62,7 +30,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi if b, ok := mod.binderByNonce(nonce); ok { mod.log.Infov(2, "added idle conn to binder %v", b.Identity) bc := b.addConn(conn) - go mod.keepalive(bc) + go bc.keepalive() return stopListener, nil } @@ -107,85 +75,10 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi return stopListener, nil } -const ( - socketPingInterval = 2 * time.Second - socketPingTimeout = 3 * time.Second - socketDeadTimeout = 10 * time.Second - socketProbeMaxAttempts = 3 - socketProbeTimeout = 5 * time.Second -) - -// keepalive runs on the gateway side for each idle binder conn. -// It reads binder pings and responds with pong. When a connector arrives -// via goCh it sends ByteSignalGo and waits for ByteSignalReady. -func (mod *Module) keepalive(bc *binderConn) { - defer close(bc.done) - piped := false - defer func() { - if !piped { - bc.Close() - } - }() - - for { - if d, ok := bc.Conn.(deadliner); ok { - d.SetReadDeadline(time.Now().Add(socketDeadTimeout)) - } - var b [1]byte - if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { - return - } - if d, ok := bc.Conn.(deadliner); ok { - d.SetReadDeadline(time.Time{}) - } - - switch b[0] { - case gateway.BytePing: - select { - case respCh := <-bc.goCh: - piped = true // pipe or probeBinderConn takes ownership; don't close on exit - mod.sendSignal(bc, respCh) - return - default: - if _, err := bc.Conn.Write([]byte{gateway.BytePong}); err != nil { - return - } - } - default: - return - } - } -} - -// sendSignal sends ByteSignalGo to the binder and waits for ByteSignalReady. -// Called from keepalive when a connector signals via goCh. -func (mod *Module) sendSignal(bc *binderConn, respCh chan error) { - if _, err := bc.Conn.Write([]byte{gateway.ByteSignalGo}); err != nil { - respCh <- err - return - } - if d, ok := bc.Conn.(deadliner); ok { - d.SetReadDeadline(time.Now().Add(socketProbeTimeout)) - } - var b [1]byte - if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { - respCh <- err - return - } - if d, ok := bc.Conn.(deadliner); ok { - d.SetReadDeadline(time.Time{}) - } - if b[0] != gateway.ByteSignalReady { - respCh <- fmt.Errorf("expected signalReady, got 0x%02x", b[0]) - return - } - respCh <- nil -} - // probeBinderConn signals a binderConn via its ping loop to verify liveness. // It will try at most socketProbeMaxAttempts connections before giving up. -func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { - candidate := first +func (mod *Module) probeBinderConn(b *binder, reserved *binderConn) *binderConn { + candidate := reserved for attempts := 0; attempts < socketProbeMaxAttempts; attempts++ { if candidate == nil { @@ -196,78 +89,12 @@ func (mod *Module) probeBinderConn(b *binder, first *binderConn) *binderConn { } } - respCh := make(chan error, 1) - select { - case candidate.goCh <- respCh: - case <-time.After(socketProbeTimeout): - // goCh buffer full — another connector is already signaling this conn - candidate.Close() - candidate = nil - continue - } - - select { - case err := <-respCh: - if err != nil { - // sendSignal failed; keepalive already exited via defer, conn is closed - candidate.Close() - candidate = nil - continue - } + if candidate.signal() { return candidate - case <-candidate.done: - // keepalive exited without responding — conn was dead - candidate = nil - continue - case <-time.After(socketProbeTimeout + socketPingInterval): - candidate.Close() - candidate = nil } + candidate = nil } mod.log.Errorv(1, "binder %v probe exhausted", b.Identity) return nil } - -type deadliner interface { - SetReadDeadline(time.Time) error - SetWriteDeadline(time.Time) error -} - -func pipe(a, b io.ReadWriteCloser) { - const idle = 30 * time.Second - - done := make(chan struct{}, 2) - - copy := func(dst, src io.ReadWriteCloser) { - // note: sync.Pool could reduce per-connection allocations under high concurrency (pattern used by nginx, envoy, treafik) - buf := make([]byte, 32*1024) - srcD, srcOk := src.(deadliner) - dstD, dstOk := dst.(deadliner) - for { - if srcOk { - srcD.SetReadDeadline(time.Now().Add(idle)) - } - n, err := src.Read(buf) - if n > 0 { - if dstOk { - dstD.SetWriteDeadline(time.Now().Add(idle)) - } - if _, werr := dst.Write(buf[:n]); werr != nil { - break - } - } - if err != nil { - break - } - } - done <- struct{}{} - } - - go copy(a, b) - go copy(b, a) - - <-done - a.Close() - b.Close() -} diff --git a/mod/gateway/src/bind.go b/mod/gateway/src/bind.go index 1c4b46509..d4b006da2 100644 --- a/mod/gateway/src/bind.go +++ b/mod/gateway/src/bind.go @@ -5,23 +5,23 @@ import ( "github.com/cryptopunkscc/astrald/mod/gateway" ) -func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibility gateway.Visibility, network string) (*gateway.Socket, error) { +func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibility gateway.Visibility, network string) (gateway.Socket, error) { if !mod.canGateway(identity) { - return nil, gateway.ErrUnauthorized + return gateway.Socket{}, gateway.ErrUnauthorized } endpoint, err := mod.getGatewayEndpoint(ctx, network) if err != nil { - return nil, err + return gateway.Socket{}, err } - b := &binder{ + newBinder := &binder{ Identity: identity, Nonce: astral.NewNonce(), Visibility: visibility, } - oldBinder, ok := mod.binders.Replace(identity.String(), b) + oldBinder, ok := mod.binders.Replace(identity.String(), newBinder) if ok { if err = oldBinder.Close(); err != nil { mod.log.Error("failed to close old binder: %v", err) @@ -36,8 +36,8 @@ func (mod *Module) bind(ctx *astral.Context, identity *astral.Identity, visibili } } - return &gateway.Socket{ - Nonce: b.Nonce, + return gateway.Socket{ + Nonce: newBinder.Nonce, Endpoint: endpoint, }, nil } diff --git a/mod/gateway/src/binder.go b/mod/gateway/src/binder.go index 08ac79ccb..e517d47cb 100644 --- a/mod/gateway/src/binder.go +++ b/mod/gateway/src/binder.go @@ -2,8 +2,11 @@ package gateway import ( "errors" + "fmt" + "io" "sync" "sync/atomic" + "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" @@ -11,26 +14,55 @@ import ( "github.com/cryptopunkscc/astrald/sig" ) +// deadliner is implemented by connections that support read/write deadlines (e.g. net.Conn). +type deadliner interface { + SetReadDeadline(time.Time) error + SetWriteDeadline(time.Time) error +} + +func withReadDeadline(conn exonet.Conn, d time.Duration) { + if dl, ok := conn.(deadliner); ok { + dl.SetReadDeadline(time.Now().Add(d)) + } +} + +func withWriteDeadline(conn exonet.Conn, d time.Duration) { + if dl, ok := conn.(deadliner); ok { + dl.SetWriteDeadline(time.Now().Add(d)) + } +} + +func clearDeadlines(conn exonet.Conn) { + if dl, ok := conn.(deadliner); ok { + dl.SetReadDeadline(time.Time{}) + dl.SetWriteDeadline(time.Time{}) + } +} + type connState uint8 const ( - connStateIdle connState = iota - connStateReserved connState = iota - connStatePiped connState = iota + stateIdle connState = iota + stateReserved connState = iota + statePiped connState = iota ) // binderConn is a connection pre-opened by a binder node to the gateway, // sitting idle until a connector claims it. type binderConn struct { exonet.Conn - state connState - pipedTo *connectorConn - onClose func() - goCh chan chan error // connector signals the ping loop to send a Signal frame - done chan struct{} // closed when keepalive exits - closed atomic.Bool + state connState + pipedTo *connectorConn + onClose func() + signalCh chan chan error // connector requests keepalive to send ByteSignalGo + dead chan struct{} // closed when the connection is no longer alive + closed atomic.Bool } +func (bc *binderConn) IsIdle() bool { return bc.state == stateIdle } +func (bc *binderConn) IsReserved() bool { return bc.state == stateReserved } +func (bc *binderConn) IsPiped() bool { return bc.state == statePiped } + func (bc *binderConn) Close() error { err := bc.Conn.Close() if !bc.closed.Swap(true) && bc.onClose != nil { @@ -39,6 +71,96 @@ func (bc *binderConn) Close() error { return err } +// keepalive reads binder pings and responds with pong. +// When a connector signals via signalCh it sends ByteSignalGo and waits for ByteSignalReady. +func (bc *binderConn) keepalive() { + defer close(bc.dead) + // piped tracks whether pipe() has taken ownership of bc. + // If true, pipe() is responsible for closing — keepalive must not. + piped := false + defer func() { + if !piped { + bc.Close() + } + }() + + for { + withReadDeadline(bc.Conn, socketDeadTimeout) + var b [1]byte + if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { + return + } + clearDeadlines(bc.Conn) + + switch b[0] { + case gateway.BytePing: + select { + case respCh := <-bc.signalCh: + piped = true // pipe or probeBinderConn takes ownership; don't close on exit + bc.sendSignal(respCh) + return + default: + if err := gateway.WritePong(bc.Conn); err != nil { + return + } + } + default: + return + } + } +} + +// sendSignal sends ByteSignalGo to the binder and waits for ByteSignalReady. +// Called from keepalive when a connector signals via signalCh. +func (bc *binderConn) sendSignal(respCh chan error) { + withWriteDeadline(bc.Conn, socketProbeTimeout) + if err := gateway.WriteSignalGo(bc.Conn); err != nil { + respCh <- err + return + } + clearDeadlines(bc.Conn) + withReadDeadline(bc.Conn, socketProbeTimeout) + var b [1]byte + if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { + respCh <- err + return + } + clearDeadlines(bc.Conn) + if b[0] != gateway.ByteSignalReady { + respCh <- fmt.Errorf("expected signalReady, got 0x%02x", b[0]) + return + } + respCh <- nil +} + +// signal queues a ByteSignalGo request via the keepalive loop and waits for the result. +// Returns true if the conn is alive and was signaled successfully. +// Closes bc on failure unless keepalive already exited. +func (bc *binderConn) signal() bool { + respCh := make(chan error, 1) + select { + case bc.signalCh <- respCh: + case <-time.After(socketProbeTimeout): + // signalCh buffer full — another connector is already signaling this conn + bc.Close() + return false + } + + select { + case err := <-respCh: + if err != nil { + bc.Close() + } + return err == nil + case <-bc.dead: + // keepalive exited — conn was dead + return false + case <-time.After(socketProbeTimeout + socketPingInterval): + bc.Close() + return false + } +} + // binder represents a node registered as reachable through the gateway. // Only one binder registration per identity is allowed. type binder struct { @@ -51,10 +173,10 @@ type binder struct { func (b *binder) addConn(conn exonet.Conn) *binderConn { bc := &binderConn{ - Conn: conn, - state: connStateIdle, - goCh: make(chan chan error, 1), - done: make(chan struct{}), + Conn: conn, + state: stateIdle, + signalCh: make(chan chan error, 1), + dead: make(chan struct{}), } bc.onClose = func() { b.conns.Remove(bc) } b.conns.Add(bc) @@ -66,8 +188,8 @@ func (b *binder) takeConn() (*binderConn, bool) { b.mu.Lock() defer b.mu.Unlock() for _, bc := range b.conns.Clone() { - if bc.state == connStateIdle { - bc.state = connStateReserved + if bc.IsIdle() { + bc.state = stateReserved return bc, true } } @@ -77,7 +199,7 @@ func (b *binder) takeConn() (*binderConn, bool) { func (b *binder) markPiped(bc *binderConn, cc *connectorConn) { b.mu.Lock() defer b.mu.Unlock() - bc.state = connStatePiped + bc.state = statePiped bc.pipedTo = cc } diff --git a/mod/gateway/src/connect.go b/mod/gateway/src/connect.go index 930542326..bc802e871 100644 --- a/mod/gateway/src/connect.go +++ b/mod/gateway/src/connect.go @@ -9,9 +9,9 @@ import ( const connectTimeout = 30 * time.Second -func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, network string) (socket gateway.Socket, err error) { +func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, network string) (gateway.Socket, error) { if !mod.canGateway(caller) { - return socket, gateway.ErrUnauthorized + return gateway.Socket{}, gateway.ErrUnauthorized } endpoint, err := mod.getGatewayEndpoint(mod.ctx, network) @@ -19,14 +19,14 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n return gateway.Socket{}, err } - b, ok := mod.binderByIdentity(target) + binder, ok := mod.binderByIdentity(target) if !ok { - return socket, gateway.ErrTargetNotReachable + return gateway.Socket{}, gateway.ErrTargetNotReachable } - reserved, ok := b.takeConn() + reserved, ok := binder.takeConn() if !ok { - return socket, gateway.ErrTargetNotReachable + return gateway.Socket{}, gateway.ErrTargetNotReachable } c := &connector{ @@ -39,7 +39,9 @@ func (mod *Module) connectTo(caller *astral.Identity, target *astral.Identity, n mod.connectors.Add(c) go func() { - <-time.After(connectTimeout) + t := time.NewTimer(connectTimeout) + defer t.Stop() + <-t.C bc := c.takeReserved() if bc == nil { diff --git a/mod/gateway/src/maintain_binding_task.go b/mod/gateway/src/maintain_binding_task.go index 186430c52..fccbf79ee 100644 --- a/mod/gateway/src/maintain_binding_task.go +++ b/mod/gateway/src/maintain_binding_task.go @@ -71,7 +71,7 @@ func (task *MaintainBindingTask) Run(ctx *astral.Context) error { } count = 0 - err = task.mod.newSocketPool(ctx, task.GatewayID, socket).Run() + err = task.mod.newSocketPool(ctx, task.GatewayID, *socket).Run() if err != nil { task.mod.log.Error("rebinding to %v due to: %v", task.GatewayID, err) } diff --git a/mod/gateway/src/op_node_bind.go b/mod/gateway/src/op_node_bind.go index e8984c908..3dc0ecb4c 100644 --- a/mod/gateway/src/op_node_bind.go +++ b/mod/gateway/src/op_node_bind.go @@ -26,5 +26,5 @@ func (mod *Module) OpNodeBind( return ch.Send(astral.NewError(err.Error())) } - return ch.Send(socket) + return ch.Send(&socket) } diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index b1f259af9..0b6a35ec5 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -7,7 +7,6 @@ import ( "time" "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/astral/log" "github.com/cryptopunkscc/astrald/mod/exonet" "github.com/cryptopunkscc/astrald/mod/gateway" "github.com/cryptopunkscc/astrald/sig" @@ -16,14 +15,15 @@ import ( const ( socketPoolTargetIdle = 2 socketPoolMaxFails = 3 + socketPingInterval = 2 * time.Second + socketPingTimeout = 3 * time.Second ) type SocketPool struct { *Module ctx *astral.Context - socket *gateway.Socket + socket gateway.Socket gatewayID *astral.Identity - log *log.Logger mu sync.Mutex total int @@ -32,13 +32,12 @@ type SocketPool struct { signal chan struct{} } -func (mod *Module) newSocketPool(ctx *astral.Context, gatewayID *astral.Identity, socket *gateway.Socket) *SocketPool { +func (mod *Module) newSocketPool(ctx *astral.Context, gatewayID *astral.Identity, socket gateway.Socket) *SocketPool { return &SocketPool{ ctx: ctx, Module: mod, socket: socket, gatewayID: gatewayID, - log: mod.log, signal: make(chan struct{}, 1), } } @@ -52,7 +51,7 @@ func (p *SocketPool) Run() error { case <-p.ctx.Done(): return p.ctx.Err() case <-p.signal: - for toAdd := socketPoolTargetIdle - p.idleCount(); toAdd > 0; toAdd-- { + for p.idleCount() < socketPoolTargetIdle { conn, err := p.acquireConn() if err != nil { select { @@ -63,7 +62,6 @@ func (p *SocketPool) Run() error { return gateway.ErrSocketUnreachable } } - toAdd++ continue } @@ -115,7 +113,6 @@ func (p *SocketPool) onConnClosed(wasIdle bool) { if wasIdle { p.idle-- } - p.mu.Unlock() p.notify() } @@ -126,57 +123,58 @@ func (p *SocketPool) handoff(conn exonet.Conn) { localEndpoint: gateway.NewEndpoint(p.node.Identity(), p.node.Identity()), remoteEndpoint: gateway.NewEndpoint(p.gatewayID, p.node.Identity()), } + p.registerConn(pc) + go p.runIdleConn(conn, pc) +} - // when first write is done it means we started responding to link negotiation - pc.onFirstWrite = p.onConnTaken +// registerConn binds pool lifecycle callbacks to pc and marks it as idle. +func (p *SocketPool) registerConn(pc *socketConn) { pc.onClose = func() { p.onConnClosed(!pc.used.Load()) } p.addIdle() +} - go func() { - // Ping loop: send BytePing, await BytePong or ByteSignalGo. - // Pong responses are written to raw conn so onConnTaken fires only on ByteSignalReady. - for { - if _, err := conn.Write([]byte{gateway.BytePing}); err != nil { +// runIdleConn is the binder-side ping loop for an idle socket connection. +// It sends BytePing and waits for BytePong or ByteSignalGo. +// On ByteSignalGo the conn transitions from idle to taken before writing ByteSignalReady. +func (p *SocketPool) runIdleConn(conn exonet.Conn, pc *socketConn) { + for { + if err := gateway.WritePing(conn); err != nil { + pc.Close() + return + } + withReadDeadline(conn, socketPingTimeout) + var b [1]byte + if _, err := io.ReadFull(conn, b[:]); err != nil { + pc.Close() + return + } + clearDeadlines(conn) + switch b[0] { + case gateway.BytePong: + select { + case <-time.After(socketPingInterval): + case <-p.ctx.Done(): pc.Close() return } - if d, ok := conn.(deadliner); ok { - d.SetReadDeadline(time.Now().Add(socketPingTimeout)) - } - var b [1]byte - if _, err := io.ReadFull(conn, b[:]); err != nil { + case gateway.ByteSignalGo: + // Mark taken before notifying pool: if Close fires during write, + // onClose sees used=true → wasIdle=false → only total is decremented. + pc.used.Store(true) + p.onConnTaken() + if err := gateway.WriteSignalReady(pc); err != nil { pc.Close() return } - if d, ok := conn.(deadliner); ok { - d.SetReadDeadline(time.Time{}) - } - switch b[0] { - case gateway.BytePong: - select { - case <-time.After(socketPingInterval): - case <-p.ctx.Done(): - pc.Close() - return - } - case gateway.ByteSignalGo: - // write ByteSignalReady through pc to trigger onConnTaken - if _, err := pc.Write([]byte{gateway.ByteSignalReady}); err != nil { - pc.Close() - return - } - - // - if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { - pc.Close() - } - return - default: + if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { pc.Close() - return } + return + default: + pc.Close() + return } - }() + } } func (p *SocketPool) notify() { @@ -186,13 +184,13 @@ func (p *SocketPool) notify() { } } -// socketConn is considered a connection only after the first write is done. +// socketConn wraps an exonet.Conn with fixed endpoints and a one-shot close callback. +// used tracks whether the conn has been taken from the idle pool (for wasIdle accounting). type socketConn struct { exonet.Conn localEndpoint exonet.Endpoint remoteEndpoint exonet.Endpoint - onFirstWrite func() onClose func() closed atomic.Bool @@ -202,22 +200,10 @@ type socketConn struct { func (c *socketConn) LocalEndpoint() exonet.Endpoint { return c.localEndpoint } func (c *socketConn) RemoteEndpoint() exonet.Endpoint { return c.remoteEndpoint } -func (c *socketConn) Write(b []byte) (int, error) { - if !c.used.Swap(true) && c.onFirstWrite != nil { - c.onFirstWrite() - } - - return c.Conn.Write(b) -} - func (c *socketConn) Close() error { err := c.Conn.Close() - - if !c.closed.Swap(true) { - if c.onClose != nil { - c.onClose() - } + if !c.closed.Swap(true) && c.onClose != nil { + c.onClose() } - return err } From e68030c488f1b21ec331f7f85a64b6af4775d08a Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sun, 15 Mar 2026 16:52:07 +0000 Subject: [PATCH 40/42] mod/gateway: cleanups WIP --- mod/gateway/src/pipe.go | 44 +++++++++++++++++++++++++++++++++++++++ mod/gateway/src/server.go | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 mod/gateway/src/pipe.go create mode 100644 mod/gateway/src/server.go diff --git a/mod/gateway/src/pipe.go b/mod/gateway/src/pipe.go new file mode 100644 index 000000000..4d8504d9f --- /dev/null +++ b/mod/gateway/src/pipe.go @@ -0,0 +1,44 @@ +package gateway + +import ( + "io" + "time" +) + +func pipe(a, b io.ReadWriteCloser) { + const idle = 30 * time.Second + + done := make(chan struct{}, 2) + + forward := func(dst, src io.ReadWriteCloser) { + // note: sync.Pool could reduce per-connection allocations under high concurrency (pattern used by nginx, envoy, traefik) + buf := make([]byte, 32*1024) + srcD, srcOk := src.(deadliner) + dstD, dstOk := dst.(deadliner) + for { + if srcOk { + srcD.SetReadDeadline(time.Now().Add(idle)) + } + n, err := src.Read(buf) + if n > 0 { + if dstOk { + dstD.SetWriteDeadline(time.Now().Add(idle)) + } + if _, werr := dst.Write(buf[:n]); werr != nil { + break + } + } + if err != nil { + break + } + } + done <- struct{}{} + } + + go forward(a, b) + go forward(b, a) + + <-done + a.Close() + b.Close() +} diff --git a/mod/gateway/src/server.go b/mod/gateway/src/server.go new file mode 100644 index 000000000..9100d4e49 --- /dev/null +++ b/mod/gateway/src/server.go @@ -0,0 +1,43 @@ +package gateway + +import ( + "strings" + + "github.com/cryptopunkscc/astrald/astral" + "github.com/cryptopunkscc/astrald/mod/tcp" +) + +func (mod *Module) startServers(ctx *astral.Context) { + for _, addr := range mod.config.Gateway.Listen { + parts := strings.SplitN(addr, ":", 2) + if len(parts) != 2 { + mod.log.Error("invalid listen address: %v", addr) + continue + } + network, address := parts[0], parts[1] + endpoint, err := mod.Exonet.Parse(network, address) + if err != nil { + mod.log.Error("parse listen address %v: %v", addr, err) + continue + } + + switch network { + case "tcp": + tcpEndpoint, ok := endpoint.(*tcp.Endpoint) + if !ok { + mod.log.Error("invalid listen address: %v", addr) + continue + } + + mod.log.Logv(1, "start listening on %v", tcpEndpoint) + if err := mod.TCP.CreateEphemeralListener(ctx, tcpEndpoint.Port, mod.acceptSocketConn); err != nil { + mod.log.Error("create ephemeral listener on %v: %v", addr, err) + continue + } + + mod.listenEndpoints.Set("tcp", tcpEndpoint) + default: + mod.log.Error("unsupported gateway socket network: %v", network) + } + } +} From a7d54796529595846d49f5e140edef494d804d19 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sun, 15 Mar 2026 23:16:54 +0000 Subject: [PATCH 41/42] mod/gateway: cleanups WIP --- mod/gateway/src/accept.go | 52 ++------- mod/gateway/src/binder.go | 176 ++---------------------------- mod/gateway/src/binding_conn.go | 182 ++++++++++++++++++++++++++++++++ mod/gateway/src/conn.go | 12 ++- mod/gateway/src/connector.go | 15 +-- mod/gateway/src/dial.go | 16 +-- mod/gateway/src/errors.go | 9 +- mod/gateway/src/pool.go | 133 ++++++----------------- mod/gateway/src/route.go | 8 +- 9 files changed, 251 insertions(+), 352 deletions(-) create mode 100644 mod/gateway/src/binding_conn.go diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 12bfb0607..8ba51c63b 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -10,9 +10,8 @@ import ( ) const ( - socketDeadTimeout = 10 * time.Second - socketProbeMaxAttempts = 3 - socketProbeTimeout = 5 * time.Second + socketDeadTimeout = 10 * time.Second + socketProbeTimeout = 5 * time.Second ) // acceptSocketConn dispatches an incoming socket connection to either the binder @@ -30,7 +29,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi if b, ok := mod.binderByNonce(nonce); ok { mod.log.Infov(2, "added idle conn to binder %v", b.Identity) bc := b.addConn(conn) - go bc.keepalive() + go bc.keepalive(nil, nil, nil) return stopListener, nil } @@ -49,52 +48,13 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi return stopListener, fmt.Errorf("no reserved conn for %v", c.Target) } - targetBinder, ok := mod.binderByIdentity(c.Target) - if !ok { - reserved.Close() + if !reserved.signal() { + mod.log.Errorv(1, "reserved conn for %v is dead", c.Target) conn.Close() return stopListener, nil } - alive := mod.probeBinderConn(targetBinder, reserved) - if alive == nil { - mod.log.Errorv(1, "no alive conn for %v", c.Target) - conn.Close() - return stopListener, nil - } - - cc := &connectorConn{ - Conn: conn, - network: conn.RemoteEndpoint().Network(), - pipedTo: alive, - } - - targetBinder.markPiped(alive, cc) mod.log.Infov(2, "pipe from %v to %v created", c.Identity, c.Target) - go pipe(alive, cc) + go pipe(reserved, conn) return stopListener, nil } - -// probeBinderConn signals a binderConn via its ping loop to verify liveness. -// It will try at most socketProbeMaxAttempts connections before giving up. -func (mod *Module) probeBinderConn(b *binder, reserved *binderConn) *binderConn { - candidate := reserved - - for attempts := 0; attempts < socketProbeMaxAttempts; attempts++ { - if candidate == nil { - var ok bool - candidate, ok = b.takeConn() - if !ok { - return nil - } - } - - if candidate.signal() { - return candidate - } - candidate = nil - } - - mod.log.Errorv(1, "binder %v probe exhausted", b.Identity) - return nil -} diff --git a/mod/gateway/src/binder.go b/mod/gateway/src/binder.go index e517d47cb..48cd8e0af 100644 --- a/mod/gateway/src/binder.go +++ b/mod/gateway/src/binder.go @@ -2,11 +2,7 @@ package gateway import ( "errors" - "fmt" - "io" "sync" - "sync/atomic" - "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" @@ -14,153 +10,6 @@ import ( "github.com/cryptopunkscc/astrald/sig" ) -// deadliner is implemented by connections that support read/write deadlines (e.g. net.Conn). -type deadliner interface { - SetReadDeadline(time.Time) error - SetWriteDeadline(time.Time) error -} - -func withReadDeadline(conn exonet.Conn, d time.Duration) { - if dl, ok := conn.(deadliner); ok { - dl.SetReadDeadline(time.Now().Add(d)) - } -} - -func withWriteDeadline(conn exonet.Conn, d time.Duration) { - if dl, ok := conn.(deadliner); ok { - dl.SetWriteDeadline(time.Now().Add(d)) - } -} - -func clearDeadlines(conn exonet.Conn) { - if dl, ok := conn.(deadliner); ok { - dl.SetReadDeadline(time.Time{}) - dl.SetWriteDeadline(time.Time{}) - } -} - -type connState uint8 - -const ( - stateIdle connState = iota - stateReserved connState = iota - statePiped connState = iota -) - -// binderConn is a connection pre-opened by a binder node to the gateway, -// sitting idle until a connector claims it. -type binderConn struct { - exonet.Conn - state connState - pipedTo *connectorConn - onClose func() - signalCh chan chan error // connector requests keepalive to send ByteSignalGo - dead chan struct{} // closed when the connection is no longer alive - closed atomic.Bool -} - -func (bc *binderConn) IsIdle() bool { return bc.state == stateIdle } -func (bc *binderConn) IsReserved() bool { return bc.state == stateReserved } -func (bc *binderConn) IsPiped() bool { return bc.state == statePiped } - -func (bc *binderConn) Close() error { - err := bc.Conn.Close() - if !bc.closed.Swap(true) && bc.onClose != nil { - bc.onClose() - } - return err -} - -// keepalive reads binder pings and responds with pong. -// When a connector signals via signalCh it sends ByteSignalGo and waits for ByteSignalReady. -func (bc *binderConn) keepalive() { - defer close(bc.dead) - // piped tracks whether pipe() has taken ownership of bc. - // If true, pipe() is responsible for closing — keepalive must not. - piped := false - defer func() { - if !piped { - bc.Close() - } - }() - - for { - withReadDeadline(bc.Conn, socketDeadTimeout) - var b [1]byte - if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { - return - } - clearDeadlines(bc.Conn) - - switch b[0] { - case gateway.BytePing: - select { - case respCh := <-bc.signalCh: - piped = true // pipe or probeBinderConn takes ownership; don't close on exit - bc.sendSignal(respCh) - return - default: - if err := gateway.WritePong(bc.Conn); err != nil { - return - } - } - default: - return - } - } -} - -// sendSignal sends ByteSignalGo to the binder and waits for ByteSignalReady. -// Called from keepalive when a connector signals via signalCh. -func (bc *binderConn) sendSignal(respCh chan error) { - withWriteDeadline(bc.Conn, socketProbeTimeout) - if err := gateway.WriteSignalGo(bc.Conn); err != nil { - respCh <- err - return - } - clearDeadlines(bc.Conn) - withReadDeadline(bc.Conn, socketProbeTimeout) - var b [1]byte - if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { - respCh <- err - return - } - clearDeadlines(bc.Conn) - if b[0] != gateway.ByteSignalReady { - respCh <- fmt.Errorf("expected signalReady, got 0x%02x", b[0]) - return - } - respCh <- nil -} - -// signal queues a ByteSignalGo request via the keepalive loop and waits for the result. -// Returns true if the conn is alive and was signaled successfully. -// Closes bc on failure unless keepalive already exited. -func (bc *binderConn) signal() bool { - respCh := make(chan error, 1) - select { - case bc.signalCh <- respCh: - case <-time.After(socketProbeTimeout): - // signalCh buffer full — another connector is already signaling this conn - bc.Close() - return false - } - - select { - case err := <-respCh: - if err != nil { - bc.Close() - } - return err == nil - case <-bc.dead: - // keepalive exited — conn was dead - return false - case <-time.After(socketProbeTimeout + socketPingInterval): - bc.Close() - return false - } -} - // binder represents a node registered as reachable through the gateway. // Only one binder registration per identity is allowed. type binder struct { @@ -168,41 +17,28 @@ type binder struct { Identity *astral.Identity Nonce astral.Nonce Visibility gateway.Visibility - conns sig.Set[*binderConn] + conns sig.Set[*bindingConn] } -func (b *binder) addConn(conn exonet.Conn) *binderConn { - bc := &binderConn{ - Conn: conn, - state: stateIdle, - signalCh: make(chan chan error, 1), - dead: make(chan struct{}), - } +func (b *binder) addConn(conn exonet.Conn) *bindingConn { + bc := newGatewayConn(conn, nil) bc.onClose = func() { b.conns.Remove(bc) } b.conns.Add(bc) return bc } -// takeConn reserves an idle binderConn for a connector. -func (b *binder) takeConn() (*binderConn, bool) { +// takeConn reserves an idle bindingConn for a connector via atomic CAS. +func (b *binder) takeConn() (*bindingConn, bool) { b.mu.Lock() defer b.mu.Unlock() for _, bc := range b.conns.Clone() { - if bc.IsIdle() { - bc.state = stateReserved + if bc.active.CompareAndSwap(false, true) { return bc, true } } return nil, false } -func (b *binder) markPiped(bc *binderConn, cc *connectorConn) { - b.mu.Lock() - defer b.mu.Unlock() - bc.state = statePiped - bc.pipedTo = cc -} - func (b *binder) Close() error { b.mu.Lock() defer b.mu.Unlock() diff --git a/mod/gateway/src/binding_conn.go b/mod/gateway/src/binding_conn.go new file mode 100644 index 000000000..201390d77 --- /dev/null +++ b/mod/gateway/src/binding_conn.go @@ -0,0 +1,182 @@ +package gateway + +import ( + "fmt" + "io" + "sync/atomic" + "time" + + "github.com/cryptopunkscc/astrald/mod/exonet" + "github.com/cryptopunkscc/astrald/mod/gateway" +) + +// deadliner is implemented by connections that support read/write deadlines (e.g. net.Conn). +type deadliner interface { + SetReadDeadline(time.Time) error + SetWriteDeadline(time.Time) error +} + +// bindingConn is a unified idle socket connection for both binder and gateway sides +type bindingConn struct { + exonet.Conn + closed atomic.Bool + active atomic.Bool // set on idle→active; guards idle counters against double-decrement + dead chan struct{} // closed when keepalive exits + signalCh chan chan error // non-nil → gateway (responder) mode + onClose func() +} + +func newGatewayConn(conn exonet.Conn, onClose func()) *bindingConn { + return &bindingConn{ + Conn: conn, + dead: make(chan struct{}), + signalCh: make(chan chan error, 1), + onClose: onClose, + } +} + +func newBinderConn(conn exonet.Conn, onClose func()) *bindingConn { + return &bindingConn{ + Conn: conn, + dead: make(chan struct{}), + onClose: onClose, + } +} + +func (bc *bindingConn) Close() error { + err := bc.Conn.Close() + if !bc.closed.Swap(true) && bc.onClose != nil { + bc.onClose() + } + return err +} + +// keepalive runs the ping/pong loop until activation or connection loss. +func (bc *bindingConn) keepalive(done <-chan struct{}, onActive func(), onActivate func() error) { + defer close(bc.dead) + activated := false + defer func() { + if !activated { + bc.Close() + } + }() + + binder := bc.signalCh == nil + dl, _ := bc.Conn.(deadliner) + + for { + if binder { + if err := gateway.WritePing(bc.Conn); err != nil { + return + } + } + + timeout := socketDeadTimeout + if binder { + timeout = socketPingTimeout + } + if dl != nil { + dl.SetReadDeadline(time.Now().Add(timeout)) + } + var b [1]byte + _, err := io.ReadFull(bc.Conn, b[:]) + if dl != nil { + dl.SetReadDeadline(time.Time{}) + } + if err != nil { + return + } + + switch b[0] { + case gateway.BytePing: // gateway only + select { + case respCh := <-bc.signalCh: + bc.sendSignalGo(respCh) + activated = true + return + default: + if err := gateway.WritePong(bc.Conn); err != nil { + return + } + } + case gateway.BytePong: // binder only + select { + case <-time.After(socketPingInterval): + case <-done: + return + } + case gateway.ByteSignalGo: // binder only + // Set active before onActive so that if Close fires during WriteSignalReady, + // onClose sees active=true and does not double-decrement the idle counter. + bc.active.Store(true) + if onActive != nil { + onActive() + } + if err := gateway.WriteSignalReady(bc.Conn); err != nil { + bc.Close() + activated = true // active is set; defer must not double-close + return + } + if onActivate != nil { + if err := onActivate(); err != nil { + bc.Close() + } + } + activated = true + return + default: + return + } + } +} + +// sendSignalGo sends ByteSignalGo and waits for ByteSignalReady, reporting the result on respCh. +func (bc *bindingConn) sendSignalGo(respCh chan error) { + dl, _ := bc.Conn.(deadliner) + if dl != nil { + defer dl.SetReadDeadline(time.Time{}) + defer dl.SetWriteDeadline(time.Time{}) + dl.SetWriteDeadline(time.Now().Add(socketProbeTimeout)) + } + if err := gateway.WriteSignalGo(bc.Conn); err != nil { + respCh <- err + return + } + if dl != nil { + dl.SetReadDeadline(time.Now().Add(socketProbeTimeout)) + } + var b [1]byte + if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { + respCh <- err + return + } + if b[0] != gateway.ByteSignalReady { + respCh <- fmt.Errorf("expected signalReady, got 0x%02x", b[0]) + return + } + respCh <- nil +} + +// signal queues a ByteSignalGo via the keepalive loop and waits for acknowledgement. +// Returns true if the binder acknowledged. Gateway mode only. +func (bc *bindingConn) signal() bool { + respCh := make(chan error, 1) + select { + case bc.signalCh <- respCh: + case <-time.After(socketProbeTimeout): + bc.Close() + return false + } + select { + case err := <-respCh: + if err != nil { + bc.Close() + } + return err == nil + case <-bc.dead: + return false + case <-time.After(socketProbeTimeout + socketPingInterval): + bc.Close() + return false + } +} diff --git a/mod/gateway/src/conn.go b/mod/gateway/src/conn.go index 2a7b0e886..15f1d1b31 100644 --- a/mod/gateway/src/conn.go +++ b/mod/gateway/src/conn.go @@ -8,8 +8,18 @@ import ( var _ exonet.Conn = (*gwConn)(nil) -type gwConn struct { +// routeConn adapts an io.ReadWriteCloser (e.g. astral.Conn from query.Route) to exonet.Conn. +// Endpoint methods return nil — gwConn overrides all of them. +type routeConn struct { io.ReadWriteCloser +} + +func (c *routeConn) LocalEndpoint() exonet.Endpoint { return nil } +func (c *routeConn) RemoteEndpoint() exonet.Endpoint { return nil } +func (c *routeConn) Outbound() bool { return true } + +type gwConn struct { + *bindingConn local exonet.Endpoint remote exonet.Endpoint outbound bool diff --git a/mod/gateway/src/connector.go b/mod/gateway/src/connector.go index 0458bbd2d..2b1789f34 100644 --- a/mod/gateway/src/connector.go +++ b/mod/gateway/src/connector.go @@ -4,17 +4,8 @@ import ( "sync" "github.com/cryptopunkscc/astrald/astral" - "github.com/cryptopunkscc/astrald/mod/exonet" ) -// connectorConn is the connection opened by a connector node to the gateway, -// to be piped to a reserved binderConn. -type connectorConn struct { - exonet.Conn - network string - pipedTo *binderConn -} - // connector represents a pending connection request from a node that wants // to reach a binder through the gateway. Multiple connectors per identity // are allowed. @@ -23,12 +14,12 @@ type connector struct { Identity *astral.Identity Nonce astral.Nonce Target *astral.Identity - reserved *binderConn + reserved *bindingConn } -// takeReserved atomically takes the reserved binderConn, returning nil if +// takeReserved atomically takes the reserved bindingConn, returning nil if // already taken (connection already established or timed out). -func (c *connector) takeReserved() *binderConn { +func (c *connector) takeReserved() *bindingConn { c.mu.Lock() defer c.mu.Unlock() diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go index be3f98f02..a5d6f7686 100644 --- a/mod/gateway/src/dial.go +++ b/mod/gateway/src/dial.go @@ -44,10 +44,10 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C } return &gwConn{ - ReadWriteCloser: conn, - local: conn.LocalEndpoint(), - remote: gwEndpoint, - outbound: conn.Outbound(), + bindingConn: newBinderConn(conn, nil), + local: conn.LocalEndpoint(), + remote: gwEndpoint, + outbound: conn.Outbound(), }, nil } @@ -67,9 +67,9 @@ func (mod *Module) route(ctx *astral.Context, gwEndpoint *gateway.Endpoint) (exo } return &gwConn{ - ReadWriteCloser: conn, - local: gateway.NewEndpoint(mod.node.Identity(), mod.node.Identity()), - remote: gwEndpoint, - outbound: true, + bindingConn: newBinderConn(&routeConn{ReadWriteCloser: conn}, nil), + local: gateway.NewEndpoint(mod.node.Identity(), mod.node.Identity()), + remote: gwEndpoint, + outbound: true, }, nil } diff --git a/mod/gateway/src/errors.go b/mod/gateway/src/errors.go index 5b0b9af38..1d805751b 100644 --- a/mod/gateway/src/errors.go +++ b/mod/gateway/src/errors.go @@ -1,13 +1,6 @@ package gateway -import ( - "errors" - "fmt" -) - -var ErrInvalidGateway = errors.New("invalid gateway") -var ErrAlreadySubscribed = errors.New("already subscribed") -var ErrNotSubscribed = errors.New("subscription not found") +import "fmt" type ErrParseError struct { msg string diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index 0b6a35ec5..64943ace2 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -1,9 +1,7 @@ package gateway import ( - "io" "sync" - "sync/atomic" "time" "github.com/cryptopunkscc/astrald/astral" @@ -19,17 +17,17 @@ const ( socketPingTimeout = 3 * time.Second ) +// SocketPool maintains socketPoolTargetIdle idle socket connections to a gateway. type SocketPool struct { *Module ctx *astral.Context socket gateway.Socket gatewayID *astral.Identity - mu sync.Mutex - total int - idle int + mu sync.Mutex + idle int - signal chan struct{} + wake chan struct{} } func (mod *Module) newSocketPool(ctx *astral.Context, gatewayID *astral.Identity, socket gateway.Socket) *SocketPool { @@ -38,7 +36,7 @@ func (mod *Module) newSocketPool(ctx *astral.Context, gatewayID *astral.Identity Module: mod, socket: socket, gatewayID: gatewayID, - signal: make(chan struct{}, 1), + wake: make(chan struct{}, 1), } } @@ -50,8 +48,15 @@ func (p *SocketPool) Run() error { select { case <-p.ctx.Done(): return p.ctx.Err() - case <-p.signal: - for p.idleCount() < socketPoolTargetIdle { + case <-p.wake: + for { + p.mu.Lock() + if p.idle >= socketPoolTargetIdle { + p.mu.Unlock() + break + } + p.mu.Unlock() + conn, err := p.acquireConn() if err != nil { select { @@ -66,7 +71,7 @@ func (p *SocketPool) Run() error { } retry.Reset() - p.handoff(conn) + p.startIdleSocket(conn) } } } @@ -87,16 +92,9 @@ func (p *SocketPool) acquireConn() (exonet.Conn, error) { return conn, nil } -func (p *SocketPool) idleCount() int { - p.mu.Lock() - defer p.mu.Unlock() - return p.idle -} - func (p *SocketPool) addIdle() { p.mu.Lock() p.idle++ - p.total++ p.mu.Unlock() } @@ -107,103 +105,32 @@ func (p *SocketPool) onConnTaken() { p.notify() } -func (p *SocketPool) onConnClosed(wasIdle bool) { - p.mu.Lock() - p.total-- - if wasIdle { +func (p *SocketPool) onConnClosed(idle bool) { + if idle { + p.mu.Lock() p.idle-- + p.mu.Unlock() } - p.mu.Unlock() p.notify() } -func (p *SocketPool) handoff(conn exonet.Conn) { - pc := &socketConn{ - Conn: conn, - localEndpoint: gateway.NewEndpoint(p.node.Identity(), p.node.Identity()), - remoteEndpoint: gateway.NewEndpoint(p.gatewayID, p.node.Identity()), +func (p *SocketPool) startIdleSocket(conn exonet.Conn) { + bc := newBinderConn(conn, nil) + gc := &gwConn{ + bindingConn: bc, + local: gateway.NewEndpoint(p.node.Identity(), p.node.Identity()), + remote: gateway.NewEndpoint(p.gatewayID, p.node.Identity()), } - p.registerConn(pc) - go p.runIdleConn(conn, pc) -} - -// registerConn binds pool lifecycle callbacks to pc and marks it as idle. -func (p *SocketPool) registerConn(pc *socketConn) { - pc.onClose = func() { p.onConnClosed(!pc.used.Load()) } + bc.onClose = func() { p.onConnClosed(!bc.active.Load()) } p.addIdle() -} - -// runIdleConn is the binder-side ping loop for an idle socket connection. -// It sends BytePing and waits for BytePong or ByteSignalGo. -// On ByteSignalGo the conn transitions from idle to taken before writing ByteSignalReady. -func (p *SocketPool) runIdleConn(conn exonet.Conn, pc *socketConn) { - for { - if err := gateway.WritePing(conn); err != nil { - pc.Close() - return - } - withReadDeadline(conn, socketPingTimeout) - var b [1]byte - if _, err := io.ReadFull(conn, b[:]); err != nil { - pc.Close() - return - } - clearDeadlines(conn) - switch b[0] { - case gateway.BytePong: - select { - case <-time.After(socketPingInterval): - case <-p.ctx.Done(): - pc.Close() - return - } - case gateway.ByteSignalGo: - // Mark taken before notifying pool: if Close fires during write, - // onClose sees used=true → wasIdle=false → only total is decremented. - pc.used.Store(true) - p.onConnTaken() - if err := gateway.WriteSignalReady(pc); err != nil { - pc.Close() - return - } - if err := p.Nodes.EstablishInboundLink(p.ctx, pc); err != nil { - pc.Close() - } - return - default: - pc.Close() - return - } - } + go bc.keepalive(p.ctx.Done(), p.onConnTaken, func() error { + return p.Nodes.EstablishInboundLink(p.ctx, gc) + }) } func (p *SocketPool) notify() { select { - case p.signal <- struct{}{}: + case p.wake <- struct{}{}: default: } } - -// socketConn wraps an exonet.Conn with fixed endpoints and a one-shot close callback. -// used tracks whether the conn has been taken from the idle pool (for wasIdle accounting). -type socketConn struct { - exonet.Conn - - localEndpoint exonet.Endpoint - remoteEndpoint exonet.Endpoint - onClose func() - - closed atomic.Bool - used atomic.Bool -} - -func (c *socketConn) LocalEndpoint() exonet.Endpoint { return c.localEndpoint } -func (c *socketConn) RemoteEndpoint() exonet.Endpoint { return c.remoteEndpoint } - -func (c *socketConn) Close() error { - err := c.Conn.Close() - if !c.closed.Swap(true) && c.onClose != nil { - c.onClose() - } - return err -} diff --git a/mod/gateway/src/route.go b/mod/gateway/src/route.go index 8a7bdda87..44e7186bb 100644 --- a/mod/gateway/src/route.go +++ b/mod/gateway/src/route.go @@ -28,10 +28,10 @@ func (mod *Module) routeQuery(ctx *astral.Context, q *astral.Query, w io.WriteCl if targetKey == mod.node.Identity().String() { return query.Accept(q, w, func(conn astral.Conn) { c := &gwConn{ - ReadWriteCloser: conn, - local: gateway.NewEndpoint(q.Target, q.Target), - remote: gateway.NewEndpoint(q.Caller, q.Target), - outbound: false, + bindingConn: newBinderConn(&routeConn{ReadWriteCloser: conn}, nil), + local: gateway.NewEndpoint(q.Target, q.Target), + remote: gateway.NewEndpoint(q.Caller, q.Target), + outbound: false, } // prevents slow gateway connections From 37ce59a6472d04ec1e3a1153ecd21e03948f7e49 Mon Sep 17 00:00:00 2001 From: Rekseto Date: Sun, 15 Mar 2026 23:57:38 +0000 Subject: [PATCH 42/42] mod/gateway: WIP --- mod/gateway/src/accept.go | 2 +- mod/gateway/src/binder.go | 202 +++++++++++++++++++-- mod/gateway/src/binding_conn.go | 182 ------------------- mod/gateway/src/conn.go | 32 +++- mod/gateway/src/dial.go | 16 +- mod/gateway/src/errors.go | 14 -- mod/gateway/{ => src/frames}/ping_frame.go | 2 +- mod/gateway/src/parser.go | 31 +--- mod/gateway/src/pool.go | 63 ++----- mod/gateway/src/route.go | 7 +- 10 files changed, 249 insertions(+), 302 deletions(-) delete mode 100644 mod/gateway/src/binding_conn.go delete mode 100644 mod/gateway/src/errors.go rename mod/gateway/{ => src/frames}/ping_frame.go (97%) diff --git a/mod/gateway/src/accept.go b/mod/gateway/src/accept.go index 8ba51c63b..74814fa6a 100644 --- a/mod/gateway/src/accept.go +++ b/mod/gateway/src/accept.go @@ -29,7 +29,7 @@ func (mod *Module) acceptSocketConn(_ context.Context, conn exonet.Conn) (stopLi if b, ok := mod.binderByNonce(nonce); ok { mod.log.Infov(2, "added idle conn to binder %v", b.Identity) bc := b.addConn(conn) - go bc.keepalive(nil, nil, nil) + go bc.keepalive(nil, nil) return stopListener, nil } diff --git a/mod/gateway/src/binder.go b/mod/gateway/src/binder.go index 48cd8e0af..3ddba2896 100644 --- a/mod/gateway/src/binder.go +++ b/mod/gateway/src/binder.go @@ -1,19 +1,21 @@ package gateway import ( - "errors" - "sync" + "fmt" + "io" + "sync/atomic" + "time" "github.com/cryptopunkscc/astrald/astral" "github.com/cryptopunkscc/astrald/mod/exonet" "github.com/cryptopunkscc/astrald/mod/gateway" + "github.com/cryptopunkscc/astrald/mod/gateway/src/frames" "github.com/cryptopunkscc/astrald/sig" ) // binder represents a node registered as reachable through the gateway. // Only one binder registration per identity is allowed. type binder struct { - mu sync.Mutex Identity *astral.Identity Nonce astral.Nonce Visibility gateway.Visibility @@ -21,16 +23,15 @@ type binder struct { } func (b *binder) addConn(conn exonet.Conn) *bindingConn { - bc := newGatewayConn(conn, nil) - bc.onClose = func() { b.conns.Remove(bc) } + var bc *bindingConn + bc = newGatewayConn(conn, func() { b.conns.Remove(bc) }) b.conns.Add(bc) return bc } // takeConn reserves an idle bindingConn for a connector via atomic CAS. +// CAS on active is sufficient — no mutex needed. func (b *binder) takeConn() (*bindingConn, bool) { - b.mu.Lock() - defer b.mu.Unlock() for _, bc := range b.conns.Clone() { if bc.active.CompareAndSwap(false, true) { return bc, true @@ -40,11 +41,188 @@ func (b *binder) takeConn() (*bindingConn, bool) { } func (b *binder) Close() error { - b.mu.Lock() - defer b.mu.Unlock() - var errs []error for _, bc := range b.conns.Clone() { - errs = append(errs, bc.Close()) + bc.Close() + } + return nil +} + +type connRole uint8 + +const ( + roleBinder connRole = iota + roleGateway +) + +// bindingConn is a unified idle socket connection for both binder and gateway sides +type bindingConn struct { + exonet.Conn + role connRole + + closed atomic.Bool + active atomic.Bool // set on idle→active; guards idle counters against double-decrement + + dead chan struct{} + signalCh chan chan error + onClose func() +} + +func newGatewayConn(conn exonet.Conn, onClose func()) *bindingConn { + return &bindingConn{ + Conn: conn, + role: roleGateway, + dead: make(chan struct{}), + signalCh: make(chan chan error, 1), + onClose: onClose, + } +} + +func newBinderConn(conn exonet.Conn) *bindingConn { + return &bindingConn{ + Conn: conn, + role: roleBinder, + dead: make(chan struct{}), + } +} + +func (bc *bindingConn) SetReadDeadline(t time.Time) error { + if dl, ok := bc.Conn.(deadliner); ok { + return dl.SetReadDeadline(t) + } + return nil +} + +func (bc *bindingConn) SetWriteDeadline(t time.Time) error { + if dl, ok := bc.Conn.(deadliner); ok { + return dl.SetWriteDeadline(t) + } + return nil +} + +// readFrame reads a single control byte within the gateway keepalive/control phase. +// Must not be called after activation, when the connection becomes a raw stream. +func (bc *bindingConn) readFrame(timeout time.Duration) (byte, error) { + bc.SetReadDeadline(time.Now().Add(timeout)) + var b [1]byte + _, err := io.ReadFull(bc.Conn, b[:]) + bc.SetReadDeadline(time.Time{}) + return b[0], err +} + +func (bc *bindingConn) Close() error { + err := bc.Conn.Close() + if !bc.closed.Swap(true) && bc.onClose != nil { + bc.onClose() + } + return err +} + +// keepalive runs the ping/pong loop until activation or connection loss. +// done stops the binder-side ping sleep on shutdown (pass ctx.Done()). +// onActivate is called after WriteSignalReady; returned error causes bc to be closed. +func (bc *bindingConn) keepalive(done <-chan struct{}, onActivate func() error) { + defer close(bc.dead) + activated := false + defer func() { + if !activated { + bc.Close() + } + }() + + for { + if bc.role == roleBinder { + if err := frames.WritePing(bc.Conn); err != nil { + return + } + } + + timeout := socketDeadTimeout + if bc.role == roleBinder { + timeout = socketPingTimeout + } + frame, err := bc.readFrame(timeout) + if err != nil { + return + } + + switch frame { + case frames.BytePing: // roleGateway only + select { + case respCh := <-bc.signalCh: + bc.sendSignalGo(respCh) + activated = true + return + default: + if err := frames.WritePong(bc.Conn); err != nil { + return + } + } + case frames.BytePong: // roleBinder only + select { + case <-time.After(socketPingInterval): + case <-done: + return + } + case frames.ByteSignalGo: // roleBinder only + bc.active.Store(true) + if err := frames.WriteSignalReady(bc.Conn); err != nil { + bc.Close() + activated = true // active is set; defer must not double-close + return + } + if onActivate != nil { + if err := onActivate(); err != nil { + bc.Close() + } + } + activated = true + return + default: + return + } + } +} + +// sendSignalGo sends ByteSignalGo and waits for ByteSignalReady, reporting the result on respCh. +func (bc *bindingConn) sendSignalGo(respCh chan error) { + defer bc.SetWriteDeadline(time.Time{}) + bc.SetWriteDeadline(time.Now().Add(socketProbeTimeout)) + + if err := frames.WriteSignalGo(bc.Conn); err != nil { + respCh <- err + return + } + + frame, err := bc.readFrame(socketProbeTimeout) + if err != nil { + respCh <- err + return + } + if frame != frames.ByteSignalReady { + respCh <- fmt.Errorf("expected signalReady, got 0x%02x", frame) + return + } + respCh <- nil +} + +func (bc *bindingConn) signal() bool { + respCh := make(chan error, 1) + select { + case bc.signalCh <- respCh: + case <-time.After(socketProbeTimeout): + bc.Close() + return false + } + select { + case err := <-respCh: + if err != nil { + bc.Close() + } + return err == nil + case <-bc.dead: + return false + case <-time.After(socketProbeTimeout + socketPingInterval): + bc.Close() + return false } - return errors.Join(errs...) } diff --git a/mod/gateway/src/binding_conn.go b/mod/gateway/src/binding_conn.go deleted file mode 100644 index 201390d77..000000000 --- a/mod/gateway/src/binding_conn.go +++ /dev/null @@ -1,182 +0,0 @@ -package gateway - -import ( - "fmt" - "io" - "sync/atomic" - "time" - - "github.com/cryptopunkscc/astrald/mod/exonet" - "github.com/cryptopunkscc/astrald/mod/gateway" -) - -// deadliner is implemented by connections that support read/write deadlines (e.g. net.Conn). -type deadliner interface { - SetReadDeadline(time.Time) error - SetWriteDeadline(time.Time) error -} - -// bindingConn is a unified idle socket connection for both binder and gateway sides -type bindingConn struct { - exonet.Conn - closed atomic.Bool - active atomic.Bool // set on idle→active; guards idle counters against double-decrement - dead chan struct{} // closed when keepalive exits - signalCh chan chan error // non-nil → gateway (responder) mode - onClose func() -} - -func newGatewayConn(conn exonet.Conn, onClose func()) *bindingConn { - return &bindingConn{ - Conn: conn, - dead: make(chan struct{}), - signalCh: make(chan chan error, 1), - onClose: onClose, - } -} - -func newBinderConn(conn exonet.Conn, onClose func()) *bindingConn { - return &bindingConn{ - Conn: conn, - dead: make(chan struct{}), - onClose: onClose, - } -} - -func (bc *bindingConn) Close() error { - err := bc.Conn.Close() - if !bc.closed.Swap(true) && bc.onClose != nil { - bc.onClose() - } - return err -} - -// keepalive runs the ping/pong loop until activation or connection loss. -func (bc *bindingConn) keepalive(done <-chan struct{}, onActive func(), onActivate func() error) { - defer close(bc.dead) - activated := false - defer func() { - if !activated { - bc.Close() - } - }() - - binder := bc.signalCh == nil - dl, _ := bc.Conn.(deadliner) - - for { - if binder { - if err := gateway.WritePing(bc.Conn); err != nil { - return - } - } - - timeout := socketDeadTimeout - if binder { - timeout = socketPingTimeout - } - if dl != nil { - dl.SetReadDeadline(time.Now().Add(timeout)) - } - var b [1]byte - _, err := io.ReadFull(bc.Conn, b[:]) - if dl != nil { - dl.SetReadDeadline(time.Time{}) - } - if err != nil { - return - } - - switch b[0] { - case gateway.BytePing: // gateway only - select { - case respCh := <-bc.signalCh: - bc.sendSignalGo(respCh) - activated = true - return - default: - if err := gateway.WritePong(bc.Conn); err != nil { - return - } - } - case gateway.BytePong: // binder only - select { - case <-time.After(socketPingInterval): - case <-done: - return - } - case gateway.ByteSignalGo: // binder only - // Set active before onActive so that if Close fires during WriteSignalReady, - // onClose sees active=true and does not double-decrement the idle counter. - bc.active.Store(true) - if onActive != nil { - onActive() - } - if err := gateway.WriteSignalReady(bc.Conn); err != nil { - bc.Close() - activated = true // active is set; defer must not double-close - return - } - if onActivate != nil { - if err := onActivate(); err != nil { - bc.Close() - } - } - activated = true - return - default: - return - } - } -} - -// sendSignalGo sends ByteSignalGo and waits for ByteSignalReady, reporting the result on respCh. -func (bc *bindingConn) sendSignalGo(respCh chan error) { - dl, _ := bc.Conn.(deadliner) - if dl != nil { - defer dl.SetReadDeadline(time.Time{}) - defer dl.SetWriteDeadline(time.Time{}) - dl.SetWriteDeadline(time.Now().Add(socketProbeTimeout)) - } - if err := gateway.WriteSignalGo(bc.Conn); err != nil { - respCh <- err - return - } - if dl != nil { - dl.SetReadDeadline(time.Now().Add(socketProbeTimeout)) - } - var b [1]byte - if _, err := io.ReadFull(bc.Conn, b[:]); err != nil { - respCh <- err - return - } - if b[0] != gateway.ByteSignalReady { - respCh <- fmt.Errorf("expected signalReady, got 0x%02x", b[0]) - return - } - respCh <- nil -} - -// signal queues a ByteSignalGo via the keepalive loop and waits for acknowledgement. -// Returns true if the binder acknowledged. Gateway mode only. -func (bc *bindingConn) signal() bool { - respCh := make(chan error, 1) - select { - case bc.signalCh <- respCh: - case <-time.After(socketProbeTimeout): - bc.Close() - return false - } - select { - case err := <-respCh: - if err != nil { - bc.Close() - } - return err == nil - case <-bc.dead: - return false - case <-time.After(socketProbeTimeout + socketPingInterval): - bc.Close() - return false - } -} diff --git a/mod/gateway/src/conn.go b/mod/gateway/src/conn.go index 15f1d1b31..73159f729 100644 --- a/mod/gateway/src/conn.go +++ b/mod/gateway/src/conn.go @@ -2,24 +2,22 @@ package gateway import ( "io" + "time" "github.com/cryptopunkscc/astrald/mod/exonet" ) var _ exonet.Conn = (*gwConn)(nil) -// routeConn adapts an io.ReadWriteCloser (e.g. astral.Conn from query.Route) to exonet.Conn. -// Endpoint methods return nil — gwConn overrides all of them. -type routeConn struct { - io.ReadWriteCloser +// note: maybe can be part of exonet +type deadliner interface { + SetReadDeadline(time.Time) error + SetWriteDeadline(time.Time) error } -func (c *routeConn) LocalEndpoint() exonet.Endpoint { return nil } -func (c *routeConn) RemoteEndpoint() exonet.Endpoint { return nil } -func (c *routeConn) Outbound() bool { return true } - +// gwConn wraps any io.ReadWriteCloser with gateway endpoint metadata. type gwConn struct { - *bindingConn + io.ReadWriteCloser local exonet.Endpoint remote exonet.Endpoint outbound bool @@ -27,4 +25,18 @@ type gwConn struct { func (c *gwConn) LocalEndpoint() exonet.Endpoint { return c.local } func (c *gwConn) RemoteEndpoint() exonet.Endpoint { return c.remote } -func (c gwConn) Outbound() bool { return c.outbound } +func (c *gwConn) Outbound() bool { return c.outbound } + +func (c *gwConn) SetReadDeadline(t time.Time) error { + if dl, ok := c.ReadWriteCloser.(deadliner); ok { + return dl.SetReadDeadline(t) + } + return nil +} + +func (c *gwConn) SetWriteDeadline(t time.Time) error { + if dl, ok := c.ReadWriteCloser.(deadliner); ok { + return dl.SetWriteDeadline(t) + } + return nil +} diff --git a/mod/gateway/src/dial.go b/mod/gateway/src/dial.go index a5d6f7686..be3f98f02 100644 --- a/mod/gateway/src/dial.go +++ b/mod/gateway/src/dial.go @@ -44,10 +44,10 @@ func (mod *Module) Dial(ctx *astral.Context, endpoint exonet.Endpoint) (exonet.C } return &gwConn{ - bindingConn: newBinderConn(conn, nil), - local: conn.LocalEndpoint(), - remote: gwEndpoint, - outbound: conn.Outbound(), + ReadWriteCloser: conn, + local: conn.LocalEndpoint(), + remote: gwEndpoint, + outbound: conn.Outbound(), }, nil } @@ -67,9 +67,9 @@ func (mod *Module) route(ctx *astral.Context, gwEndpoint *gateway.Endpoint) (exo } return &gwConn{ - bindingConn: newBinderConn(&routeConn{ReadWriteCloser: conn}, nil), - local: gateway.NewEndpoint(mod.node.Identity(), mod.node.Identity()), - remote: gwEndpoint, - outbound: true, + ReadWriteCloser: conn, + local: gateway.NewEndpoint(mod.node.Identity(), mod.node.Identity()), + remote: gwEndpoint, + outbound: true, }, nil } diff --git a/mod/gateway/src/errors.go b/mod/gateway/src/errors.go deleted file mode 100644 index 1d805751b..000000000 --- a/mod/gateway/src/errors.go +++ /dev/null @@ -1,14 +0,0 @@ -package gateway - -import "fmt" - -type ErrParseError struct { - msg string -} - -func (e ErrParseError) Error() string { - if len(e.msg) == 0 { - return "parse error" - } - return fmt.Sprintf("parse error: %s", e.msg) -} diff --git a/mod/gateway/ping_frame.go b/mod/gateway/src/frames/ping_frame.go similarity index 97% rename from mod/gateway/ping_frame.go rename to mod/gateway/src/frames/ping_frame.go index aeecaaeb9..35946d082 100644 --- a/mod/gateway/ping_frame.go +++ b/mod/gateway/src/frames/ping_frame.go @@ -1,4 +1,4 @@ -package gateway +package frames import "io" diff --git a/mod/gateway/src/parser.go b/mod/gateway/src/parser.go index 4db0c8b71..1d91cc889 100644 --- a/mod/gateway/src/parser.go +++ b/mod/gateway/src/parser.go @@ -2,10 +2,11 @@ package gateway import ( "errors" - "github.com/cryptopunkscc/astrald/astral" + "fmt" + "strings" + "github.com/cryptopunkscc/astrald/mod/exonet" "github.com/cryptopunkscc/astrald/mod/gateway" - "strings" ) var _ exonet.Parser = &Module{} @@ -17,7 +18,7 @@ func (mod *Module) Parse(network string, address string) (exonet.Endpoint, error var ids = strings.SplitN(address, ":", 2) if len(ids) != 2 { - return nil, ErrParseError{msg: "invalid address string"} + return nil, fmt.Errorf("invalid endpoint: %s", address) } var err error @@ -38,27 +39,3 @@ func (mod *Module) Parse(network string, address string) (exonet.Endpoint, error return &endpoint, nil } - -// Parse converts a text representation of a gateway address to an Endpoint struct -func Parse(str string) (endpoint *gateway.Endpoint, err error) { - if len(str) != (2*66)+1 { // two public key hex strings and a separator ":" - return endpoint, ErrParseError{msg: "invalid address length"} - } - var ids = strings.SplitN(str, ":", 2) - if len(ids) != 2 { - return nil, ErrParseError{msg: "invalid address string"} - } - endpoint.GatewayID, err = astral.ParseIdentity(ids[0]) - if err != nil { - return nil, err - } - endpoint.TargetID, err = astral.ParseIdentity(ids[1]) - if err != nil { - return nil, err - } - if endpoint.GatewayID.IsEqual(endpoint.TargetID) { - return nil, errors.New("invalid endpoint") - } - - return -} diff --git a/mod/gateway/src/pool.go b/mod/gateway/src/pool.go index 64943ace2..404f6f041 100644 --- a/mod/gateway/src/pool.go +++ b/mod/gateway/src/pool.go @@ -1,7 +1,6 @@ package gateway import ( - "sync" "time" "github.com/cryptopunkscc/astrald/astral" @@ -24,10 +23,8 @@ type SocketPool struct { socket gateway.Socket gatewayID *astral.Identity - mu sync.Mutex - idle int - - wake chan struct{} + conns sig.Set[*bindingConn] + wake chan struct{} } func (mod *Module) newSocketPool(ctx *astral.Context, gatewayID *astral.Identity, socket gateway.Socket) *SocketPool { @@ -49,14 +46,7 @@ func (p *SocketPool) Run() error { case <-p.ctx.Done(): return p.ctx.Err() case <-p.wake: - for { - p.mu.Lock() - if p.idle >= socketPoolTargetIdle { - p.mu.Unlock() - break - } - p.mu.Unlock() - + for p.idleCount() < socketPoolTargetIdle { conn, err := p.acquireConn() if err != nil { select { @@ -69,7 +59,6 @@ func (p *SocketPool) Run() error { } continue } - retry.Reset() p.startIdleSocket(conn) } @@ -92,39 +81,27 @@ func (p *SocketPool) acquireConn() (exonet.Conn, error) { return conn, nil } -func (p *SocketPool) addIdle() { - p.mu.Lock() - p.idle++ - p.mu.Unlock() -} - -func (p *SocketPool) onConnTaken() { - p.mu.Lock() - p.idle-- - p.mu.Unlock() - p.notify() -} - -func (p *SocketPool) onConnClosed(idle bool) { - if idle { - p.mu.Lock() - p.idle-- - p.mu.Unlock() - } - p.notify() +// idleCount returns the number of non-active conns in the pool. +func (p *SocketPool) idleCount() int { + return len(p.conns.Select(func(a *bindingConn) bool { + return !a.active.Load() + })) } func (p *SocketPool) startIdleSocket(conn exonet.Conn) { - bc := newBinderConn(conn, nil) - gc := &gwConn{ - bindingConn: bc, - local: gateway.NewEndpoint(p.node.Identity(), p.node.Identity()), - remote: gateway.NewEndpoint(p.gatewayID, p.node.Identity()), + bc := newBinderConn(conn) + bc.onClose = func() { + p.conns.Remove(bc) + p.notify() } - bc.onClose = func() { p.onConnClosed(!bc.active.Load()) } - p.addIdle() - go bc.keepalive(p.ctx.Done(), p.onConnTaken, func() error { - return p.Nodes.EstablishInboundLink(p.ctx, gc) + + p.conns.Add(bc) + go bc.keepalive(p.ctx.Done(), func() error { + return p.Nodes.EstablishInboundLink(p.ctx, &gwConn{ + ReadWriteCloser: bc, + local: gateway.NewEndpoint(p.node.Identity(), p.node.Identity()), + remote: gateway.NewEndpoint(p.gatewayID, p.node.Identity()), + }) }) } diff --git a/mod/gateway/src/route.go b/mod/gateway/src/route.go index 44e7186bb..8ca8db1ee 100644 --- a/mod/gateway/src/route.go +++ b/mod/gateway/src/route.go @@ -28,10 +28,9 @@ func (mod *Module) routeQuery(ctx *astral.Context, q *astral.Query, w io.WriteCl if targetKey == mod.node.Identity().String() { return query.Accept(q, w, func(conn astral.Conn) { c := &gwConn{ - bindingConn: newBinderConn(&routeConn{ReadWriteCloser: conn}, nil), - local: gateway.NewEndpoint(q.Target, q.Target), - remote: gateway.NewEndpoint(q.Caller, q.Target), - outbound: false, + ReadWriteCloser: conn, + local: gateway.NewEndpoint(q.Target, q.Target), + remote: gateway.NewEndpoint(q.Caller, q.Target), } // prevents slow gateway connections