From 2381693fc6566e4c78c71e0799d750a41cae5007 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Tue, 7 Nov 2023 12:54:36 +0100 Subject: [PATCH 01/25] iterate over database --- go.mod | 2 +- go.sum | 2 + sql/atxs/atxs.go | 100 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 26e0e59d2c..f79f748ecb 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/prometheus/common v0.45.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.0 + github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3 github.com/spacemeshos/economics v0.1.1 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index b9c20e7d58..08e206d110 100644 --- a/go.sum +++ b/go.sum @@ -643,6 +643,8 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemeshos/api/release/go v1.24.0 h1:uB4ZdmbHpodtpTyb7tDGxkTyb0nqr6fOIKv47SUChwk= github.com/spacemeshos/api/release/go v1.24.0/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= +github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3 h1:SUjpIqNkAUIHTjUEq53/PYPMZANiwPJiwdzihdxHORU= +github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/economics v0.1.1 h1:BPgMoTaeQ05ME6wEA1+MvXMp+wvXr51bIuN23thrCAk= github.com/spacemeshos/economics v0.1.1/go.mod h1:76nTjugYRiQ5/eD/DQs2dXPPilp28URMswUKncfdanY= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index fc024e98cd..bf1ab674dd 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + spacemeshv1 "github.com/spacemeshos/api/release/go/spacemesh/v1" "github.com/spacemeshos/go-spacemesh/codec" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" @@ -469,3 +470,102 @@ func IterateAtxs(db sql.Executor, from, to types.EpochID, fn func(*types.Verifie } return derr } + +func IterateAtxGRPC( + db sql.Executor, + filter *spacemeshv1.ActivationStreamRequest, + fn func(*spacemeshv1.ActivationStreamResponse) bool, +) error { + query := queryFrom(filter) + bindings, err := bindingsFrom(filter) + if err != nil { + return err + } + var derr error + _, err = db.Exec(query, bindings, + decoder(func(atx *types.VerifiedActivationTx, err error) bool { + if atx != nil { + v1 := &spacemeshv1.ActivationV1{ + Id: atx.ID().Bytes(), + NodeId: atx.SmesherID.Bytes(), + Signature: atx.Signature.Bytes(), + PublishEpoch: atx.PublishEpoch.Uint32(), + Sequence: atx.Sequence, + PrevAtx: atx.PrevATXID[:], + PositioningAtx: atx.PositioningATX[:], + Coinbase: atx.Coinbase.String(), + Units: atx.NumUnits, + BaseTick: uint32(atx.BaseTickHeight()), + Ticks: uint32(atx.TickCount()), + } + return fn(&spacemeshv1.ActivationStreamResponse{V1: v1}) + } + derr = err + return derr == nil + })) + if err != nil { + return err + } + return derr +} + +// queryFrom and bindingsFrom should decode fields in the same order. + +func queryFrom(filter *spacemeshv1.ActivationStreamRequest) string { + query := fullQuery + if filter != nil { + return query + } + i := 1 + if filter.Epochs != nil { + query += fmt.Sprintf(" where epoch between ?%d and ?%d", i, i+1) + i += 2 + } + if filter.Id != nil { + query += fmt.Sprintf(" and id = ?%d", i) + i++ + } + if filter.NodeId != nil { + query += fmt.Sprintf(" and pubkey = ?%d", i) + i++ + } + if filter.Coinbase != nil { + query += fmt.Sprintf(" and coinbase = ?%d", i) + i++ + } + return query +} + +func bindingsFrom(filter *spacemeshv1.ActivationStreamRequest) (sql.Encoder, error) { + if filter == nil { + return nil, nil + } + var coinbase *types.Address + if filter.Coinbase != nil { + address, err := types.StringToAddress(filter.Coinbase.Coinbase) + if err != nil { + return nil, fmt.Errorf("invalid coinbase address: %w", err) + } + coinbase = &address + } + i := 1 + return func(stmt *sql.Statement) { + if filter.Epochs != nil { + stmt.BindInt64(i, int64(filter.Epochs.StartEpoch)) + stmt.BindInt64(i+1, int64(filter.Epochs.EndEpoch)) + i += 2 + } + if filter.Id != nil { + stmt.BindBytes(i, filter.Id.Id) + i++ + } + if filter.NodeId != nil { + stmt.BindBytes(i, filter.NodeId.NodeId) + i++ + } + if coinbase != nil { + stmt.BindBytes(i, coinbase.Bytes()) + i++ + } + }, nil +} From bab8ac6c755499dda4adf940537d2300ad004b43 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Tue, 7 Nov 2023 13:38:06 +0100 Subject: [PATCH 02/25] add half baked streaming for atxs --- api/grpcserver/activation_service.go | 19 +++++++- api/grpcserver/activation_service_test.go | 14 +++--- go.mod | 2 +- go.sum | 2 + node/node.go | 1 + sql/atxs/atxs.go | 53 ++++++++++++++--------- sql/atxs/atxs_test.go | 11 +++++ 7 files changed, 72 insertions(+), 30 deletions(-) diff --git a/api/grpcserver/activation_service.go b/api/grpcserver/activation_service.go index 5591d2b6b4..19ec1418a9 100644 --- a/api/grpcserver/activation_service.go +++ b/api/grpcserver/activation_service.go @@ -17,15 +17,18 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/atxs" ) type activationService struct { goldenAtx types.ATXID + db *sql.Database atxProvider atxProvider } -func NewActivationService(atxProvider atxProvider, goldenAtx types.ATXID) *activationService { +func NewActivationService(db *sql.Database, atxProvider atxProvider, goldenAtx types.ATXID) *activationService { return &activationService{ + db: db, goldenAtx: goldenAtx, atxProvider: atxProvider, } @@ -99,3 +102,17 @@ func (s *activationService) Highest(ctx context.Context, req *emptypb.Empty) (*p Atx: convertActivation(atx), }, nil } + +func (s *activationService) Stream(filter *pb.ActivationStreamRequest, stream pb.ActivationService_StreamServer) error { + if filter.Watch { + return status.Error(codes.InvalidArgument, "watch is not supported") + } + var ierr error + if err := atxs.IterateAtxGRPC(s.db, filter, func(atx *pb.ActivationStreamResponse) bool { + ierr = stream.Send(atx) + return ierr == nil + }); err != nil { + return status.Error(codes.Internal, err.Error()) + } + return nil +} diff --git a/api/grpcserver/activation_service_test.go b/api/grpcserver/activation_service_test.go index c6664ead29..bc0c25711e 100644 --- a/api/grpcserver/activation_service_test.go +++ b/api/grpcserver/activation_service_test.go @@ -23,7 +23,7 @@ func Test_Highest_ReturnsGoldenAtxOnError(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) goldenAtx := types.ATXID{2, 3, 4} - activationService := grpcserver.NewActivationService(atxProvider, goldenAtx) + activationService := grpcserver.NewActivationService(nil, atxProvider, goldenAtx) atxProvider.EXPECT().MaxHeightAtx().Return(types.EmptyATXID, errors.New("blah")) response, err := activationService.Highest(context.Background(), &emptypb.Empty{}) @@ -41,7 +41,7 @@ func Test_Highest_ReturnsMaxTickHeight(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) goldenAtx := types.ATXID{2, 3, 4} - activationService := grpcserver.NewActivationService(atxProvider, goldenAtx) + activationService := grpcserver.NewActivationService(nil, atxProvider, goldenAtx) atx := types.VerifiedActivationTx{ ActivationTx: &types.ActivationTx{ @@ -76,7 +76,7 @@ func Test_Highest_ReturnsMaxTickHeight(t *testing.T) { func TestGet_RejectInvalidAtxID(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) _, err := activationService.Get(context.Background(), &pb.GetRequest{Id: []byte{1, 2, 3}}) require.Error(t, err) @@ -86,7 +86,7 @@ func TestGet_RejectInvalidAtxID(t *testing.T) { func TestGet_AtxNotPresent(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) id := types.RandomATXID() atxProvider.EXPECT().GetFullAtx(id).Return(nil, nil) @@ -99,7 +99,7 @@ func TestGet_AtxNotPresent(t *testing.T) { func TestGet_AtxProviderReturnsFailure(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) id := types.RandomATXID() atxProvider.EXPECT().GetFullAtx(id).Return(&types.VerifiedActivationTx{}, errors.New("")) @@ -112,7 +112,7 @@ func TestGet_AtxProviderReturnsFailure(t *testing.T) { func TestGet_HappyPath(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) id := types.RandomATXID() atx := types.VerifiedActivationTx{ @@ -149,7 +149,7 @@ func TestGet_HappyPath(t *testing.T) { func TestGet_IdentityCanceled(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) smesher, proof := grpcserver.BallotMalfeasance(t, sql.InMemory()) id := types.RandomATXID() diff --git a/go.mod b/go.mod index f79f748ecb..9b0453e929 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/prometheus/common v0.45.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3 + github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b github.com/spacemeshos/economics v0.1.1 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 08e206d110..1fb3376e6e 100644 --- a/go.sum +++ b/go.sum @@ -645,6 +645,8 @@ github.com/spacemeshos/api/release/go v1.24.0 h1:uB4ZdmbHpodtpTyb7tDGxkTyb0nqr6f github.com/spacemeshos/api/release/go v1.24.0/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3 h1:SUjpIqNkAUIHTjUEq53/PYPMZANiwPJiwdzihdxHORU= github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= +github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b h1:zPRpcFS6YFhuZe8kXMsMAAKzS8/xWORlxZZ4VTirqyU= +github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/economics v0.1.1 h1:BPgMoTaeQ05ME6wEA1+MvXMp+wvXr51bIuN23thrCAk= github.com/spacemeshos/economics v0.1.1/go.mod h1:76nTjugYRiQ5/eD/DQs2dXPPilp28URMswUKncfdanY= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= diff --git a/node/node.go b/node/node.go index 35cea41b3e..14fcc10df9 100644 --- a/node/node.go +++ b/node/node.go @@ -1314,6 +1314,7 @@ func (app *App) initService( ), nil case grpcserver.Activation: return grpcserver.NewActivationService( + app.cachedDB.Database, app.cachedDB, types.ATXID(app.Config.Genesis.GoldenATX()), ), nil diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index bf1ab674dd..456fceea47 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -476,13 +476,18 @@ func IterateAtxGRPC( filter *spacemeshv1.ActivationStreamRequest, fn func(*spacemeshv1.ActivationStreamResponse) bool, ) error { + full := fullQuery query := queryFrom(filter) + if query != "" { + full += " where " + query + } + full += " order by epoch asc" bindings, err := bindingsFrom(filter) if err != nil { return err } var derr error - _, err = db.Exec(query, bindings, + _, err = db.Exec(full, bindings, decoder(func(atx *types.VerifiedActivationTx, err error) bool { if atx != nil { v1 := &spacemeshv1.ActivationV1{ @@ -512,26 +517,31 @@ func IterateAtxGRPC( // queryFrom and bindingsFrom should decode fields in the same order. func queryFrom(filter *spacemeshv1.ActivationStreamRequest) string { - query := fullQuery - if filter != nil { + query := "" + if filter == nil { return query } i := 1 - if filter.Epochs != nil { - query += fmt.Sprintf(" where epoch between ?%d and ?%d", i, i+1) + and := "" + if filter.StartEpoch != 0 || filter.EndEpoch != 0 { + query += fmt.Sprintf(" epoch between ?%d and ?%d", i, i+1) + and = "and" i += 2 } - if filter.Id != nil { - query += fmt.Sprintf(" and id = ?%d", i) + if len(filter.Id) != 0 { + query += fmt.Sprintf(" %s id = ?%d", and, i) i++ + and = "and" } - if filter.NodeId != nil { - query += fmt.Sprintf(" and pubkey = ?%d", i) + if len(filter.NodeId) != 0 { + query += fmt.Sprintf(" %s pubkey = ?%d", and, i) i++ + and = "and" } - if filter.Coinbase != nil { - query += fmt.Sprintf(" and coinbase = ?%d", i) + if len(filter.Coinbase) != 0 { + query += fmt.Sprintf(" %s coinbase = ?%d", and, i) i++ + and = "and" } return query } @@ -541,8 +551,8 @@ func bindingsFrom(filter *spacemeshv1.ActivationStreamRequest) (sql.Encoder, err return nil, nil } var coinbase *types.Address - if filter.Coinbase != nil { - address, err := types.StringToAddress(filter.Coinbase.Coinbase) + if len(filter.Coinbase) != 0 { + address, err := types.StringToAddress(filter.Coinbase) if err != nil { return nil, fmt.Errorf("invalid coinbase address: %w", err) } @@ -550,17 +560,18 @@ func bindingsFrom(filter *spacemeshv1.ActivationStreamRequest) (sql.Encoder, err } i := 1 return func(stmt *sql.Statement) { - if filter.Epochs != nil { - stmt.BindInt64(i, int64(filter.Epochs.StartEpoch)) - stmt.BindInt64(i+1, int64(filter.Epochs.EndEpoch)) - i += 2 + if filter.StartEpoch != 0 || filter.EndEpoch != 0 { + stmt.BindInt64(i, int64(filter.StartEpoch)) + i++ + stmt.BindInt64(i, int64(filter.EndEpoch)) + i++ } - if filter.Id != nil { - stmt.BindBytes(i, filter.Id.Id) + if len(filter.Id) != 0 { + stmt.BindBytes(i, filter.Id) i++ } - if filter.NodeId != nil { - stmt.BindBytes(i, filter.NodeId.NodeId) + if len(filter.NodeId) != 0 { + stmt.BindBytes(i, filter.NodeId) i++ } if coinbase != nil { diff --git a/sql/atxs/atxs_test.go b/sql/atxs/atxs_test.go index d4f08bc25d..7325008ad0 100644 --- a/sql/atxs/atxs_test.go +++ b/sql/atxs/atxs_test.go @@ -1,12 +1,14 @@ package atxs_test import ( + "fmt" "os" "testing" "time" "github.com/stretchr/testify/require" + spacemeshv1 "github.com/spacemeshos/api/release/go/spacemesh/v1" "github.com/spacemeshos/go-spacemesh/activation" "github.com/spacemeshos/go-spacemesh/codec" "github.com/spacemeshos/go-spacemesh/common/types" @@ -745,6 +747,15 @@ func TestLatest(t *testing.T) { latest, err := atxs.LatestEpoch(db) require.NoError(t, err) require.EqualValues(t, tc.expect, latest) + + require.NoError(t, atxs.IterateAtxGRPC( + db, + &spacemeshv1.ActivationStreamRequest{StartEpoch: 7, EndEpoch: 7}, + func(asr *spacemeshv1.ActivationStreamResponse) bool { + fmt.Println(asr) + return true + }, + )) }) } } From a617a050614d3a6ae40e88a34035753a30827b32 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 09:01:45 +0100 Subject: [PATCH 03/25] refactor atxs db api --- api/grpcserver/activation_service.go | 21 +++- go.mod | 2 +- go.sum | 2 + sql/atxs/atxs.go | 151 ++++++++++++--------------- sql/atxs/atxs_test.go | 11 -- 5 files changed, 87 insertions(+), 100 deletions(-) diff --git a/api/grpcserver/activation_service.go b/api/grpcserver/activation_service.go index 19ec1418a9..949ac86e79 100644 --- a/api/grpcserver/activation_service.go +++ b/api/grpcserver/activation_service.go @@ -108,11 +108,28 @@ func (s *activationService) Stream(filter *pb.ActivationStreamRequest, stream pb return status.Error(codes.InvalidArgument, "watch is not supported") } var ierr error - if err := atxs.IterateAtxGRPC(s.db, filter, func(atx *pb.ActivationStreamResponse) bool { - ierr = stream.Send(atx) + if err := atxs.IterateAtxsOps(s.db, toOperations(filter), func(atx *types.VerifiedActivationTx) bool { + v1 := &pb.ActivationV1{ + Id: atx.ID().Bytes(), + NodeId: atx.SmesherID.Bytes(), + Signature: atx.Signature.Bytes(), + PublishEpoch: atx.PublishEpoch.Uint32(), + Sequence: atx.Sequence, + PrevAtx: atx.PrevATXID[:], + PositioningAtx: atx.PositioningATX[:], + Coinbase: atx.Coinbase.String(), + Units: atx.NumUnits, + BaseTick: uint32(atx.BaseTickHeight()), + Ticks: uint32(atx.TickCount()), + } + ierr = stream.Send(&pb.ActivationStreamResponse{V1: v1}) return ierr == nil }); err != nil { return status.Error(codes.Internal, err.Error()) } return nil } + +func toOperations(filter *pb.ActivationStreamRequest) atxs.Operations { + return atxs.Operations{} +} diff --git a/go.mod b/go.mod index 9b0453e929..43234745f8 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/prometheus/common v0.45.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b + github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c github.com/spacemeshos/economics v0.1.1 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 1fb3376e6e..052816f82f 100644 --- a/go.sum +++ b/go.sum @@ -647,6 +647,8 @@ github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3 h1:S github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b h1:zPRpcFS6YFhuZe8kXMsMAAKzS8/xWORlxZZ4VTirqyU= github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= +github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c h1:9uA5Yh9J0xeHNvhv4ky1tOat7qAk0hJLPC12FYsFQNU= +github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/economics v0.1.1 h1:BPgMoTaeQ05ME6wEA1+MvXMp+wvXr51bIuN23thrCAk= github.com/spacemeshos/economics v0.1.1/go.mod h1:76nTjugYRiQ5/eD/DQs2dXPPilp28URMswUKncfdanY= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index 456fceea47..acbc24ac68 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -2,9 +2,9 @@ package atxs import ( "fmt" + "strconv" "time" - spacemeshv1 "github.com/spacemeshos/api/release/go/spacemesh/v1" "github.com/spacemeshos/go-spacemesh/codec" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" @@ -471,39 +471,53 @@ func IterateAtxs(db sql.Executor, from, to types.EpochID, fn func(*types.Verifie return derr } -func IterateAtxGRPC( +type token string + +const ( + Eq token = "=" + NotEq token = "!=" + Gt token = ">" + Gte token = ">=" + Lt token = "<" + Lte token = "<=" + And token = "and" + Where token = "where" +) + +type field string + +const ( + Epoch field = "epoch" + Smesher field = "pubkey" + Coinbase field = "coinbase" + Id field = "id" +) + +type Op struct { + Field field + Token token + // Value will be type casted to one the expected types. + // Operation will panic if it doesn't match any of expected. + Value any +} + +type Operations struct { + Filter []Op + Other []Op +} + +func IterateAtxsOps( db sql.Executor, - filter *spacemeshv1.ActivationStreamRequest, - fn func(*spacemeshv1.ActivationStreamResponse) bool, + operations Operations, + fn func(*types.VerifiedActivationTx) bool, ) error { - full := fullQuery - query := queryFrom(filter) - if query != "" { - full += " where " + query - } - full += " order by epoch asc" - bindings, err := bindingsFrom(filter) - if err != nil { - return err - } var derr error - _, err = db.Exec(full, bindings, + _, err := db.Exec( + fullQuery+filterFrom(operations.Filter)+" order by epoch asc, id", + bindingsFrom(operations.Filter), decoder(func(atx *types.VerifiedActivationTx, err error) bool { if atx != nil { - v1 := &spacemeshv1.ActivationV1{ - Id: atx.ID().Bytes(), - NodeId: atx.SmesherID.Bytes(), - Signature: atx.Signature.Bytes(), - PublishEpoch: atx.PublishEpoch.Uint32(), - Sequence: atx.Sequence, - PrevAtx: atx.PrevATXID[:], - PositioningAtx: atx.PositioningATX[:], - Coinbase: atx.Coinbase.String(), - Units: atx.NumUnits, - BaseTick: uint32(atx.BaseTickHeight()), - Ticks: uint32(atx.TickCount()), - } - return fn(&spacemeshv1.ActivationStreamResponse{V1: v1}) + return fn(atx) } derr = err return derr == nil @@ -514,69 +528,34 @@ func IterateAtxGRPC( return derr } -// queryFrom and bindingsFrom should decode fields in the same order. - -func queryFrom(filter *spacemeshv1.ActivationStreamRequest) string { - query := "" - if filter == nil { - return query - } - i := 1 - and := "" - if filter.StartEpoch != 0 || filter.EndEpoch != 0 { - query += fmt.Sprintf(" epoch between ?%d and ?%d", i, i+1) - and = "and" - i += 2 - } - if len(filter.Id) != 0 { - query += fmt.Sprintf(" %s id = ?%d", and, i) - i++ - and = "and" - } - if len(filter.NodeId) != 0 { - query += fmt.Sprintf(" %s pubkey = ?%d", and, i) - i++ - and = "and" +func filterFrom(filter []Op) string { + if len(filter) == 0 { + return "" } - if len(filter.Coinbase) != 0 { - query += fmt.Sprintf(" %s coinbase = ?%d", and, i) - i++ - and = "and" + query := "where " + for i, op := range filter { + if i != 0 { + query += " " + string(And) + " " + } + query += string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1) } return query } -func bindingsFrom(filter *spacemeshv1.ActivationStreamRequest) (sql.Encoder, error) { - if filter == nil { - return nil, nil +func bindingsFrom(filter []Op) sql.Encoder { + if len(filter) == 0 { + return nil } - var coinbase *types.Address - if len(filter.Coinbase) != 0 { - address, err := types.StringToAddress(filter.Coinbase) - if err != nil { - return nil, fmt.Errorf("invalid coinbase address: %w", err) - } - coinbase = &address - } - i := 1 return func(stmt *sql.Statement) { - if filter.StartEpoch != 0 || filter.EndEpoch != 0 { - stmt.BindInt64(i, int64(filter.StartEpoch)) - i++ - stmt.BindInt64(i, int64(filter.EndEpoch)) - i++ - } - if len(filter.Id) != 0 { - stmt.BindBytes(i, filter.Id) - i++ - } - if len(filter.NodeId) != 0 { - stmt.BindBytes(i, filter.NodeId) - i++ - } - if coinbase != nil { - stmt.BindBytes(i, coinbase.Bytes()) - i++ + for i, op := range filter { + switch value := op.Value.(type) { + case int64: + stmt.BindInt64(i+1, value) + case []byte: + stmt.BindBytes(i+1, value) + default: + panic(fmt.Sprintf("unexpected type %T", value)) + } } - }, nil + } } diff --git a/sql/atxs/atxs_test.go b/sql/atxs/atxs_test.go index 7325008ad0..d4f08bc25d 100644 --- a/sql/atxs/atxs_test.go +++ b/sql/atxs/atxs_test.go @@ -1,14 +1,12 @@ package atxs_test import ( - "fmt" "os" "testing" "time" "github.com/stretchr/testify/require" - spacemeshv1 "github.com/spacemeshos/api/release/go/spacemesh/v1" "github.com/spacemeshos/go-spacemesh/activation" "github.com/spacemeshos/go-spacemesh/codec" "github.com/spacemeshos/go-spacemesh/common/types" @@ -747,15 +745,6 @@ func TestLatest(t *testing.T) { latest, err := atxs.LatestEpoch(db) require.NoError(t, err) require.EqualValues(t, tc.expect, latest) - - require.NoError(t, atxs.IterateAtxGRPC( - db, - &spacemeshv1.ActivationStreamRequest{StartEpoch: 7, EndEpoch: 7}, - func(asr *spacemeshv1.ActivationStreamResponse) bool { - fmt.Println(asr) - return true - }, - )) }) } } From 4e9610dadecb9e046662a66990df5a56cadcc10b Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 09:51:15 +0100 Subject: [PATCH 04/25] save progress --- api/grpcserver/activation_service.go | 53 +++++++++++++++++++++++-- api/grpcserver/v2/activation.go | 58 ++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 api/grpcserver/v2/activation.go diff --git a/api/grpcserver/activation_service.go b/api/grpcserver/activation_service.go index 949ac86e79..5298113423 100644 --- a/api/grpcserver/activation_service.go +++ b/api/grpcserver/activation_service.go @@ -107,8 +107,12 @@ func (s *activationService) Stream(filter *pb.ActivationStreamRequest, stream pb if filter.Watch { return status.Error(codes.InvalidArgument, "watch is not supported") } + ops, err := toOperations(filter) + if err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } var ierr error - if err := atxs.IterateAtxsOps(s.db, toOperations(filter), func(atx *types.VerifiedActivationTx) bool { + if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { v1 := &pb.ActivationV1{ Id: atx.ID().Bytes(), NodeId: atx.SmesherID.Bytes(), @@ -130,6 +134,49 @@ func (s *activationService) Stream(filter *pb.ActivationStreamRequest, stream pb return nil } -func toOperations(filter *pb.ActivationStreamRequest) atxs.Operations { - return atxs.Operations{} +func toOperations(filter *pb.ActivationStreamRequest) (atxs.Operations, error) { + ops := atxs.Operations{} + if filter == nil { + return ops, nil + } + if filter.NodeId != nil { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Smesher, + Token: atxs.Eq, + Value: filter.NodeId, + }) + } + if filter.Id != nil { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Id, + Token: atxs.Eq, + Value: filter.Id, + }) + } + if len(filter.Coinbase) > 0 { + addr, err := types.StringToAddress(filter.Coinbase) + if err != nil { + return atxs.Operations{}, err + } + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Coinbase, + Token: atxs.Eq, + Value: addr.Bytes(), + }) + } + if filter.StartEpoch != 0 { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Epoch, + Token: atxs.Gte, + Value: int64(filter.StartEpoch), + }) + } + if filter.EndEpoch != 0 { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Epoch, + Token: atxs.Lte, + Value: int64(filter.EndEpoch), + }) + } + return atxs.Operations{}, nil } diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go new file mode 100644 index 0000000000..55b033b68e --- /dev/null +++ b/api/grpcserver/v2/activation.go @@ -0,0 +1,58 @@ +package v2 + +import ( + "context" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "google.golang.org/grpc" + + spacemeshv2 "github.com/spacemeshos/api/release/go/spacemesh/v2" + + "github.com/spacemeshos/go-spacemesh/api/grpcserver" + "github.com/spacemeshos/go-spacemesh/sql" +) + +func NewActivationStreamService(db *sql.Database) *ActivationStreamService { + return &ActivationStreamService{db: db} +} + +type ActivationStreamService struct { + db *sql.Database +} + +var _ grpcserver.ServiceAPI = (*ActivationStreamService)(nil) + +func (s *ActivationStreamService) RegisterService(server *grpc.Server) { + spacemeshv2.RegisterActivationStreamServiceServer(server, s) +} + +func (s *ActivationStreamService) RegisterHandlerService(mux *runtime.ServeMux) error { + return spacemeshv2.RegisterActivationStreamServiceHandlerServer(context.Background(), mux, s) +} + +func (s *ActivationStreamService) String() string { + return "ActivationStreamService" +} + +func NewActivationService(db *sql.Database) *ActivationService { + return &ActivationService{db: db} +} + +type ActivationService struct { + db *sql.Database +} + +var _ grpcserver.ServiceAPI = (*ActivationService)(nil) + +func (s *ActivationService) RegisterService(server *grpc.Server) { + spacemeshv2.RegisterActivationServiceServer(server, s) +} + +func (s *ActivationService) RegisterHandlerService(mux *runtime.ServeMux) error { + return spacemeshv2.RegisterActivationServiceHandlerServer(context.Background(), mux, s) +} + +// String returns the service name. +func (s *ActivationService) String() string { + return "ActivationService" +} From a767855e8818589765c7ed3e3646b5367680ada1 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 10:48:04 +0100 Subject: [PATCH 05/25] register v2 --- api/grpcserver/activation_service.go | 83 +-------- api/grpcserver/activation_service_test.go | 14 +- api/grpcserver/config.go | 4 +- api/grpcserver/v2/activation.go | 217 ++++++++++++++++++++++ go.mod | 2 +- go.sum | 2 + node/node.go | 6 +- 7 files changed, 235 insertions(+), 93 deletions(-) diff --git a/api/grpcserver/activation_service.go b/api/grpcserver/activation_service.go index 5298113423..5591d2b6b4 100644 --- a/api/grpcserver/activation_service.go +++ b/api/grpcserver/activation_service.go @@ -17,18 +17,15 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/sql" - "github.com/spacemeshos/go-spacemesh/sql/atxs" ) type activationService struct { goldenAtx types.ATXID - db *sql.Database atxProvider atxProvider } -func NewActivationService(db *sql.Database, atxProvider atxProvider, goldenAtx types.ATXID) *activationService { +func NewActivationService(atxProvider atxProvider, goldenAtx types.ATXID) *activationService { return &activationService{ - db: db, goldenAtx: goldenAtx, atxProvider: atxProvider, } @@ -102,81 +99,3 @@ func (s *activationService) Highest(ctx context.Context, req *emptypb.Empty) (*p Atx: convertActivation(atx), }, nil } - -func (s *activationService) Stream(filter *pb.ActivationStreamRequest, stream pb.ActivationService_StreamServer) error { - if filter.Watch { - return status.Error(codes.InvalidArgument, "watch is not supported") - } - ops, err := toOperations(filter) - if err != nil { - return status.Error(codes.InvalidArgument, err.Error()) - } - var ierr error - if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { - v1 := &pb.ActivationV1{ - Id: atx.ID().Bytes(), - NodeId: atx.SmesherID.Bytes(), - Signature: atx.Signature.Bytes(), - PublishEpoch: atx.PublishEpoch.Uint32(), - Sequence: atx.Sequence, - PrevAtx: atx.PrevATXID[:], - PositioningAtx: atx.PositioningATX[:], - Coinbase: atx.Coinbase.String(), - Units: atx.NumUnits, - BaseTick: uint32(atx.BaseTickHeight()), - Ticks: uint32(atx.TickCount()), - } - ierr = stream.Send(&pb.ActivationStreamResponse{V1: v1}) - return ierr == nil - }); err != nil { - return status.Error(codes.Internal, err.Error()) - } - return nil -} - -func toOperations(filter *pb.ActivationStreamRequest) (atxs.Operations, error) { - ops := atxs.Operations{} - if filter == nil { - return ops, nil - } - if filter.NodeId != nil { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Smesher, - Token: atxs.Eq, - Value: filter.NodeId, - }) - } - if filter.Id != nil { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Id, - Token: atxs.Eq, - Value: filter.Id, - }) - } - if len(filter.Coinbase) > 0 { - addr, err := types.StringToAddress(filter.Coinbase) - if err != nil { - return atxs.Operations{}, err - } - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Coinbase, - Token: atxs.Eq, - Value: addr.Bytes(), - }) - } - if filter.StartEpoch != 0 { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Epoch, - Token: atxs.Gte, - Value: int64(filter.StartEpoch), - }) - } - if filter.EndEpoch != 0 { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Epoch, - Token: atxs.Lte, - Value: int64(filter.EndEpoch), - }) - } - return atxs.Operations{}, nil -} diff --git a/api/grpcserver/activation_service_test.go b/api/grpcserver/activation_service_test.go index bc0c25711e..c6664ead29 100644 --- a/api/grpcserver/activation_service_test.go +++ b/api/grpcserver/activation_service_test.go @@ -23,7 +23,7 @@ func Test_Highest_ReturnsGoldenAtxOnError(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) goldenAtx := types.ATXID{2, 3, 4} - activationService := grpcserver.NewActivationService(nil, atxProvider, goldenAtx) + activationService := grpcserver.NewActivationService(atxProvider, goldenAtx) atxProvider.EXPECT().MaxHeightAtx().Return(types.EmptyATXID, errors.New("blah")) response, err := activationService.Highest(context.Background(), &emptypb.Empty{}) @@ -41,7 +41,7 @@ func Test_Highest_ReturnsMaxTickHeight(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) goldenAtx := types.ATXID{2, 3, 4} - activationService := grpcserver.NewActivationService(nil, atxProvider, goldenAtx) + activationService := grpcserver.NewActivationService(atxProvider, goldenAtx) atx := types.VerifiedActivationTx{ ActivationTx: &types.ActivationTx{ @@ -76,7 +76,7 @@ func Test_Highest_ReturnsMaxTickHeight(t *testing.T) { func TestGet_RejectInvalidAtxID(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) _, err := activationService.Get(context.Background(), &pb.GetRequest{Id: []byte{1, 2, 3}}) require.Error(t, err) @@ -86,7 +86,7 @@ func TestGet_RejectInvalidAtxID(t *testing.T) { func TestGet_AtxNotPresent(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) id := types.RandomATXID() atxProvider.EXPECT().GetFullAtx(id).Return(nil, nil) @@ -99,7 +99,7 @@ func TestGet_AtxNotPresent(t *testing.T) { func TestGet_AtxProviderReturnsFailure(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) id := types.RandomATXID() atxProvider.EXPECT().GetFullAtx(id).Return(&types.VerifiedActivationTx{}, errors.New("")) @@ -112,7 +112,7 @@ func TestGet_AtxProviderReturnsFailure(t *testing.T) { func TestGet_HappyPath(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) id := types.RandomATXID() atx := types.VerifiedActivationTx{ @@ -149,7 +149,7 @@ func TestGet_HappyPath(t *testing.T) { func TestGet_IdentityCanceled(t *testing.T) { ctrl := gomock.NewController(t) atxProvider := grpcserver.NewMockatxProvider(ctrl) - activationService := grpcserver.NewActivationService(nil, atxProvider, types.ATXID{1}) + activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1}) smesher, proof := grpcserver.BallotMalfeasance(t, sql.InMemory()) id := types.RandomATXID() diff --git a/api/grpcserver/config.go b/api/grpcserver/config.go index c08d63beff..488e78f611 100644 --- a/api/grpcserver/config.go +++ b/api/grpcserver/config.go @@ -39,9 +39,9 @@ const ( // DefaultConfig defines the default configuration options for api. func DefaultConfig() Config { return Config{ - PublicServices: []Service{Debug, GlobalState, Mesh, Transaction, Node, Activation}, + PublicServices: []Service{Debug, GlobalState, Mesh, Transaction, Node, Activation, "activation_v2"}, PublicListener: "0.0.0.0:9092", - PrivateServices: []Service{Admin, Smesher, Post}, + PrivateServices: []Service{Admin, Smesher, Post, "activation_stream_v2"}, PrivateListener: "127.0.0.1:9093", TLSServices: []Service{}, TLSListener: "0.0.0.0:9094", diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go index 55b033b68e..07faa63107 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2/activation.go @@ -5,11 +5,20 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" spacemeshv2 "github.com/spacemeshos/api/release/go/spacemesh/v2" "github.com/spacemeshos/go-spacemesh/api/grpcserver" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/atxs" +) + +const ( + Activation = "activation_v2" + ActivationStream = "activation_stream_v2" ) func NewActivationStreamService(db *sql.Database) *ActivationStreamService { @@ -34,6 +43,77 @@ func (s *ActivationStreamService) String() string { return "ActivationStreamService" } +func (s *ActivationStreamService) Stream( + request *spacemeshv2.ActivationStreamRequest, + stream spacemeshv2.ActivationStreamService_StreamServer, +) error { + if request.Watch { + return status.Error(codes.InvalidArgument, "watch is not supported") + } + ops, err := toOperations(request) + if err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } + var ierr error + if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { + ierr = stream.Send(&spacemeshv2.Activation{Versioned: &spacemeshv2.Activation_V1{V1: toAtx(atx)}}) + return ierr == nil + }); err != nil { + return status.Error(codes.Internal, err.Error()) + } + return nil +} + +func (s *ActivationStreamService) StreamHeaders( + request *spacemeshv2.ActivationStreamRequest, + stream spacemeshv2.ActivationStreamService_StreamHeadersServer, +) error { + if request.Watch { + return status.Error(codes.InvalidArgument, "watch is not supported") + } + ops, err := toOperations(request) + if err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } + var ierr error + if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { + ierr = stream.Send(&spacemeshv2.ActivationHeader{Versioned: &spacemeshv2.ActivationHeader_V1{ + V1: toHeader(atx)}}) + return ierr == nil + }); err != nil { + return status.Error(codes.Internal, err.Error()) + } + return nil +} + +func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { + return &spacemeshv2.ActivationV1{ + Id: atx.ID().Bytes(), + NodeId: atx.SmesherID.Bytes(), + Signature: atx.Signature.Bytes(), + PublishEpoch: atx.PublishEpoch.Uint32(), + Sequence: atx.Sequence, + PrevAtx: atx.PrevATXID[:], + PositioningAtx: atx.PositioningATX[:], + Coinbase: atx.Coinbase.String(), + Units: atx.NumUnits, + BaseHeight: uint32(atx.BaseTickHeight()), + Ticks: uint32(atx.TickCount()), + } +} + +func toHeader(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationHeaderV1 { + return &spacemeshv2.ActivationHeaderV1{ + Id: atx.ID().Bytes(), + NodeId: atx.SmesherID.Bytes(), + PublishEpoch: atx.PublishEpoch.Uint32(), + Coinbase: atx.Coinbase.String(), + Units: atx.NumUnits, + BaseHeight: uint32(atx.BaseTickHeight()), + Ticks: uint32(atx.TickCount()), + } +} + func NewActivationService(db *sql.Database) *ActivationService { return &ActivationService{db: db} } @@ -56,3 +136,140 @@ func (s *ActivationService) RegisterHandlerService(mux *runtime.ServeMux) error func (s *ActivationService) String() string { return "ActivationService" } + +func (s *ActivationService) List( + ctx context.Context, + request *spacemeshv2.ActivationRequest, +) (*spacemeshv2.ActivationList, error) { + ops, err := toOperations2(request) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + // every full atx is ~1KB. 100 atxs is ~100KB. + if request.Limit > 100 { + return nil, status.Error(codes.InvalidArgument, "limit is capped at 100") + } + rst := make([]*spacemeshv2.Activation, 0, request.Limit) + if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { + rst = append(rst, &spacemeshv2.Activation{Versioned: &spacemeshv2.Activation_V1{V1: toAtx(atx)}}) + return true + }); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return &spacemeshv2.ActivationList{Activations: rst}, nil +} + +func (s *ActivationService) ListHeaders( + ctx context.Context, + request *spacemeshv2.ActivationRequest, +) (*spacemeshv2.ActivationHeaderList, error) { + ops, err := toOperations2(request) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if request.Limit > 10000 { + return nil, status.Error(codes.InvalidArgument, "limit is capped at 10000") + } + rst := make([]*spacemeshv2.ActivationHeader, 0, request.Limit) + if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { + rst = append(rst, &spacemeshv2.ActivationHeader{Versioned: &spacemeshv2.ActivationHeader_V1{V1: toHeader(atx)}}) + return true + }); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return &spacemeshv2.ActivationHeaderList{Headers: rst}, nil +} + +func toOperations(filter *spacemeshv2.ActivationStreamRequest) (atxs.Operations, error) { + ops := atxs.Operations{} + if filter == nil { + return ops, nil + } + if filter.NodeId != nil { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Smesher, + Token: atxs.Eq, + Value: filter.NodeId, + }) + } + if filter.Id != nil { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Id, + Token: atxs.Eq, + Value: filter.Id, + }) + } + if len(filter.Coinbase) > 0 { + addr, err := types.StringToAddress(filter.Coinbase) + if err != nil { + return atxs.Operations{}, err + } + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Coinbase, + Token: atxs.Eq, + Value: addr.Bytes(), + }) + } + if filter.StartEpoch != 0 { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Epoch, + Token: atxs.Gte, + Value: int64(filter.StartEpoch), + }) + } + if filter.EndEpoch != 0 { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Epoch, + Token: atxs.Lte, + Value: int64(filter.EndEpoch), + }) + } + return atxs.Operations{}, nil +} + +func toOperations2(filter *spacemeshv2.ActivationRequest) (atxs.Operations, error) { + ops := atxs.Operations{} + if filter == nil { + return ops, nil + } + if filter.NodeId != nil { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Smesher, + Token: atxs.Eq, + Value: filter.NodeId, + }) + } + if filter.Id != nil { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Id, + Token: atxs.Eq, + Value: filter.Id, + }) + } + if len(filter.Coinbase) > 0 { + addr, err := types.StringToAddress(filter.Coinbase) + if err != nil { + return atxs.Operations{}, err + } + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Coinbase, + Token: atxs.Eq, + Value: addr.Bytes(), + }) + } + if filter.StartEpoch != 0 { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Epoch, + Token: atxs.Gte, + Value: int64(filter.StartEpoch), + }) + } + if filter.EndEpoch != 0 { + ops.Filter = append(ops.Filter, atxs.Op{ + Field: atxs.Epoch, + Token: atxs.Lte, + Value: int64(filter.EndEpoch), + }) + } + return atxs.Operations{}, nil +} diff --git a/go.mod b/go.mod index 43234745f8..366bc6360d 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/prometheus/common v0.45.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c + github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20 github.com/spacemeshos/economics v0.1.1 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 052816f82f..4bf7d156e4 100644 --- a/go.sum +++ b/go.sum @@ -649,6 +649,8 @@ github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b h1:z github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c h1:9uA5Yh9J0xeHNvhv4ky1tOat7qAk0hJLPC12FYsFQNU= github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= +github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20 h1:Ih/n+v1cprkGbcdKzQ4kyoSBE8jGjNSo6rZk1ifZHtI= +github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/economics v0.1.1 h1:BPgMoTaeQ05ME6wEA1+MvXMp+wvXr51bIuN23thrCAk= github.com/spacemeshos/economics v0.1.1/go.mod h1:76nTjugYRiQ5/eD/DQs2dXPPilp28URMswUKncfdanY= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= diff --git a/node/node.go b/node/node.go index 14fcc10df9..e665de6d03 100644 --- a/node/node.go +++ b/node/node.go @@ -31,6 +31,7 @@ import ( "github.com/spacemeshos/go-spacemesh/activation" "github.com/spacemeshos/go-spacemesh/api/grpcserver" + v2 "github.com/spacemeshos/go-spacemesh/api/grpcserver/v2" "github.com/spacemeshos/go-spacemesh/atxsdata" "github.com/spacemeshos/go-spacemesh/beacon" "github.com/spacemeshos/go-spacemesh/blocks" @@ -1314,10 +1315,13 @@ func (app *App) initService( ), nil case grpcserver.Activation: return grpcserver.NewActivationService( - app.cachedDB.Database, app.cachedDB, types.ATXID(app.Config.Genesis.GoldenATX()), ), nil + case v2.Activation: + return v2.NewActivationService(app.db), nil + case v2.ActivationStream: + return v2.NewActivationStreamService(app.db), nil } return nil, fmt.Errorf("unknown service %s", svc) } From d30636f7c3b214e7e585b973680f31ba05d612dc Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 12:03:23 +0100 Subject: [PATCH 06/25] support for offset and limit --- api/grpcserver/v2/activation.go | 77 ++++++++++++--------------------- sql/atxs/atxs.go | 26 ++++++----- 2 files changed, 42 insertions(+), 61 deletions(-) diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go index 07faa63107..6d9fc35846 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2/activation.go @@ -50,7 +50,7 @@ func (s *ActivationStreamService) Stream( if request.Watch { return status.Error(codes.InvalidArgument, "watch is not supported") } - ops, err := toOperations(request) + ops, err := toOperations(toRequest(request)) if err != nil { return status.Error(codes.InvalidArgument, err.Error()) } @@ -71,7 +71,7 @@ func (s *ActivationStreamService) StreamHeaders( if request.Watch { return status.Error(codes.InvalidArgument, "watch is not supported") } - ops, err := toOperations(request) + ops, err := toOperations(toRequest(request)) if err != nil { return status.Error(codes.InvalidArgument, err.Error()) } @@ -141,13 +141,15 @@ func (s *ActivationService) List( ctx context.Context, request *spacemeshv2.ActivationRequest, ) (*spacemeshv2.ActivationList, error) { - ops, err := toOperations2(request) + ops, err := toOperations(request) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } // every full atx is ~1KB. 100 atxs is ~100KB. if request.Limit > 100 { return nil, status.Error(codes.InvalidArgument, "limit is capped at 100") + } else if request.Limit == 0 { + return nil, status.Error(codes.InvalidArgument, "limit must be set to a value below 100") } rst := make([]*spacemeshv2.Activation, 0, request.Limit) if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { @@ -163,12 +165,14 @@ func (s *ActivationService) ListHeaders( ctx context.Context, request *spacemeshv2.ActivationRequest, ) (*spacemeshv2.ActivationHeaderList, error) { - ops, err := toOperations2(request) + ops, err := toOperations(request) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } if request.Limit > 10000 { return nil, status.Error(codes.InvalidArgument, "limit is capped at 10000") + } else if request.Limit == 0 { + return nil, status.Error(codes.InvalidArgument, "limit must be set to a value below 10000") } rst := make([]*spacemeshv2.ActivationHeader, 0, request.Limit) if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { @@ -180,7 +184,17 @@ func (s *ActivationService) ListHeaders( return &spacemeshv2.ActivationHeaderList{Headers: rst}, nil } -func toOperations(filter *spacemeshv2.ActivationStreamRequest) (atxs.Operations, error) { +func toRequest(filter *spacemeshv2.ActivationStreamRequest) *spacemeshv2.ActivationRequest { + return &spacemeshv2.ActivationRequest{ + NodeId: filter.NodeId, + Id: filter.Id, + Coinbase: filter.Coinbase, + StartEpoch: filter.StartEpoch, + EndEpoch: filter.EndEpoch, + } +} + +func toOperations(filter *spacemeshv2.ActivationRequest) (atxs.Operations, error) { ops := atxs.Operations{} if filter == nil { return ops, nil @@ -224,52 +238,17 @@ func toOperations(filter *spacemeshv2.ActivationStreamRequest) (atxs.Operations, Value: int64(filter.EndEpoch), }) } - return atxs.Operations{}, nil -} - -func toOperations2(filter *spacemeshv2.ActivationRequest) (atxs.Operations, error) { - ops := atxs.Operations{} - if filter == nil { - return ops, nil - } - if filter.NodeId != nil { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Smesher, - Token: atxs.Eq, - Value: filter.NodeId, - }) - } - if filter.Id != nil { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Id, - Token: atxs.Eq, - Value: filter.Id, - }) - } - if len(filter.Coinbase) > 0 { - addr, err := types.StringToAddress(filter.Coinbase) - if err != nil { - return atxs.Operations{}, err - } - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Coinbase, - Token: atxs.Eq, - Value: addr.Bytes(), - }) - } - if filter.StartEpoch != 0 { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Epoch, - Token: atxs.Gte, - Value: int64(filter.StartEpoch), + if filter.Offset != 0 { + ops.Other = append(ops.Other, atxs.Op{ + Field: atxs.Offset, + Value: int64(filter.Offset), }) } - if filter.EndEpoch != 0 { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Epoch, - Token: atxs.Lte, - Value: int64(filter.EndEpoch), + if filter.Limit != 0 { + ops.Other = append(ops.Other, atxs.Op{ + Field: atxs.Limit, + Value: int64(filter.Limit), }) } - return atxs.Operations{}, nil + return ops, nil } diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index acbc24ac68..b19a1b43b7 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -491,6 +491,8 @@ const ( Smesher field = "pubkey" Coinbase field = "coinbase" Id field = "id" + Offset field = "offset" + Limit field = "limit" ) type Op struct { @@ -513,8 +515,8 @@ func IterateAtxsOps( ) error { var derr error _, err := db.Exec( - fullQuery+filterFrom(operations.Filter)+" order by epoch asc, id", - bindingsFrom(operations.Filter), + fullQuery+filterFrom(operations)+" order by epoch asc, id", + bindingsFrom(operations), decoder(func(atx *types.VerifiedActivationTx, err error) bool { if atx != nil { return fn(atx) @@ -528,26 +530,26 @@ func IterateAtxsOps( return derr } -func filterFrom(filter []Op) string { - if len(filter) == 0 { - return "" +func filterFrom(operations Operations) string { + query := " " + if len(operations.Filter) > 0 { + query = "where " } - query := "where " - for i, op := range filter { + for i, op := range operations.Filter { if i != 0 { query += " " + string(And) + " " } query += string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1) } + for i, op := range operations.Other { + query += " " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1+len(operations.Filter)) + } return query } -func bindingsFrom(filter []Op) sql.Encoder { - if len(filter) == 0 { - return nil - } +func bindingsFrom(operations Operations) sql.Encoder { return func(stmt *sql.Statement) { - for i, op := range filter { + for i, op := range append(operations.Filter, operations.Other...) { switch value := op.Value.(type) { case int64: stmt.BindInt64(i+1, value) From 7ff7f6754b263fe47e74ad10a065bb18d2636f54 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 12:13:37 +0100 Subject: [PATCH 07/25] encode the rest of the fields --- api/grpcserver/v2/activation.go | 40 ++++++++++++++++++++++++++++++++- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go index 6d9fc35846..34462ffb21 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2/activation.go @@ -87,7 +87,7 @@ func (s *ActivationStreamService) StreamHeaders( } func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { - return &spacemeshv2.ActivationV1{ + v1 := &spacemeshv2.ActivationV1{ Id: atx.ID().Bytes(), NodeId: atx.SmesherID.Bytes(), Signature: atx.Signature.Bytes(), @@ -100,6 +100,44 @@ func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { BaseHeight: uint32(atx.BaseTickHeight()), Ticks: uint32(atx.TickCount()), } + if atx.CommitmentATX != nil { + v1.CommittmentAtx = atx.CommitmentATX.Bytes() + } + if atx.VRFNonce != nil { + v1.VrfPostIndex = &spacemeshv2.VRFPostIndex{ + Nonce: uint64(*atx.VRFNonce), + } + } + if atx.InitialPost != nil { + v1.InitialPost = &spacemeshv2.Post{ + Nonce: atx.InitialPost.Nonce, + Indices: atx.InitialPost.Indices, + Pow: atx.InitialPost.Pow, + } + } + if nipost := atx.NIPost; nipost != nil { + if nipost.Post != nil { + v1.Post = &spacemeshv2.Post{ + Nonce: nipost.Post.Nonce, + Indices: nipost.Post.Indices, + Pow: nipost.Post.Pow, + } + } + if nipost.PostMetadata != nil { + v1.PostMeta = &spacemeshv2.PostMeta{ + Challenge: nipost.PostMetadata.Challenge, + Labels: nipost.PostMetadata.LabelsPerUnit, + } + } + v1.PoetProof = &spacemeshv2.PoetProof{ + ProofNodes: make([][]byte, len(nipost.Membership.Nodes)), + Leaf: nipost.Membership.LeafIndex, + } + for i, node := range nipost.Membership.Nodes { + v1.PoetProof.ProofNodes[i] = node.Bytes() + } + } + return v1 } func toHeader(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationHeaderV1 { diff --git a/go.mod b/go.mod index 366bc6360d..6d3ec4b89e 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/prometheus/common v0.45.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20 + github.com/spacemeshos/api/release/go v1.24.1-0.20231109110859-b10d491252b4 github.com/spacemeshos/economics v0.1.1 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 4bf7d156e4..30500bb035 100644 --- a/go.sum +++ b/go.sum @@ -651,6 +651,8 @@ github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c h1:9 github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20 h1:Ih/n+v1cprkGbcdKzQ4kyoSBE8jGjNSo6rZk1ifZHtI= github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= +github.com/spacemeshos/api/release/go v1.24.1-0.20231109110859-b10d491252b4 h1:8AUxvXZSFWYYSMbRp0zuPUUygcQMDDWXgmj1Uc9VtP8= +github.com/spacemeshos/api/release/go v1.24.1-0.20231109110859-b10d491252b4/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= github.com/spacemeshos/economics v0.1.1 h1:BPgMoTaeQ05ME6wEA1+MvXMp+wvXr51bIuN23thrCAk= github.com/spacemeshos/economics v0.1.1/go.mod h1:76nTjugYRiQ5/eD/DQs2dXPPilp28URMswUKncfdanY= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= From f4abc11d2451d2c9806c9de4fe47644983d14122 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 12:26:16 +0100 Subject: [PATCH 08/25] allow to watch --- api/grpcserver/v2/activation.go | 73 +++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go index 34462ffb21..44172c5a07 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2/activation.go @@ -2,16 +2,20 @@ package v2 import ( "context" + "errors" + "io" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" spacemeshv2 "github.com/spacemeshos/api/release/go/spacemesh/v2" "github.com/spacemeshos/go-spacemesh/api/grpcserver" "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/atxs" ) @@ -47,8 +51,18 @@ func (s *ActivationStreamService) Stream( request *spacemeshv2.ActivationStreamRequest, stream spacemeshv2.ActivationStreamService_StreamServer, ) error { + // TODO(dshulyak) implement matcher based on filter + var sub *events.BufferedSubscription[events.ActivationTx] if request.Watch { - return status.Error(codes.InvalidArgument, "watch is not supported") + var err error + sub, err = events.Subscribe[events.ActivationTx]() + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + defer sub.Close() + if err := stream.SendHeader(metadata.MD{}); err != nil { + return status.Errorf(codes.Unavailable, "can't send header") + } } ops, err := toOperations(toRequest(request)) if err != nil { @@ -61,15 +75,47 @@ func (s *ActivationStreamService) Stream( }); err != nil { return status.Error(codes.Internal, err.Error()) } - return nil + if sub == nil { + return nil + } + for { + select { + case <-stream.Context().Done(): + return nil + case <-sub.Full(): + return status.Error(codes.Canceled, "buffer overflow") + case rst := <-sub.Out(): + if err := stream.Send(&spacemeshv2.Activation{ + Versioned: &spacemeshv2.Activation_V1{V1: toAtx(rst.VerifiedActivationTx)}}, + ); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return status.Error(codes.Internal, err.Error()) + } + } + } } func (s *ActivationStreamService) StreamHeaders( request *spacemeshv2.ActivationStreamRequest, stream spacemeshv2.ActivationStreamService_StreamHeadersServer, ) error { + // TODO(dshulyak) the code below is almost the same as code in Stream + // it can be refactored by implementing generic with toAtx/toHeader + + // TODO(dshulyak) implement matcher based on filter + var sub *events.BufferedSubscription[events.ActivationTx] if request.Watch { - return status.Error(codes.InvalidArgument, "watch is not supported") + var err error + sub, err = events.Subscribe[events.ActivationTx]() + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + defer sub.Close() + if err := stream.SendHeader(metadata.MD{}); err != nil { + return status.Errorf(codes.Unavailable, "can't send header") + } } ops, err := toOperations(toRequest(request)) if err != nil { @@ -83,7 +129,26 @@ func (s *ActivationStreamService) StreamHeaders( }); err != nil { return status.Error(codes.Internal, err.Error()) } - return nil + if sub == nil { + return nil + } + for { + select { + case <-stream.Context().Done(): + return nil + case <-sub.Full(): + return status.Error(codes.Canceled, "buffer overflow") + case rst := <-sub.Out(): + if err := stream.Send(&spacemeshv2.ActivationHeader{ + Versioned: &spacemeshv2.ActivationHeader_V1{V1: toHeader(rst.VerifiedActivationTx)}}, + ); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return status.Error(codes.Internal, err.Error()) + } + } + } } func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { From 0c4328218ad32cf56beb77da5f96dc3f85c4908d Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 12:29:01 +0100 Subject: [PATCH 09/25] track todo --- sql/atxs/atxs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index f74f18bae2..bdcdaaced1 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -471,6 +471,8 @@ func IterateAtxs(db sql.Executor, from, to types.EpochID, fn func(*types.Verifie return derr } +// TODO(dshulyak) extract code for query building into separate module + type token string const ( From 2da36346d11f7a772245bfc016edff38973f5aa0 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 12:38:09 +0100 Subject: [PATCH 10/25] fix where --- sql/atxs/atxs.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index bdcdaaced1..99bb6fa0e3 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -533,15 +533,16 @@ func IterateAtxsOps( } func filterFrom(operations Operations) string { - query := " " - if len(operations.Filter) > 0 { - query = "where " - } + // TODO(dshulyak) using string writer will be more efficient + query := "" for i, op := range operations.Filter { + if i == 0 { + query += " " + string(Where) + } if i != 0 { - query += " " + string(And) + " " + query += " " + string(And) } - query += string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1) + query += " " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1) } for i, op := range operations.Other { query += " " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1+len(operations.Filter)) From 82fd29ded5ab393840d0eb50bf3d5f7d6c2fb065 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 12:53:29 +0100 Subject: [PATCH 11/25] refactor offset / limit --- api/grpcserver/v2/activation.go | 4 ++-- sql/atxs/atxs.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go index 44172c5a07..7f20bdabba 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2/activation.go @@ -252,7 +252,7 @@ func (s *ActivationService) List( if request.Limit > 100 { return nil, status.Error(codes.InvalidArgument, "limit is capped at 100") } else if request.Limit == 0 { - return nil, status.Error(codes.InvalidArgument, "limit must be set to a value below 100") + return nil, status.Error(codes.InvalidArgument, "limit must be set to <= 100") } rst := make([]*spacemeshv2.Activation, 0, request.Limit) if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { @@ -275,7 +275,7 @@ func (s *ActivationService) ListHeaders( if request.Limit > 10000 { return nil, status.Error(codes.InvalidArgument, "limit is capped at 10000") } else if request.Limit == 0 { - return nil, status.Error(codes.InvalidArgument, "limit must be set to a value below 10000") + return nil, status.Error(codes.InvalidArgument, "limit must be set to <= 10000") } rst := make([]*spacemeshv2.ActivationHeader, 0, request.Limit) if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index 99bb6fa0e3..0ae4d6244a 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -544,15 +544,15 @@ func filterFrom(operations Operations) string { } query += " " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1) } - for i, op := range operations.Other { - query += " " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1+len(operations.Filter)) + for _, op := range operations.Other { + query += fmt.Sprintf(" %s %v", string(op.Field), op.Value) } return query } func bindingsFrom(operations Operations) sql.Encoder { return func(stmt *sql.Statement) { - for i, op := range append(operations.Filter, operations.Other...) { + for i, op := range operations.Filter { switch value := op.Value.(type) { case int64: stmt.BindInt64(i+1, value) From af95f33d74d94c6a111124ee046bebbfdfcc1d2b Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Nov 2023 13:06:19 +0100 Subject: [PATCH 12/25] debug --- api/grpcserver/v2/activation.go | 12 ++++++------ sql/atxs/atxs.go | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go index 7f20bdabba..dfb31aa40e 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2/activation.go @@ -341,17 +341,17 @@ func toOperations(filter *spacemeshv2.ActivationRequest) (atxs.Operations, error Value: int64(filter.EndEpoch), }) } - if filter.Offset != 0 { - ops.Other = append(ops.Other, atxs.Op{ - Field: atxs.Offset, - Value: int64(filter.Offset), - }) - } if filter.Limit != 0 { ops.Other = append(ops.Other, atxs.Op{ Field: atxs.Limit, Value: int64(filter.Limit), }) } + if filter.Offset != 0 { + ops.Other = append(ops.Other, atxs.Op{ + Field: atxs.Offset, + Value: int64(filter.Offset), + }) + } return ops, nil } diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index 0ae4d6244a..92b84ee8f8 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -517,7 +517,7 @@ func IterateAtxsOps( ) error { var derr error _, err := db.Exec( - fullQuery+filterFrom(operations)+" order by epoch asc, id", + fullQuery+filterFrom(operations), bindingsFrom(operations), decoder(func(atx *types.VerifiedActivationTx, err error) bool { if atx != nil { @@ -544,6 +544,7 @@ func filterFrom(operations Operations) string { } query += " " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1) } + query += " order by epoch asc, id" for _, op := range operations.Other { query += fmt.Sprintf(" %s %v", string(op.Field), op.Value) } From 13a10f9e2149d5b57f8821ad7a9aa7281e67d284 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 9 Feb 2024 14:05:31 +0100 Subject: [PATCH 13/25] Use string builder to build query --- go.mod | 2 +- go.sum | 14 ++------------ sql/atxs/atxs.go | 17 +++++++++-------- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index a0dd017215..e34db8ae1c 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/quic-go/quic-go v0.41.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20231109110859-b10d491252b4 + github.com/spacemeshos/api/release/go v1.24.1-0.20240208115250-3c4d8451b3a3 github.com/spacemeshos/economics v0.1.2 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 749cb4ebc4..8b4f4a3cc9 100644 --- a/go.sum +++ b/go.sum @@ -622,18 +622,8 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemeshos/api/release/go v1.28.0 h1:HmrNf0kV7U9o2rwqrmZ0tfPgQA8JQWY8EYGoiCB4bnI= -github.com/spacemeshos/api/release/go v1.28.0/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= -github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3 h1:SUjpIqNkAUIHTjUEq53/PYPMZANiwPJiwdzihdxHORU= -github.com/spacemeshos/api/release/go v1.24.1-0.20231107112546-a48bc0143bd3/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= -github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b h1:zPRpcFS6YFhuZe8kXMsMAAKzS8/xWORlxZZ4VTirqyU= -github.com/spacemeshos/api/release/go v1.24.1-0.20231107121856-9cea84c8888b/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= -github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c h1:9uA5Yh9J0xeHNvhv4ky1tOat7qAk0hJLPC12FYsFQNU= -github.com/spacemeshos/api/release/go v1.24.1-0.20231109072853-7ec08711115c/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= -github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20 h1:Ih/n+v1cprkGbcdKzQ4kyoSBE8jGjNSo6rZk1ifZHtI= -github.com/spacemeshos/api/release/go v1.24.1-0.20231109094211-d9b5d4ad4b20/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= -github.com/spacemeshos/api/release/go v1.24.1-0.20231109110859-b10d491252b4 h1:8AUxvXZSFWYYSMbRp0zuPUUygcQMDDWXgmj1Uc9VtP8= -github.com/spacemeshos/api/release/go v1.24.1-0.20231109110859-b10d491252b4/go.mod h1:SwqQxbhAF7tN3Qr34eVzczCB3KyTbkHH12U82eqfy6M= +github.com/spacemeshos/api/release/go v1.24.1-0.20240208115250-3c4d8451b3a3 h1:tQMR1nVnvZZvXK0m+yKy6RaT1oMp77FEXruhFjoqANg= +github.com/spacemeshos/api/release/go v1.24.1-0.20240208115250-3c4d8451b3a3/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= github.com/spacemeshos/economics v0.1.2 h1:kw8cE5SMa/7svHOGorCd2w8ef1y8iP0p47/2VDOK8Ns= github.com/spacemeshos/economics v0.1.2/go.mod h1:ngeWn5E/jy9dJP1MHyuk3ehF8NBMTYhchqVDhAHUUNk= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index 6dc0f89017..bce63f09ef 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -3,6 +3,7 @@ package atxs import ( "fmt" "strconv" + "strings" "time" "github.com/spacemeshos/go-spacemesh/codec" @@ -568,22 +569,22 @@ func IterateAtxsOps( } func filterFrom(operations Operations) string { - // TODO(dshulyak) using string writer will be more efficient - query := "" + var queryBuilder strings.Builder + for i, op := range operations.Filter { if i == 0 { - query += " " + string(Where) + queryBuilder.WriteString(" " + string(Where)) } if i != 0 { - query += " " + string(And) + queryBuilder.WriteString(" " + string(And)) } - query += " " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1) + queryBuilder.WriteString(" " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1)) } - query += " order by epoch asc, id" + queryBuilder.WriteString(" order by epoch asc, id") for _, op := range operations.Other { - query += fmt.Sprintf(" %s %v", string(op.Field), op.Value) + queryBuilder.WriteString(fmt.Sprintf(" %s %v", string(op.Field), op.Value)) } - return query + return queryBuilder.String() } func bindingsFrom(operations Operations) sql.Encoder { From 7edbb427ac7cdf8daa9a69fa1e816827177d52b8 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 9 Feb 2024 15:58:57 +0100 Subject: [PATCH 14/25] add matcher to stream func --- api/grpcserver/v2/activation.go | 48 +++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2/activation.go index dfb31aa40e..2f82a2b848 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2/activation.go @@ -3,6 +3,8 @@ package v2 import ( "context" "errors" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" "io" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" @@ -51,11 +53,53 @@ func (s *ActivationStreamService) Stream( request *spacemeshv2.ActivationStreamRequest, stream spacemeshv2.ActivationStreamService_StreamServer, ) error { - // TODO(dshulyak) implement matcher based on filter var sub *events.BufferedSubscription[events.ActivationTx] if request.Watch { var err error - sub, err = events.Subscribe[events.ActivationTx]() + sub, err = events.SubscribeMatched(func(t *events.ActivationTx) bool { + if len(request.NodeId) > 0 { + var nodeId types.NodeID + copy(nodeId[:], request.NodeId) + + if t.SmesherID != nodeId { + return false + } + } + + if len(request.Id) > 0 { + var atxId types.ATXID + copy(atxId[:], request.Id) + + if t.ID() != atxId { + return false + } + } + + if len(request.Coinbase) > 0 { + addr, err := types.StringToAddress(request.Coinbase) + if err != nil { + ctxzap.Error(stream.Context(), "unable to convert atx coinbase", zap.Error(err)) + return false + } + if t.Coinbase != addr { + return false + } + } + + if request.StartEpoch != 0 { + if t.PublishEpoch.Uint32() < request.StartEpoch { + return false + } + } + + if request.EndEpoch != 0 { + if t.PublishEpoch.Uint32() > request.EndEpoch { + return false + } + } + + return true + }) if err != nil { return status.Error(codes.Internal, err.Error()) } From e07f5fb5b55a168f96da4a578866be8eabb0a5c3 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Mon, 12 Feb 2024 11:58:10 +0100 Subject: [PATCH 15/25] Move to v2alpha1 and remove headers --- api/grpcserver/config.go | 4 +- api/grpcserver/{v2 => v2alpha1}/activation.go | 145 ++++-------------- go.mod | 2 +- go.sum | 4 +- node/node.go | 10 +- 5 files changed, 38 insertions(+), 127 deletions(-) rename api/grpcserver/{v2 => v2alpha1}/activation.go (59%) diff --git a/api/grpcserver/config.go b/api/grpcserver/config.go index dfbd28069b..43fd88c872 100644 --- a/api/grpcserver/config.go +++ b/api/grpcserver/config.go @@ -41,9 +41,9 @@ const ( // DefaultConfig defines the default configuration options for api. func DefaultConfig() Config { return Config{ - PublicServices: []Service{GlobalState, Mesh, Transaction, Node, Activation, "activation_v2"}, + PublicServices: []Service{GlobalState, Mesh, Transaction, Node, Activation, "activation_v2alpha1"}, PublicListener: "0.0.0.0:9092", - PrivateServices: []Service{Admin, Smesher, Debug, "activation_stream_v2"}, + PrivateServices: []Service{Admin, Smesher, Debug, "activation_stream_v2alpha1"}, PrivateListener: "127.0.0.1:9093", PostServices: []Service{Post}, PostListener: "127.0.0.1:9094", diff --git a/api/grpcserver/v2/activation.go b/api/grpcserver/v2alpha1/activation.go similarity index 59% rename from api/grpcserver/v2/activation.go rename to api/grpcserver/v2alpha1/activation.go index 2f82a2b848..4f1cd32e3d 100644 --- a/api/grpcserver/v2/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -1,4 +1,4 @@ -package v2 +package v2alpha1 import ( "context" @@ -13,7 +13,7 @@ import ( "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - spacemeshv2 "github.com/spacemeshos/api/release/go/spacemesh/v2" + spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" "github.com/spacemeshos/go-spacemesh/api/grpcserver" "github.com/spacemeshos/go-spacemesh/common/types" @@ -23,8 +23,8 @@ import ( ) const ( - Activation = "activation_v2" - ActivationStream = "activation_stream_v2" + Activation = "activation_v2alpha1" + ActivationStream = "activation_stream_v2alpha1" ) func NewActivationStreamService(db *sql.Database) *ActivationStreamService { @@ -38,11 +38,11 @@ type ActivationStreamService struct { var _ grpcserver.ServiceAPI = (*ActivationStreamService)(nil) func (s *ActivationStreamService) RegisterService(server *grpc.Server) { - spacemeshv2.RegisterActivationStreamServiceServer(server, s) + spacemeshv2alpha1.RegisterActivationStreamServiceServer(server, s) } func (s *ActivationStreamService) RegisterHandlerService(mux *runtime.ServeMux) error { - return spacemeshv2.RegisterActivationStreamServiceHandlerServer(context.Background(), mux, s) + return spacemeshv2alpha1.RegisterActivationStreamServiceHandlerServer(context.Background(), mux, s) } func (s *ActivationStreamService) String() string { @@ -50,8 +50,8 @@ func (s *ActivationStreamService) String() string { } func (s *ActivationStreamService) Stream( - request *spacemeshv2.ActivationStreamRequest, - stream spacemeshv2.ActivationStreamService_StreamServer, + request *spacemeshv2alpha1.ActivationStreamRequest, + stream spacemeshv2alpha1.ActivationStreamService_StreamServer, ) error { var sub *events.BufferedSubscription[events.ActivationTx] if request.Watch { @@ -114,7 +114,7 @@ func (s *ActivationStreamService) Stream( } var ierr error if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { - ierr = stream.Send(&spacemeshv2.Activation{Versioned: &spacemeshv2.Activation_V1{V1: toAtx(atx)}}) + ierr = stream.Send(&spacemeshv2alpha1.Activation{Versioned: &spacemeshv2alpha1.Activation_V1{V1: toAtx(atx)}}) return ierr == nil }); err != nil { return status.Error(codes.Internal, err.Error()) @@ -129,8 +129,8 @@ func (s *ActivationStreamService) Stream( case <-sub.Full(): return status.Error(codes.Canceled, "buffer overflow") case rst := <-sub.Out(): - if err := stream.Send(&spacemeshv2.Activation{ - Versioned: &spacemeshv2.Activation_V1{V1: toAtx(rst.VerifiedActivationTx)}}, + if err := stream.Send(&spacemeshv2alpha1.Activation{ + Versioned: &spacemeshv2alpha1.Activation_V1{V1: toAtx(rst.VerifiedActivationTx)}}, ); err != nil { if errors.Is(err, io.EOF) { return nil @@ -141,62 +141,8 @@ func (s *ActivationStreamService) Stream( } } -func (s *ActivationStreamService) StreamHeaders( - request *spacemeshv2.ActivationStreamRequest, - stream spacemeshv2.ActivationStreamService_StreamHeadersServer, -) error { - // TODO(dshulyak) the code below is almost the same as code in Stream - // it can be refactored by implementing generic with toAtx/toHeader - - // TODO(dshulyak) implement matcher based on filter - var sub *events.BufferedSubscription[events.ActivationTx] - if request.Watch { - var err error - sub, err = events.Subscribe[events.ActivationTx]() - if err != nil { - return status.Error(codes.Internal, err.Error()) - } - defer sub.Close() - if err := stream.SendHeader(metadata.MD{}); err != nil { - return status.Errorf(codes.Unavailable, "can't send header") - } - } - ops, err := toOperations(toRequest(request)) - if err != nil { - return status.Error(codes.InvalidArgument, err.Error()) - } - var ierr error - if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { - ierr = stream.Send(&spacemeshv2.ActivationHeader{Versioned: &spacemeshv2.ActivationHeader_V1{ - V1: toHeader(atx)}}) - return ierr == nil - }); err != nil { - return status.Error(codes.Internal, err.Error()) - } - if sub == nil { - return nil - } - for { - select { - case <-stream.Context().Done(): - return nil - case <-sub.Full(): - return status.Error(codes.Canceled, "buffer overflow") - case rst := <-sub.Out(): - if err := stream.Send(&spacemeshv2.ActivationHeader{ - Versioned: &spacemeshv2.ActivationHeader_V1{V1: toHeader(rst.VerifiedActivationTx)}}, - ); err != nil { - if errors.Is(err, io.EOF) { - return nil - } - return status.Error(codes.Internal, err.Error()) - } - } - } -} - -func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { - v1 := &spacemeshv2.ActivationV1{ +func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2alpha1.ActivationV1 { + v1 := &spacemeshv2alpha1.ActivationV1{ Id: atx.ID().Bytes(), NodeId: atx.SmesherID.Bytes(), Signature: atx.Signature.Bytes(), @@ -213,12 +159,12 @@ func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { v1.CommittmentAtx = atx.CommitmentATX.Bytes() } if atx.VRFNonce != nil { - v1.VrfPostIndex = &spacemeshv2.VRFPostIndex{ + v1.VrfPostIndex = &spacemeshv2alpha1.VRFPostIndex{ Nonce: uint64(*atx.VRFNonce), } } if atx.InitialPost != nil { - v1.InitialPost = &spacemeshv2.Post{ + v1.InitialPost = &spacemeshv2alpha1.Post{ Nonce: atx.InitialPost.Nonce, Indices: atx.InitialPost.Indices, Pow: atx.InitialPost.Pow, @@ -226,19 +172,19 @@ func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { } if nipost := atx.NIPost; nipost != nil { if nipost.Post != nil { - v1.Post = &spacemeshv2.Post{ + v1.Post = &spacemeshv2alpha1.Post{ Nonce: nipost.Post.Nonce, Indices: nipost.Post.Indices, Pow: nipost.Post.Pow, } } if nipost.PostMetadata != nil { - v1.PostMeta = &spacemeshv2.PostMeta{ + v1.PostMeta = &spacemeshv2alpha1.PostMeta{ Challenge: nipost.PostMetadata.Challenge, Labels: nipost.PostMetadata.LabelsPerUnit, } } - v1.PoetProof = &spacemeshv2.PoetProof{ + v1.PoetProof = &spacemeshv2alpha1.PoetProof{ ProofNodes: make([][]byte, len(nipost.Membership.Nodes)), Leaf: nipost.Membership.LeafIndex, } @@ -249,18 +195,6 @@ func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationV1 { return v1 } -func toHeader(atx *types.VerifiedActivationTx) *spacemeshv2.ActivationHeaderV1 { - return &spacemeshv2.ActivationHeaderV1{ - Id: atx.ID().Bytes(), - NodeId: atx.SmesherID.Bytes(), - PublishEpoch: atx.PublishEpoch.Uint32(), - Coinbase: atx.Coinbase.String(), - Units: atx.NumUnits, - BaseHeight: uint32(atx.BaseTickHeight()), - Ticks: uint32(atx.TickCount()), - } -} - func NewActivationService(db *sql.Database) *ActivationService { return &ActivationService{db: db} } @@ -272,11 +206,11 @@ type ActivationService struct { var _ grpcserver.ServiceAPI = (*ActivationService)(nil) func (s *ActivationService) RegisterService(server *grpc.Server) { - spacemeshv2.RegisterActivationServiceServer(server, s) + spacemeshv2alpha1.RegisterActivationServiceServer(server, s) } func (s *ActivationService) RegisterHandlerService(mux *runtime.ServeMux) error { - return spacemeshv2.RegisterActivationServiceHandlerServer(context.Background(), mux, s) + return spacemeshv2alpha1.RegisterActivationServiceHandlerServer(context.Background(), mux, s) } // String returns the service name. @@ -286,8 +220,8 @@ func (s *ActivationService) String() string { func (s *ActivationService) List( ctx context.Context, - request *spacemeshv2.ActivationRequest, -) (*spacemeshv2.ActivationList, error) { + request *spacemeshv2alpha1.ActivationRequest, +) (*spacemeshv2alpha1.ActivationList, error) { ops, err := toOperations(request) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) @@ -298,41 +232,18 @@ func (s *ActivationService) List( } else if request.Limit == 0 { return nil, status.Error(codes.InvalidArgument, "limit must be set to <= 100") } - rst := make([]*spacemeshv2.Activation, 0, request.Limit) - if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { - rst = append(rst, &spacemeshv2.Activation{Versioned: &spacemeshv2.Activation_V1{V1: toAtx(atx)}}) - return true - }); err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - return &spacemeshv2.ActivationList{Activations: rst}, nil -} - -func (s *ActivationService) ListHeaders( - ctx context.Context, - request *spacemeshv2.ActivationRequest, -) (*spacemeshv2.ActivationHeaderList, error) { - ops, err := toOperations(request) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - if request.Limit > 10000 { - return nil, status.Error(codes.InvalidArgument, "limit is capped at 10000") - } else if request.Limit == 0 { - return nil, status.Error(codes.InvalidArgument, "limit must be set to <= 10000") - } - rst := make([]*spacemeshv2.ActivationHeader, 0, request.Limit) + rst := make([]*spacemeshv2alpha1.Activation, 0, request.Limit) if err := atxs.IterateAtxsOps(s.db, ops, func(atx *types.VerifiedActivationTx) bool { - rst = append(rst, &spacemeshv2.ActivationHeader{Versioned: &spacemeshv2.ActivationHeader_V1{V1: toHeader(atx)}}) + rst = append(rst, &spacemeshv2alpha1.Activation{Versioned: &spacemeshv2alpha1.Activation_V1{V1: toAtx(atx)}}) return true }); err != nil { return nil, status.Error(codes.Internal, err.Error()) } - return &spacemeshv2.ActivationHeaderList{Headers: rst}, nil + return &spacemeshv2alpha1.ActivationList{Activations: rst}, nil } -func toRequest(filter *spacemeshv2.ActivationStreamRequest) *spacemeshv2.ActivationRequest { - return &spacemeshv2.ActivationRequest{ +func toRequest(filter *spacemeshv2alpha1.ActivationStreamRequest) *spacemeshv2alpha1.ActivationRequest { + return &spacemeshv2alpha1.ActivationRequest{ NodeId: filter.NodeId, Id: filter.Id, Coinbase: filter.Coinbase, @@ -341,7 +252,7 @@ func toRequest(filter *spacemeshv2.ActivationStreamRequest) *spacemeshv2.Activat } } -func toOperations(filter *spacemeshv2.ActivationRequest) (atxs.Operations, error) { +func toOperations(filter *spacemeshv2alpha1.ActivationRequest) (atxs.Operations, error) { ops := atxs.Operations{} if filter == nil { return ops, nil diff --git a/go.mod b/go.mod index e34db8ae1c..73a2474e58 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/quic-go/quic-go v0.41.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20240208115250-3c4d8451b3a3 + github.com/spacemeshos/api/release/go v1.24.1-0.20240212102809-bab033316726 github.com/spacemeshos/economics v0.1.2 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 8b4f4a3cc9..198590e9c5 100644 --- a/go.sum +++ b/go.sum @@ -622,8 +622,8 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemeshos/api/release/go v1.24.1-0.20240208115250-3c4d8451b3a3 h1:tQMR1nVnvZZvXK0m+yKy6RaT1oMp77FEXruhFjoqANg= -github.com/spacemeshos/api/release/go v1.24.1-0.20240208115250-3c4d8451b3a3/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= +github.com/spacemeshos/api/release/go v1.24.1-0.20240212102809-bab033316726 h1:cvB60rt81p/FSTxKgcGJ25y+jwMbsJK6/jnv/7RZvR0= +github.com/spacemeshos/api/release/go v1.24.1-0.20240212102809-bab033316726/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= github.com/spacemeshos/economics v0.1.2 h1:kw8cE5SMa/7svHOGorCd2w8ef1y8iP0p47/2VDOK8Ns= github.com/spacemeshos/economics v0.1.2/go.mod h1:ngeWn5E/jy9dJP1MHyuk3ehF8NBMTYhchqVDhAHUUNk= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= diff --git a/node/node.go b/node/node.go index ee9f35f97d..ec4f23d3ec 100644 --- a/node/node.go +++ b/node/node.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/spacemeshos/go-spacemesh/api/grpcserver/v2alpha1" "net" "net/http" "net/url" @@ -34,7 +35,6 @@ import ( "github.com/spacemeshos/go-spacemesh/activation" "github.com/spacemeshos/go-spacemesh/api/grpcserver" - v2 "github.com/spacemeshos/go-spacemesh/api/grpcserver/v2" "github.com/spacemeshos/go-spacemesh/atxsdata" "github.com/spacemeshos/go-spacemesh/beacon" "github.com/spacemeshos/go-spacemesh/blocks" @@ -1365,10 +1365,10 @@ func (app *App) grpcService(svc grpcserver.Service, lg log.Log) (grpcserver.Serv app.cachedDB, types.ATXID(app.Config.Genesis.GoldenATX()), ), nil - case v2.Activation: - return v2.NewActivationService(app.db), nil - case v2.ActivationStream: - return v2.NewActivationStreamService(app.db), nil + case v2alpha1.Activation: + return v2alpha1.NewActivationService(app.db), nil + case v2alpha1.ActivationStream: + return v2alpha1.NewActivationStreamService(app.db), nil } return nil, fmt.Errorf("unknown service %s", svc) } From cea030a06d544e60f7f6fe032ae7ff121620861e Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 13 Feb 2024 11:15:26 +0100 Subject: [PATCH 16/25] Add tests --- api/grpcserver/v2alpha1/activation.go | 96 ++++++----- api/grpcserver/v2alpha1/activation_test.go | 185 +++++++++++++++++++++ api/grpcserver/v2alpha1/v2alpha1_test.go | 38 +++++ common/fixture/atxs.go | 85 ++++++++++ 4 files changed, 360 insertions(+), 44 deletions(-) create mode 100644 api/grpcserver/v2alpha1/activation_test.go create mode 100644 api/grpcserver/v2alpha1/v2alpha1_test.go create mode 100644 common/fixture/atxs.go diff --git a/api/grpcserver/v2alpha1/activation.go b/api/grpcserver/v2alpha1/activation.go index 4f1cd32e3d..89116ea48b 100644 --- a/api/grpcserver/v2alpha1/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -55,51 +55,9 @@ func (s *ActivationStreamService) Stream( ) error { var sub *events.BufferedSubscription[events.ActivationTx] if request.Watch { + matcher := resultsMatcher{request, stream.Context()} var err error - sub, err = events.SubscribeMatched(func(t *events.ActivationTx) bool { - if len(request.NodeId) > 0 { - var nodeId types.NodeID - copy(nodeId[:], request.NodeId) - - if t.SmesherID != nodeId { - return false - } - } - - if len(request.Id) > 0 { - var atxId types.ATXID - copy(atxId[:], request.Id) - - if t.ID() != atxId { - return false - } - } - - if len(request.Coinbase) > 0 { - addr, err := types.StringToAddress(request.Coinbase) - if err != nil { - ctxzap.Error(stream.Context(), "unable to convert atx coinbase", zap.Error(err)) - return false - } - if t.Coinbase != addr { - return false - } - } - - if request.StartEpoch != 0 { - if t.PublishEpoch.Uint32() < request.StartEpoch { - return false - } - } - - if request.EndEpoch != 0 { - if t.PublishEpoch.Uint32() > request.EndEpoch { - return false - } - } - - return true - }) + sub, err = events.SubscribeMatched(matcher.match) if err != nil { return status.Error(codes.Internal, err.Error()) } @@ -310,3 +268,53 @@ func toOperations(filter *spacemeshv2alpha1.ActivationRequest) (atxs.Operations, } return ops, nil } + +type resultsMatcher struct { + *spacemeshv2alpha1.ActivationStreamRequest + ctx context.Context +} + +func (m *resultsMatcher) match(t *events.ActivationTx) bool { + if len(m.NodeId) > 0 { + var nodeId types.NodeID + copy(nodeId[:], m.NodeId) + + if t.SmesherID != nodeId { + return false + } + } + + if len(m.Id) > 0 { + var atxId types.ATXID + copy(atxId[:], m.Id) + + if t.ID() != atxId { + return false + } + } + + if len(m.Coinbase) > 0 { + addr, err := types.StringToAddress(m.Coinbase) + if err != nil { + ctxzap.Error(m.ctx, "unable to convert atx coinbase", zap.Error(err)) + return false + } + if t.Coinbase != addr { + return false + } + } + + if m.StartEpoch != 0 { + if t.PublishEpoch.Uint32() < m.StartEpoch { + return false + } + } + + if m.EndEpoch != 0 { + if t.PublishEpoch.Uint32() > m.EndEpoch { + return false + } + } + + return true +} diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go new file mode 100644 index 0000000000..ccd9e1b105 --- /dev/null +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -0,0 +1,185 @@ +package v2alpha1 + +import ( + "context" + "errors" + spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" + "github.com/spacemeshos/go-spacemesh/common/fixture" + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/events" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/atxs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "io" + "testing" + "time" +) + +func TestActivationService_List(t *testing.T) { + db := sql.InMemory() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + gen := fixture.NewAtxsGenerator() + activations := make([]types.VerifiedActivationTx, 100) + require.NoError(t, db.WithTx(ctx, func(dtx *sql.Tx) error { + for i := range activations { + atx := gen.Next() + require.NoError(t, atxs.Add(dtx, atx)) + activations[i] = *atx + } + return nil + })) + + svc := NewActivationService(db) + cfg, cleanup := launchServer(t, svc) + t.Cleanup(cleanup) + + conn := dialGrpc(ctx, t, cfg) + client := spacemeshv2alpha1.NewActivationServiceClient(conn) + + t.Run("limit set too high", func(t *testing.T) { + _, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{Limit: 200}) + require.Error(t, err) + + s, ok := status.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.InvalidArgument, s.Code()) + require.Equal(t, s.Message(), "limit is capped at 100") + }) + + t.Run("no limit set", func(t *testing.T) { + _, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{}) + require.Error(t, err) + + s, ok := status.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.InvalidArgument, s.Code()) + require.Equal(t, s.Message(), "limit must be set to <= 100") + }) + + t.Run("all", func(t *testing.T) { + list, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{Limit: 100}) + require.NoError(t, err) + require.Equal(t, len(activations), len(list.Activations)) + }) +} + +func TestActivationStreamService_Stream(t *testing.T) { + db := sql.InMemory() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + gen := fixture.NewAtxsGenerator() + activations := make([]types.VerifiedActivationTx, 100) + require.NoError(t, db.WithTx(ctx, func(dtx *sql.Tx) error { + for i := range activations { + atx := gen.Next() + require.NoError(t, atxs.Add(dtx, atx)) + activations[i] = *atx + } + return nil + })) + + svc := NewActivationStreamService(db) + cfg, cleanup := launchServer(t, svc) + t.Cleanup(cleanup) + + conn := dialGrpc(ctx, t, cfg) + client := spacemeshv2alpha1.NewActivationStreamServiceClient(conn) + + t.Run("all", func(t *testing.T) { + events.InitializeReporter() + t.Cleanup(events.CloseEventReporter) + + stream, err := client.Stream(ctx, &spacemeshv2alpha1.ActivationStreamRequest{}) + require.NoError(t, err) + + var i int + for { + _, err := stream.Recv() + if errors.Is(err, io.EOF) { + break + } + i++ + } + require.Equal(t, len(activations), i) + }) + + t.Run("watch", func(t *testing.T) { + events.InitializeReporter() + t.Cleanup(events.CloseEventReporter) + + const ( + start = 100 + n = 10 + ) + + gen = fixture.NewAtxsGenerator().WithEpochs(start, 10) + var streamed []*events.ActivationTx + for i := 0; i < n; i++ { + streamed = append(streamed, &events.ActivationTx{VerifiedActivationTx: gen.Next()}) + } + + for _, tc := range []struct { + desc string + request *spacemeshv2alpha1.ActivationStreamRequest + }{ + { + desc: "ID", + request: &spacemeshv2alpha1.ActivationStreamRequest{ + Id: streamed[3].ID().Bytes(), + StartEpoch: start, + Watch: true, + }, + }, + { + desc: "NodeID", + request: &spacemeshv2alpha1.ActivationStreamRequest{ + NodeId: streamed[3].NodeID.Bytes(), + StartEpoch: start, + Watch: true, + }, + }, + { + desc: "Coinbase", + request: &spacemeshv2alpha1.ActivationStreamRequest{ + Coinbase: streamed[3].Coinbase.String(), + StartEpoch: start, + Watch: true, + }, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := client.Stream(ctx, tc.request) + require.NoError(t, err) + _, err = stream.Header() + require.NoError(t, err) + + var expect []*types.VerifiedActivationTx + for _, rst := range streamed { + events.ReportNewActivation(rst.VerifiedActivationTx) + matcher := resultsMatcher{tc.request, ctx} + if matcher.match(rst) { + expect = append(expect, rst.VerifiedActivationTx) + } + } + + for _, rst := range expect { + received, err := stream.Recv() + require.NoError(t, err) + require.Equal(t, toAtx(rst).String(), received.GetV1().String()) + } + }) + } + }) +} diff --git a/api/grpcserver/v2alpha1/v2alpha1_test.go b/api/grpcserver/v2alpha1/v2alpha1_test.go new file mode 100644 index 0000000000..28faa3387a --- /dev/null +++ b/api/grpcserver/v2alpha1/v2alpha1_test.go @@ -0,0 +1,38 @@ +package v2alpha1 + +import ( + "context" + "github.com/spacemeshos/go-spacemesh/api/grpcserver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "testing" +) + +func launchServer(tb testing.TB, services ...grpcserver.ServiceAPI) (grpcserver.Config, func()) { + cfg := grpcserver.DefaultTestConfig() + grpcService, err := grpcserver.NewWithServices(cfg.PublicListener, zaptest.NewLogger(tb).Named("grpc"), cfg, services) + require.NoError(tb, err) + + // start gRPC server + require.NoError(tb, grpcService.Start()) + + // update config with bound addresses + cfg.PublicListener = grpcService.BoundAddress + + return cfg, func() { assert.NoError(tb, grpcService.Close()) } +} + +func dialGrpc(ctx context.Context, tb testing.TB, cfg grpcserver.Config) *grpc.ClientConn { + tb.Helper() + conn, err := grpc.DialContext(ctx, + cfg.PublicListener, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + ) + require.NoError(tb, err) + tb.Cleanup(func() { require.NoError(tb, conn.Close()) }) + return conn +} diff --git a/common/fixture/atxs.go b/common/fixture/atxs.go new file mode 100644 index 0000000000..e21a53d241 --- /dev/null +++ b/common/fixture/atxs.go @@ -0,0 +1,85 @@ +package fixture + +import ( + "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" + "github.com/spacemeshos/go-spacemesh/signing" + "log" + "math/rand" + "os" + "time" + + "github.com/spacemeshos/go-spacemesh/common/types" +) + +// NewAtxsGenerator with some random parameters. +func NewAtxsGenerator() *AtxsGenerator { + return new(AtxsGenerator). + WithSeed(time.Now().UnixNano()). + WithEpochs(0, 10) +} + +// AtxsGenerator generates random activations. +// Activations are not syntactically or contextually valid. This is for testing databases and APIs. +type AtxsGenerator struct { + rng *rand.Rand + + Epochs []types.EpochID + Addrs []types.Address +} + +// WithSeed update randomness source. +func (g *AtxsGenerator) WithSeed(seed int64) *AtxsGenerator { + g.rng = rand.New(rand.NewSource(seed)) + return g +} + +// WithEpochs update epochs ids. +func (g *AtxsGenerator) WithEpochs(start, n int) *AtxsGenerator { + g.Epochs = nil + for i := 1; i <= n; i++ { + g.Epochs = append(g.Epochs, types.EpochID(start+i)) + } + return g +} + +// Next generates VerifiedActivationTx. +func (g *AtxsGenerator) Next() *types.VerifiedActivationTx { + var atx types.VerifiedActivationTx + + var prevAtxId types.ATXID + g.rng.Read(prevAtxId[:]) + var posAtxId types.ATXID + g.rng.Read(posAtxId[:]) + var nodeId types.NodeID + g.rng.Read(nodeId[:]) + + signer, err := signing.NewEdSigner() + if err != nil { + log.Println("failed to create signer:", err) + os.Exit(1) + } + + atx = types.VerifiedActivationTx{ + ActivationTx: &types.ActivationTx{ + InnerActivationTx: types.InnerActivationTx{ + NIPostChallenge: types.NIPostChallenge{ + Sequence: g.rng.Uint64(), + PrevATXID: prevAtxId, + PublishEpoch: g.Epochs[g.rng.Intn(len(g.Epochs))], + PositioningATX: posAtxId, + }, + Coinbase: wallet.Address(signer.PublicKey().Bytes()), + NumUnits: g.rng.Uint32(), + NodeID: &nodeId, + }, + }, + } + + atx.SetEffectiveNumUnits(atx.NumUnits) + + var atxId types.ATXID + g.rng.Read(atxId[:]) + atx.SetID(atxId) + + return &atx +} From c20d397dfeacc152f2e75e4a1b4e7aa73f242d41 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 13 Feb 2024 12:17:01 +0100 Subject: [PATCH 17/25] Add ActivationsCount handler --- api/grpcserver/v2alpha1/activation.go | 21 ++++++++++++++++ api/grpcserver/v2alpha1/activation_test.go | 29 ++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +-- sql/atxs/atxs.go | 12 +++++++++ 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/api/grpcserver/v2alpha1/activation.go b/api/grpcserver/v2alpha1/activation.go index 89116ea48b..d8c70db7f1 100644 --- a/api/grpcserver/v2alpha1/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -200,6 +200,27 @@ func (s *ActivationService) List( return &spacemeshv2alpha1.ActivationList{Activations: rst}, nil } +func (s *ActivationService) ActivationsCount( + ctx context.Context, + request *spacemeshv2alpha1.ActivationsCountRequest, +) (*spacemeshv2alpha1.ActivationsCountResponse, error) { + ops := atxs.Operations{Filter: []atxs.Op{ + { + Field: atxs.Epoch, + Token: atxs.Eq, + Value: int64(request.Epoch), + }, + }} + + count, err := atxs.CountAtxsByEpoch(s.db, ops) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return &spacemeshv2alpha1.ActivationsCountResponse{Count: count}, nil + +} + func toRequest(filter *spacemeshv2alpha1.ActivationStreamRequest) *spacemeshv2alpha1.ActivationRequest { return &spacemeshv2alpha1.ActivationRequest{ NodeId: filter.NodeId, diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go index ccd9e1b105..8bef861f78 100644 --- a/api/grpcserver/v2alpha1/activation_test.go +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -183,3 +183,32 @@ func TestActivationStreamService_Stream(t *testing.T) { } }) } + +func TestActivationService_ActivationsCount(t *testing.T) { + db := sql.InMemory() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + gen := fixture.NewAtxsGenerator().WithEpochs(0, 1) + activations := make([]types.VerifiedActivationTx, 30) + require.NoError(t, db.WithTx(ctx, func(dtx *sql.Tx) error { + for i := range activations { + atx := gen.Next() + require.NoError(t, atxs.Add(dtx, atx)) + activations[i] = *atx + } + return nil + })) + + svc := NewActivationService(db) + cfg, cleanup := launchServer(t, svc) + t.Cleanup(cleanup) + + conn := dialGrpc(ctx, t, cfg) + client := spacemeshv2alpha1.NewActivationServiceClient(conn) + + count, err := client.ActivationsCount(ctx, &spacemeshv2alpha1.ActivationsCountRequest{Epoch: activations[3].PublishEpoch.Uint32()}) + require.NoError(t, err) + require.Equal(t, len(activations), int(count.Count)) +} diff --git a/go.mod b/go.mod index 73a2474e58..9e7c2a6ad0 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/quic-go/quic-go v0.41.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20240212102809-bab033316726 + github.com/spacemeshos/api/release/go v1.24.1-0.20240212185822-83bc5a25a78e github.com/spacemeshos/economics v0.1.2 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 198590e9c5..94407fa5b7 100644 --- a/go.sum +++ b/go.sum @@ -622,8 +622,8 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemeshos/api/release/go v1.24.1-0.20240212102809-bab033316726 h1:cvB60rt81p/FSTxKgcGJ25y+jwMbsJK6/jnv/7RZvR0= -github.com/spacemeshos/api/release/go v1.24.1-0.20240212102809-bab033316726/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= +github.com/spacemeshos/api/release/go v1.24.1-0.20240212185822-83bc5a25a78e h1:Vf1V+J9ExZolNie6W6fSyi2H3DQvIPPC5p/IP6Dzc0o= +github.com/spacemeshos/api/release/go v1.24.1-0.20240212185822-83bc5a25a78e/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= github.com/spacemeshos/economics v0.1.2 h1:kw8cE5SMa/7svHOGorCd2w8ef1y8iP0p47/2VDOK8Ns= github.com/spacemeshos/economics v0.1.2/go.mod h1:ngeWn5E/jy9dJP1MHyuk3ehF8NBMTYhchqVDhAHUUNk= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index bce63f09ef..ac0db3ff72 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -568,6 +568,18 @@ func IterateAtxsOps( return derr } +func CountAtxsByEpoch(db sql.Executor, operations Operations) (count uint32, err error) { + _, err = db.Exec( + "SELECT count(*) FROM atxs"+filterFrom(operations), + bindingsFrom(operations), + func(stmt *sql.Statement) bool { + count = uint32(stmt.ColumnInt32(0)) + return true + }, + ) + return +} + func filterFrom(operations Operations) string { var queryBuilder strings.Builder From 7a6a2eaeafb405e590e39edd56c72a2b2ec6be51 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 13 Feb 2024 13:27:39 +0100 Subject: [PATCH 18/25] Extract query builder into separate pkg --- api/grpcserver/v2alpha1/activation.go | 58 ++++++++------ api/grpcserver/v2alpha1/activation_test.go | 9 +++ sql/atxs/atxs.go | 88 ++-------------------- sql/builder/builder.go | 81 ++++++++++++++++++++ 4 files changed, 130 insertions(+), 106 deletions(-) create mode 100644 sql/builder/builder.go diff --git a/api/grpcserver/v2alpha1/activation.go b/api/grpcserver/v2alpha1/activation.go index d8c70db7f1..73e6d5c1a5 100644 --- a/api/grpcserver/v2alpha1/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -4,6 +4,7 @@ import ( "context" "errors" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/spacemeshos/go-spacemesh/sql/builder" "go.uber.org/zap" "io" @@ -204,10 +205,10 @@ func (s *ActivationService) ActivationsCount( ctx context.Context, request *spacemeshv2alpha1.ActivationsCountRequest, ) (*spacemeshv2alpha1.ActivationsCountResponse, error) { - ops := atxs.Operations{Filter: []atxs.Op{ + ops := builder.Operations{Filter: []builder.Op{ { - Field: atxs.Epoch, - Token: atxs.Eq, + Field: builder.Epoch, + Token: builder.Eq, Value: int64(request.Epoch), }, }} @@ -231,62 +232,69 @@ func toRequest(filter *spacemeshv2alpha1.ActivationStreamRequest) *spacemeshv2al } } -func toOperations(filter *spacemeshv2alpha1.ActivationRequest) (atxs.Operations, error) { - ops := atxs.Operations{} +func toOperations(filter *spacemeshv2alpha1.ActivationRequest) (builder.Operations, error) { + ops := builder.Operations{} if filter == nil { return ops, nil } if filter.NodeId != nil { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Smesher, - Token: atxs.Eq, + ops.Filter = append(ops.Filter, builder.Op{ + Field: builder.Smesher, + Token: builder.Eq, Value: filter.NodeId, }) } if filter.Id != nil { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Id, - Token: atxs.Eq, + ops.Filter = append(ops.Filter, builder.Op{ + Field: builder.Id, + Token: builder.Eq, Value: filter.Id, }) } if len(filter.Coinbase) > 0 { addr, err := types.StringToAddress(filter.Coinbase) if err != nil { - return atxs.Operations{}, err + return builder.Operations{}, err } - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Coinbase, - Token: atxs.Eq, + ops.Filter = append(ops.Filter, builder.Op{ + Field: builder.Coinbase, + Token: builder.Eq, Value: addr.Bytes(), }) } if filter.StartEpoch != 0 { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Epoch, - Token: atxs.Gte, + ops.Filter = append(ops.Filter, builder.Op{ + Field: builder.Epoch, + Token: builder.Gte, Value: int64(filter.StartEpoch), }) } if filter.EndEpoch != 0 { - ops.Filter = append(ops.Filter, atxs.Op{ - Field: atxs.Epoch, - Token: atxs.Lte, + ops.Filter = append(ops.Filter, builder.Op{ + Field: builder.Epoch, + Token: builder.Lte, Value: int64(filter.EndEpoch), }) } + + ops.Other = append(ops.Other, builder.Op{ + Field: builder.OrderBy, + Value: "epoch asc, id", + }) + if filter.Limit != 0 { - ops.Other = append(ops.Other, atxs.Op{ - Field: atxs.Limit, + ops.Other = append(ops.Other, builder.Op{ + Field: builder.Limit, Value: int64(filter.Limit), }) } if filter.Offset != 0 { - ops.Other = append(ops.Other, atxs.Op{ - Field: atxs.Offset, + ops.Other = append(ops.Other, builder.Op{ + Field: builder.Offset, Value: int64(filter.Offset), }) } + return ops, nil } diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go index 8bef861f78..3bf6a394ff 100644 --- a/api/grpcserver/v2alpha1/activation_test.go +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -62,6 +62,15 @@ func TestActivationService_List(t *testing.T) { require.Equal(t, s.Message(), "limit must be set to <= 100") }) + t.Run("limit and offset", func(t *testing.T) { + list, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{ + Limit: 25, + Offset: 50, + }) + require.NoError(t, err) + require.Equal(t, 25, len(list.Activations)) + }) + t.Run("all", func(t *testing.T) { list, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{Limit: 100}) require.NoError(t, err) diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index ac0db3ff72..ab57e4a625 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -2,8 +2,7 @@ package atxs import ( "fmt" - "strconv" - "strings" + "github.com/spacemeshos/go-spacemesh/sql/builder" "time" "github.com/spacemeshos/go-spacemesh/codec" @@ -507,54 +506,15 @@ func SetValidity(db sql.Executor, id types.ATXID, validity types.Validity) error return nil } -// TODO(dshulyak) extract code for query building into separate module - -type token string - -const ( - Eq token = "=" - NotEq token = "!=" - Gt token = ">" - Gte token = ">=" - Lt token = "<" - Lte token = "<=" - And token = "and" - Where token = "where" -) - -type field string - -const ( - Epoch field = "epoch" - Smesher field = "pubkey" - Coinbase field = "coinbase" - Id field = "id" - Offset field = "offset" - Limit field = "limit" -) - -type Op struct { - Field field - Token token - // Value will be type casted to one the expected types. - // Operation will panic if it doesn't match any of expected. - Value any -} - -type Operations struct { - Filter []Op - Other []Op -} - func IterateAtxsOps( db sql.Executor, - operations Operations, + operations builder.Operations, fn func(*types.VerifiedActivationTx) bool, ) error { var derr error _, err := db.Exec( - fullQuery+filterFrom(operations), - bindingsFrom(operations), + fullQuery+builder.FilterFrom(operations), + builder.BindingsFrom(operations), decoder(func(atx *types.VerifiedActivationTx, err error) bool { if atx != nil { return fn(atx) @@ -568,10 +528,10 @@ func IterateAtxsOps( return derr } -func CountAtxsByEpoch(db sql.Executor, operations Operations) (count uint32, err error) { +func CountAtxsByEpoch(db sql.Executor, operations builder.Operations) (count uint32, err error) { _, err = db.Exec( - "SELECT count(*) FROM atxs"+filterFrom(operations), - bindingsFrom(operations), + "SELECT count(*) FROM atxs"+builder.FilterFrom(operations), + builder.BindingsFrom(operations), func(stmt *sql.Statement) bool { count = uint32(stmt.ColumnInt32(0)) return true @@ -579,37 +539,3 @@ func CountAtxsByEpoch(db sql.Executor, operations Operations) (count uint32, err ) return } - -func filterFrom(operations Operations) string { - var queryBuilder strings.Builder - - for i, op := range operations.Filter { - if i == 0 { - queryBuilder.WriteString(" " + string(Where)) - } - if i != 0 { - queryBuilder.WriteString(" " + string(And)) - } - queryBuilder.WriteString(" " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1)) - } - queryBuilder.WriteString(" order by epoch asc, id") - for _, op := range operations.Other { - queryBuilder.WriteString(fmt.Sprintf(" %s %v", string(op.Field), op.Value)) - } - return queryBuilder.String() -} - -func bindingsFrom(operations Operations) sql.Encoder { - return func(stmt *sql.Statement) { - for i, op := range operations.Filter { - switch value := op.Value.(type) { - case int64: - stmt.BindInt64(i+1, value) - case []byte: - stmt.BindBytes(i+1, value) - default: - panic(fmt.Sprintf("unexpected type %T", value)) - } - } - } -} diff --git a/sql/builder/builder.go b/sql/builder/builder.go new file mode 100644 index 0000000000..ea7aacb771 --- /dev/null +++ b/sql/builder/builder.go @@ -0,0 +1,81 @@ +package builder + +import ( + "fmt" + "github.com/spacemeshos/go-spacemesh/sql" + "strconv" + "strings" +) + +type token string + +const ( + Eq token = "=" + NotEq token = "!=" + Gt token = ">" + Gte token = ">=" + Lt token = "<" + Lte token = "<=" + And token = "and" + Where token = "where" +) + +type field string + +const ( + Epoch field = "epoch" + Smesher field = "pubkey" + Coinbase field = "coinbase" + Id field = "id" + Offset field = "offset" + Limit field = "limit" + OrderBy field = "order by" +) + +type Op struct { + Field field + Token token + // Value will be type casted to one the expected types. + // Operation will panic if it doesn't match any of expected. + Value any +} + +type Operations struct { + Filter []Op + Other []Op +} + +func FilterFrom(operations Operations) string { + var queryBuilder strings.Builder + + for i, op := range operations.Filter { + if i == 0 { + queryBuilder.WriteString(" " + string(Where)) + } + if i != 0 { + queryBuilder.WriteString(" " + string(And)) + } + queryBuilder.WriteString(" " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1)) + } + + for _, op := range operations.Other { + queryBuilder.WriteString(fmt.Sprintf(" %s %v", string(op.Field), op.Value)) + } + + return queryBuilder.String() +} + +func BindingsFrom(operations Operations) sql.Encoder { + return func(stmt *sql.Statement) { + for i, op := range operations.Filter { + switch value := op.Value.(type) { + case int64: + stmt.BindInt64(i+1, value) + case []byte: + stmt.BindBytes(i+1, value) + default: + panic(fmt.Sprintf("unexpected type %T", value)) + } + } + } +} From f90fc70c114d7641bdd830b4fe70c56c83099fce Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 13 Feb 2024 15:05:50 +0100 Subject: [PATCH 19/25] lint fix --- api/grpcserver/v2alpha1/activation.go | 13 ++++++------- api/grpcserver/v2alpha1/activation_test.go | 20 ++++++++++++-------- api/grpcserver/v2alpha1/v2alpha1_test.go | 6 ++++-- common/fixture/atxs.go | 4 ++-- node/node.go | 2 +- sql/atxs/atxs.go | 2 +- sql/builder/builder.go | 3 ++- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/api/grpcserver/v2alpha1/activation.go b/api/grpcserver/v2alpha1/activation.go index 73e6d5c1a5..1056ef4e64 100644 --- a/api/grpcserver/v2alpha1/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -3,24 +3,23 @@ package v2alpha1 import ( "context" "errors" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - "github.com/spacemeshos/go-spacemesh/sql/builder" - "go.uber.org/zap" "io" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" - "github.com/spacemeshos/go-spacemesh/api/grpcserver" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/atxs" + "github.com/spacemeshos/go-spacemesh/sql/builder" ) const ( @@ -89,7 +88,8 @@ func (s *ActivationStreamService) Stream( return status.Error(codes.Canceled, "buffer overflow") case rst := <-sub.Out(): if err := stream.Send(&spacemeshv2alpha1.Activation{ - Versioned: &spacemeshv2alpha1.Activation_V1{V1: toAtx(rst.VerifiedActivationTx)}}, + Versioned: &spacemeshv2alpha1.Activation_V1{V1: toAtx(rst.VerifiedActivationTx)}, + }, ); err != nil { if errors.Is(err, io.EOF) { return nil @@ -219,7 +219,6 @@ func (s *ActivationService) ActivationsCount( } return &spacemeshv2alpha1.ActivationsCountResponse{Count: count}, nil - } func toRequest(filter *spacemeshv2alpha1.ActivationStreamRequest) *spacemeshv2alpha1.ActivationRequest { diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go index 3bf6a394ff..238a070cac 100644 --- a/api/grpcserver/v2alpha1/activation_test.go +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -3,19 +3,21 @@ package v2alpha1 import ( "context" "errors" + "io" + "testing" + "time" + spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/spacemeshos/go-spacemesh/common/fixture" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/atxs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "io" - "testing" - "time" ) func TestActivationService_List(t *testing.T) { @@ -217,7 +219,9 @@ func TestActivationService_ActivationsCount(t *testing.T) { conn := dialGrpc(ctx, t, cfg) client := spacemeshv2alpha1.NewActivationServiceClient(conn) - count, err := client.ActivationsCount(ctx, &spacemeshv2alpha1.ActivationsCountRequest{Epoch: activations[3].PublishEpoch.Uint32()}) + count, err := client.ActivationsCount(ctx, &spacemeshv2alpha1.ActivationsCountRequest{ + Epoch: activations[3].PublishEpoch.Uint32(), + }) require.NoError(t, err) require.Equal(t, len(activations), int(count.Count)) } diff --git a/api/grpcserver/v2alpha1/v2alpha1_test.go b/api/grpcserver/v2alpha1/v2alpha1_test.go index 28faa3387a..d0010e8511 100644 --- a/api/grpcserver/v2alpha1/v2alpha1_test.go +++ b/api/grpcserver/v2alpha1/v2alpha1_test.go @@ -2,13 +2,15 @@ package v2alpha1 import ( "context" - "github.com/spacemeshos/go-spacemesh/api/grpcserver" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "testing" + + "github.com/spacemeshos/go-spacemesh/api/grpcserver" ) func launchServer(tb testing.TB, services ...grpcserver.ServiceAPI) (grpcserver.Config, func()) { diff --git a/common/fixture/atxs.go b/common/fixture/atxs.go index e21a53d241..50205925f2 100644 --- a/common/fixture/atxs.go +++ b/common/fixture/atxs.go @@ -1,14 +1,14 @@ package fixture import ( - "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" - "github.com/spacemeshos/go-spacemesh/signing" "log" "math/rand" "os" "time" "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" + "github.com/spacemeshos/go-spacemesh/signing" ) // NewAtxsGenerator with some random parameters. diff --git a/node/node.go b/node/node.go index ec4f23d3ec..12e5f64d63 100644 --- a/node/node.go +++ b/node/node.go @@ -7,7 +7,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/spacemeshos/go-spacemesh/api/grpcserver/v2alpha1" "net" "net/http" "net/url" @@ -35,6 +34,7 @@ import ( "github.com/spacemeshos/go-spacemesh/activation" "github.com/spacemeshos/go-spacemesh/api/grpcserver" + "github.com/spacemeshos/go-spacemesh/api/grpcserver/v2alpha1" "github.com/spacemeshos/go-spacemesh/atxsdata" "github.com/spacemeshos/go-spacemesh/beacon" "github.com/spacemeshos/go-spacemesh/blocks" diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index ab57e4a625..97ab78ddee 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -2,12 +2,12 @@ package atxs import ( "fmt" - "github.com/spacemeshos/go-spacemesh/sql/builder" "time" "github.com/spacemeshos/go-spacemesh/codec" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/builder" ) const fullQuery = `select id, atx, base_tick_height, tick_count, pubkey, diff --git a/sql/builder/builder.go b/sql/builder/builder.go index ea7aacb771..8c808046b0 100644 --- a/sql/builder/builder.go +++ b/sql/builder/builder.go @@ -2,9 +2,10 @@ package builder import ( "fmt" - "github.com/spacemeshos/go-spacemesh/sql" "strconv" "strings" + + "github.com/spacemeshos/go-spacemesh/sql" ) type token string From 4f77497ff62d6848b425bffffcb0160b4dfbc11f Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 13 Feb 2024 15:44:27 +0100 Subject: [PATCH 20/25] Move service name to const --- api/grpcserver/config.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/api/grpcserver/config.go b/api/grpcserver/config.go index 43fd88c872..d1ea971662 100644 --- a/api/grpcserver/config.go +++ b/api/grpcserver/config.go @@ -27,23 +27,25 @@ type Config struct { type Service = string const ( - Admin Service = "admin" - Debug Service = "debug" - GlobalState Service = "global" - Mesh Service = "mesh" - Transaction Service = "transaction" - Activation Service = "activation" - Smesher Service = "smesher" - Post Service = "post" - Node Service = "node" + Admin Service = "admin" + Debug Service = "debug" + GlobalState Service = "global" + Mesh Service = "mesh" + Transaction Service = "transaction" + Activation Service = "activation" + Smesher Service = "smesher" + Post Service = "post" + Node Service = "node" + ActivationV2Alpha1 Service = "activation_v2alpha1" + ActivationStreamV2Alpha1 Service = "activation_stream_v2alpha1" ) // DefaultConfig defines the default configuration options for api. func DefaultConfig() Config { return Config{ - PublicServices: []Service{GlobalState, Mesh, Transaction, Node, Activation, "activation_v2alpha1"}, + PublicServices: []Service{GlobalState, Mesh, Transaction, Node, Activation, ActivationV2Alpha1}, PublicListener: "0.0.0.0:9092", - PrivateServices: []Service{Admin, Smesher, Debug, "activation_stream_v2alpha1"}, + PrivateServices: []Service{Admin, Smesher, Debug, ActivationStreamV2Alpha1}, PrivateListener: "127.0.0.1:9093", PostServices: []Service{Post}, PostListener: "127.0.0.1:9094", From 033872e9b035db1735e00c052e87fcc321588301 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Thu, 15 Feb 2024 12:24:24 +0100 Subject: [PATCH 21/25] Add unit tests & bump api version --- api/grpcserver/v2alpha1/activation.go | 10 ++++---- api/grpcserver/v2alpha1/activation_test.go | 27 ++++++++++++++++++++++ common/fixture/atxs.go | 1 + go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/api/grpcserver/v2alpha1/activation.go b/api/grpcserver/v2alpha1/activation.go index 1056ef4e64..6c251470d0 100644 --- a/api/grpcserver/v2alpha1/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -107,7 +107,7 @@ func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2alpha1.ActivationV1 { Signature: atx.Signature.Bytes(), PublishEpoch: atx.PublishEpoch.Uint32(), Sequence: atx.Sequence, - PrevAtx: atx.PrevATXID[:], + PreviousAtx: atx.PrevATXID[:], PositioningAtx: atx.PositioningATX[:], Coinbase: atx.Coinbase.String(), Units: atx.NumUnits, @@ -139,16 +139,16 @@ func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2alpha1.ActivationV1 { } if nipost.PostMetadata != nil { v1.PostMeta = &spacemeshv2alpha1.PostMeta{ - Challenge: nipost.PostMetadata.Challenge, - Labels: nipost.PostMetadata.LabelsPerUnit, + Challenge: nipost.PostMetadata.Challenge, + LabelsPerUnit: nipost.PostMetadata.LabelsPerUnit, } } - v1.PoetProof = &spacemeshv2alpha1.PoetProof{ + v1.Membership = &spacemeshv2alpha1.PoetMembershipProof{ ProofNodes: make([][]byte, len(nipost.Membership.Nodes)), Leaf: nipost.Membership.LeafIndex, } for i, node := range nipost.Membership.Nodes { - v1.PoetProof.ProofNodes[i] = node.Bytes() + v1.Membership.ProofNodes[i] = node.Bytes() } } return v1 diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go index 238a070cac..052298d110 100644 --- a/api/grpcserver/v2alpha1/activation_test.go +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -78,6 +78,33 @@ func TestActivationService_List(t *testing.T) { require.NoError(t, err) require.Equal(t, len(activations), len(list.Activations)) }) + + t.Run("coinbase", func(t *testing.T) { + list, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{ + Limit: 1, + Coinbase: activations[3].Coinbase.String(), + }) + require.NoError(t, err) + require.Equal(t, activations[3].ID().Bytes(), list.GetActivations()[0].GetV1().GetId()) + }) + + t.Run("nodeId", func(t *testing.T) { + list, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{ + Limit: 1, + NodeId: activations[1].SmesherID.Bytes(), + }) + require.NoError(t, err) + require.Equal(t, activations[1].ID().Bytes(), list.GetActivations()[0].GetV1().GetId()) + }) + + t.Run("id", func(t *testing.T) { + list, err := client.List(ctx, &spacemeshv2alpha1.ActivationRequest{ + Limit: 1, + Id: activations[3].ID().Bytes(), + }) + require.NoError(t, err) + require.Equal(t, activations[3].ID().Bytes(), list.GetActivations()[0].GetV1().GetId()) + }) } func TestActivationStreamService_Stream(t *testing.T) { diff --git a/common/fixture/atxs.go b/common/fixture/atxs.go index 50205925f2..efc7d57913 100644 --- a/common/fixture/atxs.go +++ b/common/fixture/atxs.go @@ -72,6 +72,7 @@ func (g *AtxsGenerator) Next() *types.VerifiedActivationTx { NumUnits: g.rng.Uint32(), NodeID: &nodeId, }, + SmesherID: nodeId, }, } diff --git a/go.mod b/go.mod index 9e7c2a6ad0..2bf0566eac 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/quic-go/quic-go v0.41.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/seehuhn/mt19937 v1.0.0 - github.com/spacemeshos/api/release/go v1.24.1-0.20240212185822-83bc5a25a78e + github.com/spacemeshos/api/release/go v1.28.1-0.20240215101325-80fe2752264b github.com/spacemeshos/economics v0.1.2 github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.1.12 diff --git a/go.sum b/go.sum index 94407fa5b7..93771c2b9a 100644 --- a/go.sum +++ b/go.sum @@ -622,8 +622,8 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemeshos/api/release/go v1.24.1-0.20240212185822-83bc5a25a78e h1:Vf1V+J9ExZolNie6W6fSyi2H3DQvIPPC5p/IP6Dzc0o= -github.com/spacemeshos/api/release/go v1.24.1-0.20240212185822-83bc5a25a78e/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= +github.com/spacemeshos/api/release/go v1.28.1-0.20240215101325-80fe2752264b h1:SYHzzlFfoJwJeeDh0WZ3/+hObt33mZPlkGTNU2hD5Ds= +github.com/spacemeshos/api/release/go v1.28.1-0.20240215101325-80fe2752264b/go.mod h1:fK9RBD8eTVXHrqkkal2bwQB4N8M9sOhPs4rnVmWqEc0= github.com/spacemeshos/economics v0.1.2 h1:kw8cE5SMa/7svHOGorCd2w8ef1y8iP0p47/2VDOK8Ns= github.com/spacemeshos/economics v0.1.2/go.mod h1:ngeWn5E/jy9dJP1MHyuk3ehF8NBMTYhchqVDhAHUUNk= github.com/spacemeshos/fixed v0.1.1 h1:N1y4SUpq1EV+IdJrWJwUCt1oBFzeru/VKVcBsvPc2Fk= From f533c6d5cdb8fd36633c645146b6b1c7117069eb Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 16 Feb 2024 12:54:30 +0100 Subject: [PATCH 22/25] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bartosz Różański --- api/grpcserver/v2alpha1/activation.go | 17 +++++++++-------- api/grpcserver/v2alpha1/activation_test.go | 15 ++++++--------- sql/builder/builder.go | 7 +++---- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/api/grpcserver/v2alpha1/activation.go b/api/grpcserver/v2alpha1/activation.go index 6c251470d0..8e5bc7777f 100644 --- a/api/grpcserver/v2alpha1/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -87,13 +87,13 @@ func (s *ActivationStreamService) Stream( case <-sub.Full(): return status.Error(codes.Canceled, "buffer overflow") case rst := <-sub.Out(): - if err := stream.Send(&spacemeshv2alpha1.Activation{ + err := stream.Send(&spacemeshv2alpha1.Activation{ Versioned: &spacemeshv2alpha1.Activation_V1{V1: toAtx(rst.VerifiedActivationTx)}, - }, - ); err != nil { - if errors.Is(err, io.EOF) { - return nil - } + }) + switch { + case errors.Is(err, io.EOF): + return nil + case err != nil: return status.Error(codes.Internal, err.Error()) } } @@ -186,9 +186,10 @@ func (s *ActivationService) List( return nil, status.Error(codes.InvalidArgument, err.Error()) } // every full atx is ~1KB. 100 atxs is ~100KB. - if request.Limit > 100 { + switch { + case request.Limit > 100: return nil, status.Error(codes.InvalidArgument, "limit is capped at 100") - } else if request.Limit == 0 { + case request.Limit == 0: return nil, status.Error(codes.InvalidArgument, "limit must be set to <= 100") } rst := make([]*spacemeshv2alpha1.Activation, 0, request.Limit) diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go index 052298d110..e248bba390 100644 --- a/api/grpcserver/v2alpha1/activation_test.go +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -28,14 +28,11 @@ func TestActivationService_List(t *testing.T) { gen := fixture.NewAtxsGenerator() activations := make([]types.VerifiedActivationTx, 100) - require.NoError(t, db.WithTx(ctx, func(dtx *sql.Tx) error { - for i := range activations { - atx := gen.Next() - require.NoError(t, atxs.Add(dtx, atx)) - activations[i] = *atx - } - return nil - })) + for i := range activations { + atx := gen.Next() + require.NoError(t, atxs.Add(db, atx)) + activations[i] = *atx + } svc := NewActivationService(db) cfg, cleanup := launchServer(t, svc) @@ -70,7 +67,7 @@ func TestActivationService_List(t *testing.T) { Offset: 50, }) require.NoError(t, err) - require.Equal(t, 25, len(list.Activations)) + require.Len(t, list.Activations, 25) }) t.Run("all", func(t *testing.T) { diff --git a/sql/builder/builder.go b/sql/builder/builder.go index 8c808046b0..ed831f9d07 100644 --- a/sql/builder/builder.go +++ b/sql/builder/builder.go @@ -51,10 +51,9 @@ func FilterFrom(operations Operations) string { for i, op := range operations.Filter { if i == 0 { - queryBuilder.WriteString(" " + string(Where)) - } - if i != 0 { - queryBuilder.WriteString(" " + string(And)) + queryBuilder.WriteString(" where") + } else { + queryBuilder.WriteString(" and") } queryBuilder.WriteString(" " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1)) } From fe17373067db9d471761b4550e3b3d1eac23ebf0 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 16 Feb 2024 13:53:21 +0100 Subject: [PATCH 23/25] Apply review suggestions --- api/grpcserver/v2alpha1/activation.go | 82 ++++++++++++---------- api/grpcserver/v2alpha1/activation_test.go | 51 +++++--------- common/fixture/atxs.go | 43 ++++++++++-- node/node.go | 6 +- sql/atxs/atxs.go | 2 +- sql/builder/builder.go | 28 +++++--- 6 files changed, 122 insertions(+), 90 deletions(-) diff --git a/api/grpcserver/v2alpha1/activation.go b/api/grpcserver/v2alpha1/activation.go index 8e5bc7777f..d113a0c3fd 100644 --- a/api/grpcserver/v2alpha1/activation.go +++ b/api/grpcserver/v2alpha1/activation.go @@ -3,6 +3,7 @@ package v2alpha1 import ( "context" "errors" + "fmt" "io" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -14,7 +15,6 @@ import ( "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - "github.com/spacemeshos/go-spacemesh/api/grpcserver" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/sql" @@ -27,16 +27,14 @@ const ( ActivationStream = "activation_stream_v2alpha1" ) -func NewActivationStreamService(db *sql.Database) *ActivationStreamService { +func NewActivationStreamService(db sql.Executor) *ActivationStreamService { return &ActivationStreamService{db: db} } type ActivationStreamService struct { - db *sql.Database + db sql.Executor } -var _ grpcserver.ServiceAPI = (*ActivationStreamService)(nil) - func (s *ActivationStreamService) RegisterService(server *grpc.Server) { spacemeshv2alpha1.RegisterActivationStreamServiceServer(server, s) } @@ -129,41 +127,51 @@ func toAtx(atx *types.VerifiedActivationTx) *spacemeshv2alpha1.ActivationV1 { Pow: atx.InitialPost.Pow, } } - if nipost := atx.NIPost; nipost != nil { - if nipost.Post != nil { - v1.Post = &spacemeshv2alpha1.Post{ - Nonce: nipost.Post.Nonce, - Indices: nipost.Post.Indices, - Pow: nipost.Post.Pow, - } - } - if nipost.PostMetadata != nil { - v1.PostMeta = &spacemeshv2alpha1.PostMeta{ - Challenge: nipost.PostMetadata.Challenge, - LabelsPerUnit: nipost.PostMetadata.LabelsPerUnit, - } - } - v1.Membership = &spacemeshv2alpha1.PoetMembershipProof{ - ProofNodes: make([][]byte, len(nipost.Membership.Nodes)), - Leaf: nipost.Membership.LeafIndex, - } - for i, node := range nipost.Membership.Nodes { - v1.Membership.ProofNodes[i] = node.Bytes() - } + + if atx.NIPost == nil { + panic(fmt.Sprintf("nil nipost for atx %s", atx.ShortString())) + } + + if atx.NIPost.Post == nil { + panic(fmt.Sprintf("nil nipost post for atx %s", atx.ShortString())) + } + + if atx.NIPost.PostMetadata == nil { + panic(fmt.Sprintf("nil nipost post metadata for atx %s", atx.ShortString())) + } + + nipost := atx.NIPost + v1.Post = &spacemeshv2alpha1.Post{ + Nonce: nipost.Post.Nonce, + Indices: nipost.Post.Indices, + Pow: nipost.Post.Pow, } + + v1.PostMeta = &spacemeshv2alpha1.PostMeta{ + Challenge: nipost.PostMetadata.Challenge, + LabelsPerUnit: nipost.PostMetadata.LabelsPerUnit, + } + + v1.Membership = &spacemeshv2alpha1.PoetMembershipProof{ + ProofNodes: make([][]byte, len(nipost.Membership.Nodes)), + Leaf: nipost.Membership.LeafIndex, + } + + for i, node := range nipost.Membership.Nodes { + v1.Membership.ProofNodes[i] = node.Bytes() + } + return v1 } -func NewActivationService(db *sql.Database) *ActivationService { +func NewActivationService(db sql.Executor) *ActivationService { return &ActivationService{db: db} } type ActivationService struct { - db *sql.Database + db sql.Executor } -var _ grpcserver.ServiceAPI = (*ActivationService)(nil) - func (s *ActivationService) RegisterService(server *grpc.Server) { spacemeshv2alpha1.RegisterActivationServiceServer(server, s) } @@ -214,7 +222,7 @@ func (s *ActivationService) ActivationsCount( }, }} - count, err := atxs.CountAtxsByEpoch(s.db, ops) + count, err := atxs.CountAtxsByOps(s.db, ops) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -277,20 +285,20 @@ func toOperations(filter *spacemeshv2alpha1.ActivationRequest) (builder.Operatio }) } - ops.Other = append(ops.Other, builder.Op{ - Field: builder.OrderBy, + ops.Modifiers = append(ops.Modifiers, builder.Modifier{ + Key: builder.OrderBy, Value: "epoch asc, id", }) if filter.Limit != 0 { - ops.Other = append(ops.Other, builder.Op{ - Field: builder.Limit, + ops.Modifiers = append(ops.Modifiers, builder.Modifier{ + Key: builder.Limit, Value: int64(filter.Limit), }) } if filter.Offset != 0 { - ops.Other = append(ops.Other, builder.Op{ - Field: builder.Offset, + ops.Modifiers = append(ops.Modifiers, builder.Modifier{ + Key: builder.Offset, Value: int64(filter.Offset), }) } diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go index e248bba390..c0719f1039 100644 --- a/api/grpcserver/v2alpha1/activation_test.go +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -3,15 +3,13 @@ package v2alpha1 import ( "context" "errors" - "io" - "testing" - "time" - spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "io" + "testing" "github.com/spacemeshos/go-spacemesh/common/fixture" "github.com/spacemeshos/go-spacemesh/common/types" @@ -22,9 +20,7 @@ import ( func TestActivationService_List(t *testing.T) { db := sql.InMemory() - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := context.Background() gen := fixture.NewAtxsGenerator() activations := make([]types.VerifiedActivationTx, 100) @@ -106,20 +102,15 @@ func TestActivationService_List(t *testing.T) { func TestActivationStreamService_Stream(t *testing.T) { db := sql.InMemory() - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := context.Background() gen := fixture.NewAtxsGenerator() activations := make([]types.VerifiedActivationTx, 100) - require.NoError(t, db.WithTx(ctx, func(dtx *sql.Tx) error { - for i := range activations { - atx := gen.Next() - require.NoError(t, atxs.Add(dtx, atx)) - activations[i] = *atx - } - return nil - })) + for i := range activations { + atx := gen.Next() + require.NoError(t, atxs.Add(db, atx)) + activations[i] = *atx + } svc := NewActivationStreamService(db) cfg, cleanup := launchServer(t, svc) @@ -143,7 +134,7 @@ func TestActivationStreamService_Stream(t *testing.T) { } i++ } - require.Equal(t, len(activations), i) + require.Len(t, activations, i) }) t.Run("watch", func(t *testing.T) { @@ -192,9 +183,6 @@ func TestActivationStreamService_Stream(t *testing.T) { } { tc := tc t.Run(tc.desc, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - stream, err := client.Stream(ctx, tc.request) require.NoError(t, err) _, err = stream.Header() @@ -221,20 +209,15 @@ func TestActivationStreamService_Stream(t *testing.T) { func TestActivationService_ActivationsCount(t *testing.T) { db := sql.InMemory() - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := context.Background() gen := fixture.NewAtxsGenerator().WithEpochs(0, 1) activations := make([]types.VerifiedActivationTx, 30) - require.NoError(t, db.WithTx(ctx, func(dtx *sql.Tx) error { - for i := range activations { - atx := gen.Next() - require.NoError(t, atxs.Add(dtx, atx)) - activations[i] = *atx - } - return nil - })) + for i := range activations { + atx := gen.Next() + require.NoError(t, atxs.Add(db, atx)) + activations[i] = *atx + } svc := NewActivationService(db) cfg, cleanup := launchServer(t, svc) @@ -247,5 +230,5 @@ func TestActivationService_ActivationsCount(t *testing.T) { Epoch: activations[3].PublishEpoch.Uint32(), }) require.NoError(t, err) - require.Equal(t, len(activations), int(count.Count)) + require.Len(t, activations, int(count.Count)) } diff --git a/common/fixture/atxs.go b/common/fixture/atxs.go index efc7d57913..5630918d29 100644 --- a/common/fixture/atxs.go +++ b/common/fixture/atxs.go @@ -1,9 +1,9 @@ package fixture import ( - "log" + "github.com/spacemeshos/merkle-tree" + "github.com/spacemeshos/poet/shared" "math/rand" - "os" "time" "github.com/spacemeshos/go-spacemesh/common/types" @@ -36,12 +36,45 @@ func (g *AtxsGenerator) WithSeed(seed int64) *AtxsGenerator { // WithEpochs update epochs ids. func (g *AtxsGenerator) WithEpochs(start, n int) *AtxsGenerator { g.Epochs = nil - for i := 1; i <= n; i++ { + for i := 0; i < n; i++ { g.Epochs = append(g.Epochs, types.EpochID(start+i)) } return g } +func (g *AtxsGenerator) newNIPost() *types.NIPost { + challenge := types.HexToHash32("55555") + poetRef := []byte("66666") + tree, err := merkle.NewTreeBuilder(). + WithHashFunc(shared.HashMembershipTreeNode). + WithLeavesToProve(map[uint64]bool{0: true}). + Build() + if err != nil { + panic("failed to add leaf to tree") + } + if err := tree.AddLeaf(challenge[:]); err != nil { + panic("failed to add leaf to tree") + } + nodes := tree.Proof() + nodesH32 := make([]types.Hash32, 0, len(nodes)) + for _, n := range nodes { + nodesH32 = append(nodesH32, types.BytesToHash(n)) + } + return &types.NIPost{ + Membership: types.MerkleProof{ + Nodes: nodesH32, + }, + Post: &types.Post{ + Nonce: 0, + Indices: []byte(nil), + }, + PostMetadata: &types.PostMetadata{ + Challenge: poetRef, + LabelsPerUnit: 2048, + }, + } +} + // Next generates VerifiedActivationTx. func (g *AtxsGenerator) Next() *types.VerifiedActivationTx { var atx types.VerifiedActivationTx @@ -55,8 +88,7 @@ func (g *AtxsGenerator) Next() *types.VerifiedActivationTx { signer, err := signing.NewEdSigner() if err != nil { - log.Println("failed to create signer:", err) - os.Exit(1) + panic("failed to create signer") } atx = types.VerifiedActivationTx{ @@ -71,6 +103,7 @@ func (g *AtxsGenerator) Next() *types.VerifiedActivationTx { Coinbase: wallet.Address(signer.PublicKey().Bytes()), NumUnits: g.rng.Uint32(), NodeID: &nodeId, + NIPost: g.newNIPost(), }, SmesherID: nodeId, }, diff --git a/node/node.go b/node/node.go index a410dd1f88..a468fedf62 100644 --- a/node/node.go +++ b/node/node.go @@ -1361,10 +1361,8 @@ func (app *App) grpcService(svc grpcserver.Service, lg log.Log) (grpcserver.Serv app.grpcServices[svc] = service return service, nil case grpcserver.Activation: - return grpcserver.NewActivationService( - app.cachedDB, - types.ATXID(app.Config.Genesis.GoldenATX()), - ), nil + service := grpcserver.NewActivationService(app.cachedDB, types.ATXID(app.Config.Genesis.GoldenATX())) + return service, nil case v2alpha1.Activation: return v2alpha1.NewActivationService(app.db), nil case v2alpha1.ActivationStream: diff --git a/sql/atxs/atxs.go b/sql/atxs/atxs.go index 97ab78ddee..adb17ade93 100644 --- a/sql/atxs/atxs.go +++ b/sql/atxs/atxs.go @@ -528,7 +528,7 @@ func IterateAtxsOps( return derr } -func CountAtxsByEpoch(db sql.Executor, operations builder.Operations) (count uint32, err error) { +func CountAtxsByOps(db sql.Executor, operations builder.Operations) (count uint32, err error) { _, err = db.Exec( "SELECT count(*) FROM atxs"+builder.FilterFrom(operations), builder.BindingsFrom(operations), diff --git a/sql/builder/builder.go b/sql/builder/builder.go index ed831f9d07..e1f1303afd 100644 --- a/sql/builder/builder.go +++ b/sql/builder/builder.go @@ -17,8 +17,6 @@ const ( Gte token = ">=" Lt token = "<" Lte token = "<=" - And token = "and" - Where token = "where" ) type field string @@ -28,9 +26,14 @@ const ( Smesher field = "pubkey" Coinbase field = "coinbase" Id field = "id" - Offset field = "offset" - Limit field = "limit" - OrderBy field = "order by" +) + +type modifier string + +const ( + Offset modifier = "offset" + Limit modifier = "limit" + OrderBy modifier = "order by" ) type Op struct { @@ -41,9 +44,16 @@ type Op struct { Value any } +type Modifier struct { + Key modifier + // Value will be type casted to one the expected types. + // Modifier will panic if it doesn't match any of expected. + Value any +} + type Operations struct { - Filter []Op - Other []Op + Filter []Op + Modifiers []Modifier } func FilterFrom(operations Operations) string { @@ -58,8 +68,8 @@ func FilterFrom(operations Operations) string { queryBuilder.WriteString(" " + string(op.Field) + " " + string(op.Token) + " ?" + strconv.Itoa(i+1)) } - for _, op := range operations.Other { - queryBuilder.WriteString(fmt.Sprintf(" %s %v", string(op.Field), op.Value)) + for _, m := range operations.Modifiers { + queryBuilder.WriteString(fmt.Sprintf(" %s %v", string(m.Key), m.Value)) } return queryBuilder.String() From 7635482aa01d68296fabaf924d40e65262b56dbd Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 16 Feb 2024 13:57:27 +0100 Subject: [PATCH 24/25] lint --- api/grpcserver/v2alpha1/activation_test.go | 5 +++-- common/fixture/atxs.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/grpcserver/v2alpha1/activation_test.go b/api/grpcserver/v2alpha1/activation_test.go index c0719f1039..bb9d89cff0 100644 --- a/api/grpcserver/v2alpha1/activation_test.go +++ b/api/grpcserver/v2alpha1/activation_test.go @@ -3,13 +3,14 @@ package v2alpha1 import ( "context" "errors" + "io" + "testing" + spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "io" - "testing" "github.com/spacemeshos/go-spacemesh/common/fixture" "github.com/spacemeshos/go-spacemesh/common/types" diff --git a/common/fixture/atxs.go b/common/fixture/atxs.go index 5630918d29..727428d5f4 100644 --- a/common/fixture/atxs.go +++ b/common/fixture/atxs.go @@ -1,11 +1,12 @@ package fixture import ( - "github.com/spacemeshos/merkle-tree" - "github.com/spacemeshos/poet/shared" "math/rand" "time" + "github.com/spacemeshos/merkle-tree" + "github.com/spacemeshos/poet/shared" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" "github.com/spacemeshos/go-spacemesh/signing" From 1f80fa98d82ec46787fd977aedce7e1f2df502ae Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 16 Feb 2024 15:22:41 +0100 Subject: [PATCH 25/25] apply review suggestion --- node/node.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/node/node.go b/node/node.go index a468fedf62..7e28f77a4f 100644 --- a/node/node.go +++ b/node/node.go @@ -1362,11 +1362,16 @@ func (app *App) grpcService(svc grpcserver.Service, lg log.Log) (grpcserver.Serv return service, nil case grpcserver.Activation: service := grpcserver.NewActivationService(app.cachedDB, types.ATXID(app.Config.Genesis.GoldenATX())) + app.grpcServices[svc] = service return service, nil case v2alpha1.Activation: - return v2alpha1.NewActivationService(app.db), nil + service := v2alpha1.NewActivationService(app.db) + app.grpcServices[svc] = service + return service, nil case v2alpha1.ActivationStream: - return v2alpha1.NewActivationStreamService(app.db), nil + service := v2alpha1.NewActivationStreamService(app.db) + app.grpcServices[svc] = service + return service, nil } return nil, fmt.Errorf("unknown service %s", svc) }