diff --git a/metric/ingestor.go b/metric/ingestor.go index 7813d0933..6cc7eeddb 100644 --- a/metric/ingestor.go +++ b/metric/ingestor.go @@ -30,6 +30,12 @@ var ( Name: "errors_total", Help: "Number of search requests completed with error", }) + SearchResults = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "seq_db_ingestor", + Subsystem: "search", + Name: "results_total", + Help: "Number of search requests by result and storage tier", + }, []string{"result", "tier"}) IngestorPanics = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "seq_db_ingestor", Subsystem: "common", diff --git a/proxy/search/docs_iterator.go b/proxy/search/docs_iterator.go index cc032b7fc..25953a730 100644 --- a/proxy/search/docs_iterator.go +++ b/proxy/search/docs_iterator.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "iter" "time" "go.uber.org/zap" @@ -170,3 +171,47 @@ func (e *explainWrapperIterator) Next() (StreamingDoc, error) { } return d, nil } + +type DocsIteratorWithEvents struct { + it DocsIterator + start bool + finish bool + onStart func() + onFinish func() +} + +func SetDocsIteratorEvents(it DocsIterator, onStart, onFinish func()) DocsIterator { + return &DocsIteratorWithEvents{ + it: it, + onStart: onStart, + onFinish: onFinish, + } +} + +func (e *DocsIteratorWithEvents) Next() (StreamingDoc, error) { + if !e.start { + e.onStart() + e.start = true + } + doc, err := e.it.Next() + if err != nil && !e.finish { + e.onFinish() + e.finish = true + } + return doc, err +} + +func DocsIteratorSeq(it DocsIterator) iter.Seq2[StreamingDoc, error] { + return func(yield func(StreamingDoc, error) bool) { + doc, err := it.Next() + for ; err == nil; doc, err = it.Next() { + if !yield(doc, nil) { + return + } + } + if err != io.EOF { + var empty StreamingDoc + yield(empty, err) + } + } +} diff --git a/proxy/search/ingestor.go b/proxy/search/ingestor.go index bc81220f8..80a583ab0 100644 --- a/proxy/search/ingestor.go +++ b/proxy/search/ingestor.go @@ -35,6 +35,29 @@ type Config struct { MirrorAddr string } +// StorageTier shows whether a search was served by hot or cold stores. +type StorageTier string + +const ( + StorageTierHot StorageTier = "hot" + StorageTierCold StorageTier = "cold" + // StorageTierNone is used when a request never reached the ingestor + // (e.g. it failed validation in the proxy layer). + StorageTierNone StorageTier = "none" +) + +// SearchStats carries observability data collected while executing a search. +type SearchStats struct { + Size int + HasHist bool + HasAgg bool + + StorageTier StorageTier + HotSearchDuration time.Duration + ColdSearchDuration time.Duration + TotalSearchDuration time.Duration +} + type Ingestor struct { config Config clients map[string]storeapi.StoreApiClient @@ -67,7 +90,7 @@ func (si *Ingestor) Search( ) ( qpr *seq.QPR, docsStream DocsIterator, - overallDuration time.Duration, + stats *SearchStats, err error, ) { if sr.Explain { @@ -77,8 +100,16 @@ func (si *Ingestor) Search( zap.String("to", sr.To.String()), ) } + + stats = &SearchStats{ + Size: sr.Size + sr.Offset, + HasHist: sr.Interval > 0, + HasAgg: len(sr.AggQ) > 0, + StorageTier: StorageTierNone, + } + if sr.Size < 0 || sr.Offset < 0 { - return nil, nil, 0, fmt.Errorf("%w: negative size or offset", consts.ErrInvalidArgument) + return nil, nil, stats, fmt.Errorf("%w: negative size or offset", consts.ErrInvalidArgument) } startTime := time.Now() @@ -87,6 +118,10 @@ func (si *Ingestor) Search( searchStores = si.config.HotReadStores } qprs, err := si.searchStores(ctx, sr, searchStores, tr) + + stats.StorageTier = StorageTierHot + stats.HotSearchDuration = time.Since(startTime) + var partialRespErr error if err != nil { @@ -94,24 +129,29 @@ func (si *Ingestor) Search( case errors.Is(err, consts.ErrIngestorQueryWantsOldData): if len(si.config.ReadStores.Shards) == 0 { logger.Error("no cold stores, but hot mode is enabled, bad configuration of stores!") - return nil, nil, 0, err + return nil, nil, stats, err } metric.SearchColdTotal.Inc() + + coldStart := time.Now() qprs, err = si.searchStores(ctx, sr, si.config.ReadStores, tr) + stats.StorageTier = StorageTierCold + stats.ColdSearchDuration = time.Since(coldStart) + if err != nil { metric.SearchColdErrors.Add(1) if errors.Is(err, consts.ErrPartialResponse) { partialRespErr = err // consider partial response from cold stores as a result } else { // errors from both hot and cold stores, return error - return nil, nil, 0, err + return nil, nil, stats, err } } case errors.Is(err, consts.ErrPartialResponse): partialRespErr = err // consider partial response from hot stores as a result default: // unexpected error on all hot replica sets (usually bad query) - return nil, nil, 0, err + return nil, nil, stats, err } } @@ -143,19 +183,19 @@ func (si *Ingestor) Search( docsStream = EmptyDocsStream{} if sr.ShouldFetch && size > 0 { if util.IsCancelled(ctx) { - return nil, nil, 0, ctx.Err() + return nil, nil, stats, ctx.Err() } metric.DocumentsRequested.Observe(float64(len(ids))) fieldsFilter := tryParseFieldsFilter(string(sr.Q)) docsStream, err = si.FetchDocsStream(ctx, ids, sr.Explain, true, fieldsFilter) if err != nil { - return nil, nil, 0, err + return nil, nil, stats, err } } fetchDuration := time.Since(t) - overallDuration = time.Since(startTime) + stats.TotalSearchDuration = time.Since(startTime) if sr.Explain { logger.Info("data", zap.Any("histogram", qpr.Histogram)) @@ -166,11 +206,11 @@ func (si *Ingestor) Search( util.ZapDurationWithPrec("query_ms", queryDuration, "ms", 2), util.ZapDurationWithPrec("merge_ms", mergeDuration, "ms", 2), util.ZapDurationWithPrec("fetch_ms", fetchDuration, "ms", 2), - util.ZapDurationWithPrec("all_ms", overallDuration, "ms", 2), + util.ZapDurationWithPrec("all_ms", stats.TotalSearchDuration, "ms", 2), ) } - return qpr, docsStream, overallDuration, partialRespErr + return qpr, docsStream, stats, partialRespErr } // tryParseFieldsFilter tries to parse seq-ql query to extract fields/remove pipe. diff --git a/proxy/search/merged_docs_iterator.go b/proxy/search/merged_docs_iterator.go index 03d8cd750..80fc2d737 100644 --- a/proxy/search/merged_docs_iterator.go +++ b/proxy/search/merged_docs_iterator.go @@ -86,19 +86,22 @@ func (m *mergedDocStream) Next() (StreamingDoc, error) { } func newNMergedStreams(streams []DocsIterator, less func(seq.IDSource, seq.IDSource) bool) DocsIterator { - if len(streams) == 0 { + l := len(streams) + if l == 0 { return EmptyDocsStream{} } - - if len(streams) == 1 { + if l == 1 { return newMergedDocsStream(streams[0], EmptyDocsStream{}, less) } - - merged := newMergedDocsStream(streams[0], streams[1], less) - for _, s := range streams[2:] { - merged = newMergedDocsStream(merged, s, less) + if l == 2 { + return newMergedDocsStream(streams[0], streams[1], less) } - return merged + + half := l / 2 + a := newNMergedStreams(streams[:half], less) + b := newNMergedStreams(streams[half:], less) + + return newMergedDocsStream(a, b, less) } type mergedStreamIterator struct { diff --git a/proxyapi/grpc_async_search.go b/proxyapi/grpc_async_search.go index b8c622af6..6f5e530f9 100644 --- a/proxyapi/grpc_async_search.go +++ b/proxyapi/grpc_async_search.go @@ -225,7 +225,10 @@ func (g *grpcV1) ExportAsyncSearch(req *seqproxyapi.ExportAsyncSearchRequest, st asyncsearcher.ExportSize.WithLabelValues(protocol).Observe(float64(wrapped.size)) }() - for doc, err := docsStream.Next(); err == nil; doc, err = docsStream.Next() { + for doc, err := range search.DocsIteratorSeq(docsStream) { + if err != nil { + return status.Errorf(codes.Internal, "docs reading error: %v", err) + } eResp := &seqproxyapi.ExportResponse{ Doc: &seqproxyapi.Document{ Id: doc.ID.String(), diff --git a/proxyapi/grpc_complex_search.go b/proxyapi/grpc_complex_search.go index 7dbb6fe71..e72ccff17 100644 --- a/proxyapi/grpc_complex_search.go +++ b/proxyapi/grpc_complex_search.go @@ -13,7 +13,7 @@ import ( func (g *grpcV1) ComplexSearch( ctx context.Context, req *seqproxyapi.ComplexSearchRequest, -) (*seqproxyapi.ComplexSearchResponse, error) { +) (_ *seqproxyapi.ComplexSearchResponse, retErr error) { ctx, cancel := context.WithTimeout(ctx, g.config.SearchTimeout) defer cancel() @@ -22,7 +22,8 @@ func (g *grpcV1) ComplexSearch( } tr := querytracer.New(req.Query.Explain, "proxy/ComplexSearch") - sResp, err := g.doSearch(ctx, req, true, true, tr) + sResp, obs, err := g.doSearch(ctx, req, true, true, tr) + defer func() { obs.finish("ComplexSearch", retErr) }() if err != nil { return nil, err } diff --git a/proxyapi/grpc_complex_search_test.go b/proxyapi/grpc_complex_search_test.go index 43c1eec1a..fcd184efe 100644 --- a/proxyapi/grpc_complex_search_test.go +++ b/proxyapi/grpc_complex_search_test.go @@ -153,10 +153,10 @@ func prepareComplexSearchTestData(t *testing.T, cData cSearchTestCaseData) cSear siSearchMock = &siSearchMockData{ sr: sr, ret: siSearchRet{ - qpr: qpr, - docs: docs, - took: time.Second, - err: cData.siErr, + qpr: qpr, + docs: docs, + stats: &search.SearchStats{}, + err: cData.siErr, }, } } diff --git a/proxyapi/grpc_export.go b/proxyapi/grpc_export.go index 9582ac3a6..d8a79f2b1 100644 --- a/proxyapi/grpc_export.go +++ b/proxyapi/grpc_export.go @@ -12,6 +12,7 @@ import ( "github.com/ozontech/seq-db/config" "github.com/ozontech/seq-db/metric" "github.com/ozontech/seq-db/pkg/seqproxyapi/v1" + "github.com/ozontech/seq-db/proxy/search" ) type metricStream struct { @@ -24,7 +25,7 @@ func (s *metricStream) Send(resp *seqproxyapi.ExportResponse) error { return s.SeqProxyApi_ExportServer.Send(resp) } -func (g *grpcV1) Export(req *seqproxyapi.ExportRequest, stream seqproxyapi.SeqProxyApi_ExportServer) error { +func (g *grpcV1) Export(req *seqproxyapi.ExportRequest, stream seqproxyapi.SeqProxyApi_ExportServer) (retErr error) { ctx, cancel := context.WithTimeout(stream.Context(), g.config.ExportTimeout) defer cancel() @@ -47,7 +48,8 @@ func (g *grpcV1) Export(req *seqproxyapi.ExportRequest, stream seqproxyapi.SeqPr Offset: req.Offset, WithTotal: false, } - sResp, err := g.doSearch(ctx, proxyReq, true, true, nil) + sResp, obs, err := g.doSearch(ctx, proxyReq, true, true, nil) + defer func() { obs.finish("Export", retErr) }() if err != nil { return err } @@ -63,7 +65,10 @@ func (g *grpcV1) Export(req *seqproxyapi.ExportRequest, stream seqproxyapi.SeqPr metric.ExportSize.WithLabelValues(protocol).Observe(float64(wrapped.size)) }() - for doc, err := sResp.docsStream.Next(); err == nil; doc, err = sResp.docsStream.Next() { + for doc, err := range search.DocsIteratorSeq(sResp.docsStream) { + if err != nil { + return status.Errorf(codes.Internal, "docs reading error: %v", err) + } eResp := &seqproxyapi.ExportResponse{ Doc: &seqproxyapi.Document{ Id: doc.ID.String(), diff --git a/proxyapi/grpc_export_test.go b/proxyapi/grpc_export_test.go index 86b95a15f..c13adea32 100644 --- a/proxyapi/grpc_export_test.go +++ b/proxyapi/grpc_export_test.go @@ -81,10 +81,10 @@ func prepareExportTestData(cData exportTestCaseData) exportTestData { ShouldFetch: true, }, ret: siSearchRet{ - qpr: qpr, - docs: docs, - took: time.Second, - err: cData.siErr, + qpr: qpr, + docs: docs, + stats: &search.SearchStats{}, + err: cData.siErr, }, } } diff --git a/proxyapi/grpc_fetch.go b/proxyapi/grpc_fetch.go index fcd0c8f97..3b8f50d95 100644 --- a/proxyapi/grpc_fetch.go +++ b/proxyapi/grpc_fetch.go @@ -60,7 +60,10 @@ func (g *grpcV1) Fetch(req *seqproxyapi.FetchRequest, stream seqproxyapi.SeqProx if err != nil { return status.Errorf(codes.Internal, "can't fetch: %v", err) } - for doc, err := docsStream.Next(); err == nil; doc, err = docsStream.Next() { + for doc, err := range search.DocsIteratorSeq(docsStream) { + if err != nil { + return status.Errorf(codes.Internal, "docs reading error: %v", err) + } err := stream.Send(&seqproxyapi.Document{ Id: doc.ID.String(), Data: doc.Data, diff --git a/proxyapi/grpc_get_aggregation.go b/proxyapi/grpc_get_aggregation.go index c43c8b398..ea3863995 100644 --- a/proxyapi/grpc_get_aggregation.go +++ b/proxyapi/grpc_get_aggregation.go @@ -11,7 +11,7 @@ import ( func (g *grpcV1) GetAggregation( ctx context.Context, req *seqproxyapi.GetAggregationRequest, -) (*seqproxyapi.GetAggregationResponse, error) { +) (_ *seqproxyapi.GetAggregationResponse, retErr error) { ctx, cancel := context.WithTimeout(ctx, g.config.SearchTimeout) defer cancel() @@ -24,7 +24,8 @@ func (g *grpcV1) GetAggregation( Aggs: req.Aggs, } - sResp, err := g.doSearch(ctx, proxyReq, false, true, nil) + sResp, obs, err := g.doSearch(ctx, proxyReq, false, true, nil) + defer func() { obs.finish("GetAggregation", retErr) }() if err != nil { return nil, err } diff --git a/proxyapi/grpc_get_aggregation_test.go b/proxyapi/grpc_get_aggregation_test.go index 251d25a3c..8176a72a0 100644 --- a/proxyapi/grpc_get_aggregation_test.go +++ b/proxyapi/grpc_get_aggregation_test.go @@ -105,9 +105,9 @@ func prepareGetAggregationTestData(t *testing.T, cData getAggregationTestCaseDat siSearchMock = &siSearchMockData{ sr: sr, ret: siSearchRet{ - qpr: qpr, - took: time.Second, - err: cData.siErr, + qpr: qpr, + stats: &search.SearchStats{}, + err: cData.siErr, }, } } diff --git a/proxyapi/grpc_get_histogram.go b/proxyapi/grpc_get_histogram.go index 0c80152af..e3f6aca6a 100644 --- a/proxyapi/grpc_get_histogram.go +++ b/proxyapi/grpc_get_histogram.go @@ -11,7 +11,7 @@ import ( func (g *grpcV1) GetHistogram( ctx context.Context, req *seqproxyapi.GetHistogramRequest, -) (*seqproxyapi.GetHistogramResponse, error) { +) (_ *seqproxyapi.GetHistogramResponse, retErr error) { ctx, cancel := context.WithTimeout(ctx, g.config.SearchTimeout) defer cancel() @@ -23,7 +23,8 @@ func (g *grpcV1) GetHistogram( Query: req.Query, Hist: req.Hist, } - sResp, err := g.doSearch(ctx, proxyReq, false, true, nil) + sResp, obs, err := g.doSearch(ctx, proxyReq, false, true, nil) + defer func() { obs.finish("GetHistogram", retErr) }() if err != nil { return nil, err } diff --git a/proxyapi/grpc_get_histogram_test.go b/proxyapi/grpc_get_histogram_test.go index f498d4bab..be311e8b2 100644 --- a/proxyapi/grpc_get_histogram_test.go +++ b/proxyapi/grpc_get_histogram_test.go @@ -105,9 +105,9 @@ func prepareGetHistogramTestData(t *testing.T, cData getHistogramTestCaseData) g siSearchMock = &siSearchMockData{ sr: sr, ret: siSearchRet{ - qpr: qpr, - took: time.Second, - err: cData.siErr, + qpr: qpr, + stats: &search.SearchStats{}, + err: cData.siErr, }, } } diff --git a/proxyapi/grpc_main_test.go b/proxyapi/grpc_main_test.go index 4bd9caa22..3f60ec6f6 100644 --- a/proxyapi/grpc_main_test.go +++ b/proxyapi/grpc_main_test.go @@ -45,10 +45,10 @@ type testAggQuery struct { } type siSearchRet struct { - qpr *seq.QPR - docs search.DocsIterator - took time.Duration - err error + qpr *seq.QPR + docs search.DocsIterator + stats *search.SearchStats + err error } type siSearchMockData struct { @@ -161,7 +161,7 @@ func prepareMock(m *mocks, mData *mocksData) { ret := mData.si.search.ret m.siMock.EXPECT().Search( gomock.Any(), mData.si.search.sr, gomock.Any(), - ).Return(ret.qpr, ret.docs, ret.took, ret.err) + ).Return(ret.qpr, ret.docs, ret.stats, ret.err) } if mData.si.documents != nil { ret := mData.si.documents.ret diff --git a/proxyapi/grpc_search.go b/proxyapi/grpc_search.go index 0c2f4af8a..72d83b584 100644 --- a/proxyapi/grpc_search.go +++ b/proxyapi/grpc_search.go @@ -11,7 +11,7 @@ import ( func (g *grpcV1) Search( ctx context.Context, req *seqproxyapi.SearchRequest, -) (*seqproxyapi.SearchResponse, error) { +) (_ *seqproxyapi.SearchResponse, retErr error) { ctx, cancel := context.WithTimeout(ctx, g.config.SearchTimeout) defer cancel() @@ -27,7 +27,8 @@ func (g *grpcV1) Search( WithTotal: req.WithTotal, Order: req.Order, } - sResp, err := g.doSearch(ctx, proxyReq, true, true, nil) + sResp, obs, err := g.doSearch(ctx, proxyReq, true, true, nil) + defer func() { obs.finish("Search", retErr) }() if err != nil { return nil, err } diff --git a/proxyapi/grpc_search_test.go b/proxyapi/grpc_search_test.go index 0f99932a7..2e1482792 100644 --- a/proxyapi/grpc_search_test.go +++ b/proxyapi/grpc_search_test.go @@ -96,10 +96,10 @@ func prepareSearchTestData(t *testing.T, cData searchTestCaseData) searchTestDat siSearchMock = &siSearchMockData{ sr: sr, ret: siSearchRet{ - qpr: qpr, - docs: docs, - took: time.Second, - err: cData.siErr, + qpr: qpr, + docs: docs, + stats: &search.SearchStats{}, + err: cData.siErr, }, } } diff --git a/proxyapi/grpc_stream_search.go b/proxyapi/grpc_stream_search.go index 325a797c0..738ac39d1 100644 --- a/proxyapi/grpc_stream_search.go +++ b/proxyapi/grpc_stream_search.go @@ -31,7 +31,7 @@ const ( outcomeCancel // client canceled or disconnected ) -func (g *grpcV1) StreamSearch(stream seqproxyapi.SeqProxyApi_StreamSearchServer) error { +func (g *grpcV1) StreamSearch(stream seqproxyapi.SeqProxyApi_StreamSearchServer) (retErr error) { ctx, cancel := context.WithCancel(stream.Context()) defer cancel() @@ -60,7 +60,8 @@ func (g *grpcV1) StreamSearch(stream seqproxyapi.SeqProxyApi_StreamSearchServer) } tr := querytracer.New(q.Explain, "proxy/StreamSearch") - sResp, err := g.doSearch(ctx, proxyReq, true, false, tr) + sResp, obs, err := g.doSearch(ctx, proxyReq, true, false, tr) + defer func() { obs.finish("StreamSearch", retErr) }() if err != nil { return err } @@ -173,7 +174,10 @@ func (g *grpcV1) streamSearchDocs( } var batch []*seqproxyapi.Record - for doc, err := sResp.docsStream.Next(); err == nil; doc, err = sResp.docsStream.Next() { + for doc, err := range search.DocsIteratorSeq(sResp.docsStream) { + if err != nil { + return outcomeNone, err + } batch = append(batch, docToRecord(doc)) if len(batch) >= streamSearchBatchSize { if err := sendRecords(stream, batch); err != nil { diff --git a/proxyapi/grpc_v1.go b/proxyapi/grpc_v1.go index cb54064a2..6679e8d4a 100644 --- a/proxyapi/grpc_v1.go +++ b/proxyapi/grpc_v1.go @@ -31,7 +31,7 @@ import ( ) type SearchIngestor interface { - Search(ctx context.Context, sr *search.SearchRequest, tr *querytracer.Tracer) (*seq.QPR, search.DocsIterator, time.Duration, error) + Search(ctx context.Context, sr *search.SearchRequest, tr *querytracer.Tracer) (*seq.QPR, search.DocsIterator, *search.SearchStats, error) Documents(ctx context.Context, r search.FetchRequest) (search.DocsIterator, error) Status(ctx context.Context) *search.IngestorStatus StartAsyncSearch(context.Context, search.AsyncRequest) (search.AsyncResponse, error) @@ -190,35 +190,40 @@ func (g *grpcV1) doSearch( shouldFetch bool, shouldValidateStreamPipes bool, tr *querytracer.Tracer, -) (*proxySearchResponse, error) { +) (*proxySearchResponse, requestObservation, error) { metric.SearchOverall.Add(1) + obs := requestObservation{ + start: time.Now(), + stats: &search.SearchStats{}, + } + span := trace.FromContext(ctx) defer span.End() if req.Query == nil { - return nil, status.Error(codes.InvalidArgument, "search query must be provided") + return nil, obs, status.Error(codes.InvalidArgument, "search query must be provided") } if req.Query.From == nil || req.Query.To == nil { - return nil, status.Error(codes.InvalidArgument, `search query "from" and "to" fields must be provided`) + return nil, obs, status.Error(codes.InvalidArgument, `search query "from" and "to" fields must be provided`) } if req.Offset != 0 && req.OffsetId != "" { - return nil, status.Error(codes.InvalidArgument, `only one of "offset" and "offset_id" must be provided`) + return nil, obs, status.Error(codes.InvalidArgument, `only one of "offset" and "offset_id" must be provided`) } fromTime := req.Query.From.AsTime() toTime := req.Query.To.AsTime() if fromTime.After(toTime) { - return nil, status.Error(codes.InvalidArgument, `"from" timestamp must not be after "to" timestamp`) + return nil, obs, status.Error(codes.InvalidArgument, `"from" timestamp must not be after "to" timestamp`) } if shouldValidateStreamPipes { ast, err := parser.ParseSeqQL(req.Query.Query, nil) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("search query must be valid: %s", err)) + return nil, obs, status.Error(codes.InvalidArgument, fmt.Sprintf("search query must be valid: %s", err)) } if err := ast.ValidateStreamPipes(); err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("search query must be valid: %s", err)) + return nil, obs, status.Error(codes.InvalidArgument, fmt.Sprintf("search query must be valid: %s", err)) } } @@ -249,7 +254,7 @@ func (g *grpcV1) doSearch( rlQuery := getSearchQueryFromGRPCReqForRateLimiter(req) if !g.rateLimiter.Account(rlQuery) { - return nil, status.Error(codes.ResourceExhausted, consts.ErrRequestWasRateLimited.Error()) + return nil, obs, status.Error(codes.ResourceExhausted, consts.ErrRequestWasRateLimited.Error()) } proxyReq := &search.SearchRequest{ @@ -269,7 +274,7 @@ func (g *grpcV1) doSearch( if len(req.Aggs) > 0 { aggs, err := convertAggsQuery(req.Aggs) if err != nil { - return nil, err + return nil, obs, err } proxyReq.AggQ = aggs } @@ -277,7 +282,7 @@ func (g *grpcV1) doSearch( if req.Hist != nil { intervalDuration, err := util.ParseDuration(req.Hist.Interval) if err != nil { - return nil, status.Errorf( + return nil, obs, status.Errorf( codes.InvalidArgument, "failed to parse 'interval': %v", err, @@ -286,25 +291,31 @@ func (g *grpcV1) doSearch( proxyReq.Interval = seq.MID(intervalDuration.Nanoseconds()) } - qpr, docsStream, _, err := g.searchIngestor.Search(ctx, proxyReq, tr) + qpr, docsStream, stats, err := g.searchIngestor.Search(ctx, proxyReq, tr) psr := &proxySearchResponse{ - qpr: qpr, - docsStream: docsStream, + qpr: qpr, + docsStream: search.SetDocsIteratorEvents( + docsStream, + func() { obs.fetchStart = time.Now() }, + func() { obs.takeFetchDuration() }, + ), } + obs.stats = stats + obs.rawErr = err if e, ok := parseProxyError(err); ok { psr.err = e - return psr, nil + return psr, obs, nil } if errors.Is(err, consts.ErrInvalidArgument) { - return nil, status.Error(codes.InvalidArgument, err.Error()) + return nil, obs, status.Error(codes.InvalidArgument, err.Error()) } if st, ok := status.FromError(err); ok { // could not parse a query if st.Code() == codes.InvalidArgument { - return nil, err + return nil, obs, err } } @@ -314,16 +325,16 @@ func (g *grpcV1) doSearch( Code: seqproxyapi.ErrorCode_ERROR_CODE_PARTIAL_RESPONSE, Message: err.Error(), } - return psr, nil + return psr, obs, nil } if err = processSearchErrors(qpr, err); err != nil { metric.SearchErrors.Inc() - return nil, err + return nil, obs, err } g.tryMirrorRequest(req) - return psr, nil + return psr, obs, nil } func convertAggsQuery(aggs []*seqproxyapi.AggQuery) ([]search.AggQuery, error) { diff --git a/proxyapi/mock/grpc_v1.go b/proxyapi/mock/grpc_v1.go index c178291fa..a9f9ad720 100644 --- a/proxyapi/mock/grpc_v1.go +++ b/proxyapi/mock/grpc_v1.go @@ -7,7 +7,6 @@ package mock import ( context "context" reflect "reflect" - time "time" gomock "github.com/golang/mock/gomock" metadata "google.golang.org/grpc/metadata" @@ -116,12 +115,12 @@ func (mr *MockSearchIngestorMockRecorder) GetAsyncSearchesList(arg0, arg1 interf } // Search mocks base method. -func (m *MockSearchIngestor) Search(ctx context.Context, sr *search.SearchRequest, tr *querytracer.Tracer) (*seq.QPR, search.DocsIterator, time.Duration, error) { +func (m *MockSearchIngestor) Search(ctx context.Context, sr *search.SearchRequest, tr *querytracer.Tracer) (*seq.QPR, search.DocsIterator, *search.SearchStats, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Search", ctx, sr, tr) ret0, _ := ret[0].(*seq.QPR) ret1, _ := ret[1].(search.DocsIterator) - ret2, _ := ret[2].(time.Duration) + ret2, _ := ret[2].(*search.SearchStats) ret3, _ := ret[3].(error) return ret0, ret1, ret2, ret3 } diff --git a/proxyapi/request_observation.go b/proxyapi/request_observation.go new file mode 100644 index 000000000..ef1adb22a --- /dev/null +++ b/proxyapi/request_observation.go @@ -0,0 +1,113 @@ +package proxyapi + +import ( + "context" + "errors" + "time" + + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ozontech/seq-db/consts" + "github.com/ozontech/seq-db/logger" + "github.com/ozontech/seq-db/metric" + "github.com/ozontech/seq-db/proxy/search" +) + +// requestObservation collects timing and storage-tier data for a single search +// request. The proxy fills it in doSearch and finalizes it (log + metric) with +// finish() from the calling handler, after the document stream has been read — +// so the recorded duration includes stream consumption. +type requestObservation struct { + start time.Time + fetchStart time.Time + fetchDuration time.Duration + stats *search.SearchStats + rawErr error // raw ingestor error before status wrapping (preserves timeout type) +} + +// search result categories used as the "result" metric label. +const ( + searchResultSuccess = "success" + searchResultClientErr = "client_error" + searchResultServerErr = "server_error" + searchResultTimeout = "timeout" +) + +func (o *requestObservation) takeFetchDuration() { + if o.fetchDuration == 0 && !o.fetchStart.IsZero() { + o.fetchDuration = time.Since(o.fetchStart) + } +} + +func (o *requestObservation) finish(method string, retErr error) { + if o == nil { + return + } + + o.takeFetchDuration() // if we stopped fetching before reaching EOF, calc fetch duration here + + result, tier := classifySearchResult(retErr, o.rawErr, o.stats) + fields := []zap.Field{ + zap.String("method", method), + zap.Bool("agg", o.stats.HasAgg), + zap.Bool("hist", o.stats.HasHist), + zap.Int("docs", o.stats.Size), + zap.String("tier", tier), + zap.Duration("hot_duration", o.stats.HotSearchDuration), + zap.Duration("total_search_duration", o.stats.TotalSearchDuration), + zap.Duration("fetch_duration", o.fetchDuration), + zap.Duration("total_duration", time.Since(o.start)), + zap.String("result", result), + } + if tier == string(search.StorageTierCold) { + fields = append(fields, zap.Duration("cold_duration", o.stats.ColdSearchDuration)) + } + if retErr != nil { + fields = append(fields, zap.NamedError("error", retErr)) + } + logger.Info("search request stat", fields...) + metric.SearchResults.WithLabelValues(result, tier).Inc() +} + +// classifySearchResult maps the final handler error and the raw ingestor error +// to a result category and a storage tier. The raw error is used to detect +// timeouts, since processSearchErrors wraps them into codes.Internal. +func classifySearchResult(retErr, rawErr error, stats *search.SearchStats) (result, tier string) { + tier = string(stats.StorageTier) + if tier == "" { + tier = string(search.StorageTierNone) + } + switch { + case retErr == nil: + return searchResultSuccess, tier + case isTimeoutErr(rawErr), isTimeoutErr(retErr): + return searchResultTimeout, tier + case isClientErr(retErr, rawErr): + return searchResultClientErr, tier + default: + return searchResultServerErr, tier + } +} + +func isTimeoutErr(e error) bool { + if e == nil { + return false + } + if errors.Is(e, context.DeadlineExceeded) || errors.Is(e, context.Canceled) { + return true + } + return status.Code(e) == codes.DeadlineExceeded +} + +func isClientErr(retErr, rawErr error) bool { + if errors.Is(rawErr, consts.ErrInvalidArgument) { + return true + } + switch status.Code(retErr) { + case codes.InvalidArgument, codes.ResourceExhausted, codes.Canceled: + return true + } + return false +} diff --git a/tests/integration_tests/integration_test.go b/tests/integration_tests/integration_test.go index a020f11a1..a1bb2c884 100644 --- a/tests/integration_tests/integration_test.go +++ b/tests/integration_tests/integration_test.go @@ -85,28 +85,28 @@ func (s *IntegrationTestSuite) TestSearchOne() { } // search first - qpr, docs, _, err := env.Search(`service:a`, 1000, setup.WithTotal(withTotal)) + qpr, docs, err := env.Search(`service:a`, 1000, setup.WithTotal(withTotal)) assertSearch(qpr, err) if assert.Greater(s.T(), len(docs), 0, "no docs found") { assert.Equal(s.T(), origDocs[0], string(docs[0]), "wrong doc content") } // search first with _exists_ - qpr, docs, _, err = env.Search(`_exists_:service`, 1000, setup.WithTotal(withTotal)) + qpr, docs, err = env.Search(`_exists_:service`, 1000, setup.WithTotal(withTotal)) assertSearch(qpr, err) if assert.Greater(s.T(), len(docs), 0, "no docs found") { assert.Equal(s.T(), origDocs[0], string(docs[0]), "wrong doc content") } // search first with NOT _exists_ - qpr, docs, _, err = env.Search(`NOT _exists_:k8s_pod`, 1000, setup.WithTotal(withTotal)) + qpr, docs, err = env.Search(`NOT _exists_:k8s_pod`, 1000, setup.WithTotal(withTotal)) assertSearch(qpr, err) if assert.Greater(s.T(), len(docs), 0, "no docs found") { assert.Equal(s.T(), origDocs[0], string(docs[0]), "wrong doc content") } // search second - qpr, docs, _, err = env.Search(`k8s_pod:sq-toloka-loader-1788964-dryrun-58hmw`, 1000, setup.WithTotal(withTotal)) + qpr, docs, err = env.Search(`k8s_pod:sq-toloka-loader-1788964-dryrun-58hmw`, 1000, setup.WithTotal(withTotal)) assertSearch(qpr, err) if assert.Greater(s.T(), len(docs), 0, "no docs found") { assert.Equal(s.T(), origDocs[1], string(docs[0]), "wrong doc content") @@ -254,7 +254,7 @@ func (s *IntegrationTestSuite) TestSearchNothing() { setup.Bulk(s.T(), env.IngestorBulkAddr(), origDocs) - qpr, _, _, err := env.Search(`k8s_pod:NO`, 1000, setup.NoFetch()) + qpr, _, err := env.Search(`k8s_pod:NO`, 1000, setup.NoFetch()) assert.NoError(s.T(), err, "should be no errors") assert.Len(s.T(), qpr.IDs, 0, "wrong doc count") assert.Equal(s.T(), uint64(0), qpr.Total, "wrong doc count") @@ -283,7 +283,7 @@ func (s *IntegrationTestSuite) TestSearchSequence() { for _, o := range []seq.DocsOrder{seq.DocsOrderAsc, seq.DocsOrderDesc} { for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`service:a`, math.MaxInt32, setup.NoFetch(), setup.WithTotal(withTotal), setup.WithOrder(o)) + qpr, _, err := env.Search(`service:a`, math.MaxInt32, setup.NoFetch(), setup.WithTotal(withTotal), setup.WithOrder(o)) assert.NoError(s.T(), err, "should be no errors") assert.Len(s.T(), qpr.IDs, bulks*bulkSize, "wrong doc count") assert.Equal(s.T(), getTotal(bulks*bulkSize, withTotal), qpr.Total, "wrong doc count") @@ -326,7 +326,7 @@ func (s *IntegrationTestSuite) TestSearchMany() { env.WaitIdle() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`service:a`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`service:a`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(n, withTotal), qpr.Total, "wrong doc count") } @@ -377,7 +377,7 @@ func (s *IntegrationTestSuite) TestFetch() { env, origDocs := s.envWithDummyDocs(16) env.WaitIdle() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`service:a`, 10, setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`service:a`, 10, setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(len(origDocs), withTotal), qpr.Total, "wrong doc count") } @@ -399,7 +399,7 @@ func (s *IntegrationTestSuite) TestFetch() { } for _, withTotal := range []bool{true, false} { - qpr, docs, _, err := env.Search(`service:a`, size, setup.WithTotal(withTotal), setup.WithOrder(o)) + qpr, docs, err := env.Search(`service:a`, size, setup.WithTotal(withTotal), setup.WithOrder(o)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), size, len(docs)) @@ -450,7 +450,7 @@ func (s *IntegrationTestSuite) TestMulti() { env.WaitIdle() // search - qpr, _, _, err := env.Search(`service:*`, 10) + qpr, _, err := env.Search(`service:*`, 10) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), uint64(len(origDocs)), qpr.Total, "wrong doc count") assert.Equal(s.T(), len(origDocs), len(qpr.IDs), "wrong doc count") @@ -494,31 +494,31 @@ func (s *IntegrationTestSuite) TestSearchNot() { env.WaitIdle() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`NOT service:b`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`NOT service:b`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(2*n*bulksNum, withTotal), qpr.Total, "wrong doc count") - qpr, _, _, err = env.Search(`NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(n*bulksNum, withTotal), qpr.Total, "wrong doc count") - qpr, _, _, err = env.Search(`NOT service:a AND NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT service:a AND NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), 0, int(qpr.Total), "wrong doc count") - qpr, _, _, err = env.Search(`NOT _exists_:service`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT _exists_:service`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), 0, int(qpr.Total), "wrong doc count") - qpr, _, _, err = env.Search(`NOT _exists_:k8s_pod`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT _exists_:k8s_pod`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") - qpr, _, _, err = env.Search(`NOT _exists_:k8s_pod`, -1, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT _exists_:k8s_pod`, -1, setup.NoFetch(), setup.WithTotal(withTotal)) assert.ErrorIs(s.T(), err, consts.ErrInvalidArgument) assert.Nil(s.T(), qpr) - qpr, _, _, err = env.Search(`NOT _exists_:k8s_pod`, 1, setup.WithOffset(-1), + qpr, _, err = env.Search(`NOT _exists_:k8s_pod`, 1, setup.WithOffset(-1), setup.NoFetch(), setup.WithTotal(withTotal)) assert.ErrorIs(s.T(), err, consts.ErrInvalidArgument) assert.Nil(s.T(), qpr) @@ -528,19 +528,19 @@ func (s *IntegrationTestSuite) TestSearchNot() { for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(n*bulksNum, withTotal), qpr.Total, "wrong doc count") - qpr, _, _, err = env.Search(`NOT service:a AND NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT service:a AND NOT service:x`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), 0, int(qpr.Total), "wrong doc count") - qpr, _, _, err = env.Search(`NOT _exists_:service`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT _exists_:service`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), 0, int(qpr.Total), "wrong doc count") - qpr, _, _, err = env.Search(`NOT _exists_:k8s_pod`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`NOT _exists_:k8s_pod`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") } @@ -566,7 +566,7 @@ func (s *IntegrationTestSuite) TestSearchPattern() { env.WaitIdle() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`service:x*`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`service:x*`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") } @@ -574,7 +574,7 @@ func (s *IntegrationTestSuite) TestSearchPattern() { env.SealAll() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`service:x*`, 10, setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`service:x*`, 10, setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") } @@ -604,7 +604,7 @@ func (s *IntegrationTestSuite) TestSearchSimple() { env.WaitIdle() for _, token := range tokens { - qpr, _, _, err := env.Search("message:"+token, 10, setup.NoFetch(), setup.WithTotal(true)) + qpr, _, err := env.Search("message:"+token, 10, setup.NoFetch(), setup.WithTotal(true)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), bulksNum, int(qpr.Total), "wrong doc count for token "+token) } @@ -612,7 +612,7 @@ func (s *IntegrationTestSuite) TestSearchSimple() { env.SealAll() for _, token := range tokens { - qpr, _, _, err := env.Search("message:"+token, 10, setup.NoFetch(), setup.WithTotal(true)) + qpr, _, err := env.Search("message:"+token, 10, setup.NoFetch(), setup.WithTotal(true)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), bulksNum, int(qpr.Total), "wrong doc count for token "+token) } @@ -632,7 +632,7 @@ func (s *IntegrationTestSuite) TestManySearchRequests() { env.WaitIdle() for x := 0; x < 5000; x++ { - qpr, _, _, err := env.Search(`service:x`, 10, setup.NoFetch()) + qpr, _, err := env.Search(`service:x`, 10, setup.NoFetch()) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), uint64(n), qpr.Total, "wrong doc count") } @@ -660,13 +660,13 @@ func (s *IntegrationTestSuite) TestAgg() { r := require.New(t) for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`service:x1`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`service:x1`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(getTotal(allDocsNum/3, withTotal), qpr.Total, "wrong doc count") r.NotNil(qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: "x1"}], qpr.Aggs[0].SamplesByBin) r.Equal(int64(allDocsNum/3), qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: "x1"}].Total, "wrong doc count") - qpr, _, _, err = env.Search(`service:x*`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`service:x*`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") r.Equal(int64(allDocsNum/3), qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: "x1"}].Total, "wrong doc count") @@ -675,14 +675,14 @@ func (s *IntegrationTestSuite) TestAgg() { "service", "k8s_pod", ) - qpr, _, _, err = env.Search(`service:x1`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`service:x1`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(getTotal(allDocsNum/3, withTotal), qpr.Total, "wrong doc count") r.Equal(2, len(qpr.Aggs), "wrong agg count") r.Equal(int64(allDocsNum/3), qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: "x1"}].Total, "wrong doc count") r.Equal(int64(allDocsNum/3), qpr.Aggs[1].SamplesByBin[seq.AggBin{Token: "y1"}].Total, "wrong doc count") - qpr, _, _, err = env.Search(`service:x*`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`service:x*`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(2, len(qpr.Aggs), "wrong agg count") r.Equal(getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") @@ -693,12 +693,12 @@ func (s *IntegrationTestSuite) TestAgg() { env.SealAll() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`service:x1`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`service:x1`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(getTotal(allDocsNum/3, withTotal), qpr.Total, "wrong doc count") r.Equal(int64(allDocsNum/3), qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: "x1"}].Total, "wrong doc count") - qpr, _, _, err = env.Search(`service:x*`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`service:x*`, 10, setup.WithAggQuery("service"), setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") r.Equal(int64(allDocsNum/3), qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: "x1"}].Total, "wrong doc count") @@ -707,14 +707,14 @@ func (s *IntegrationTestSuite) TestAgg() { "service", "k8s_pod", ) - qpr, _, _, err = env.Search(`service:x1`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`service:x1`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(getTotal(allDocsNum/3, withTotal), qpr.Total, "wrong doc count") r.Equal(2, len(qpr.Aggs), "wrong agg count") r.Equal(int64(allDocsNum/3), qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: "x1"}].Total, "wrong doc count") r.Equal(int64(allDocsNum/3), qpr.Aggs[1].SamplesByBin[seq.AggBin{Token: "y1"}].Total, "wrong doc count") - qpr, _, _, err = env.Search(`service:x*`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err = env.Search(`service:x*`, 10, aggQ, setup.NoFetch(), setup.WithTotal(withTotal)) r.NoError(err, "should be no errors") r.Equal(2, len(qpr.Aggs), "wrong agg count") r.Equal(getTotal(allDocsNum, withTotal), qpr.Total, "wrong doc count") @@ -772,7 +772,7 @@ func (s *IntegrationTestSuite) TestTimeseries() { t.Run("count", func(t *testing.T) { bulkDataset("nginx-count", func(int) int { return 1 }) - qpr, _, _, err := env.Search(`service:"nginx-count"`, 1024, setup.WithAggQuery(search.AggQuery{ + qpr, _, err := env.Search(`service:"nginx-count"`, 1024, setup.WithAggQuery(search.AggQuery{ GroupBy: "level", Func: seq.AggFuncCount, Interval: seq.DurationToMID(30 * time.Second), @@ -791,7 +791,7 @@ func (s *IntegrationTestSuite) TestTimeseries() { t.Run("min", func(t *testing.T) { bulkDataset("nginx-min", func(i int) int { return i }) - qpr, _, _, err := env.Search(`service:"nginx-min"`, 1024, setup.WithAggQuery(search.AggQuery{ + qpr, _, err := env.Search(`service:"nginx-min"`, 1024, setup.WithAggQuery(search.AggQuery{ Field: "level", GroupBy: "service", Func: seq.AggFuncMin, @@ -812,7 +812,7 @@ func (s *IntegrationTestSuite) TestTimeseries() { t.Run("max", func(t *testing.T) { bulkDataset("nginx-max", func(i int) int { return i }) - qpr, _, _, err := env.Search(`service:"nginx-max"`, 1024, setup.WithAggQuery(search.AggQuery{ + qpr, _, err := env.Search(`service:"nginx-max"`, 1024, setup.WithAggQuery(search.AggQuery{ Field: "level", Func: seq.AggFuncMax, Interval: seq.DurationToMID(30 * time.Second), @@ -831,7 +831,7 @@ func (s *IntegrationTestSuite) TestTimeseries() { t.Run("avg", func(t *testing.T) { bulkDataset("nginx-avg", func(int) int { return 1 }) - qpr, _, _, err := env.Search(`service:"nginx-avg"`, 1024, setup.WithAggQuery(search.AggQuery{ + qpr, _, err := env.Search(`service:"nginx-avg"`, 1024, setup.WithAggQuery(search.AggQuery{ Field: "level", Func: seq.AggFuncAvg, Interval: seq.DurationToMID(30 * time.Second), @@ -850,7 +850,7 @@ func (s *IntegrationTestSuite) TestTimeseries() { t.Run("sum", func(t *testing.T) { bulkDataset("nginx-sum", func(int) int { return 1 }) - qpr, _, _, err := env.Search(`service:"nginx-sum"`, 1024, setup.WithAggQuery(search.AggQuery{ + qpr, _, err := env.Search(`service:"nginx-sum"`, 1024, setup.WithAggQuery(search.AggQuery{ Field: "level", Func: seq.AggFuncSum, Interval: seq.DurationToMID(30 * time.Second), @@ -869,7 +869,7 @@ func (s *IntegrationTestSuite) TestTimeseries() { t.Run("quantile", func(t *testing.T) { bulkDataset("nginx-quantile", func(i int) int { return i }) - qpr, _, _, err := env.Search(`service:"nginx-quantile"`, 1024, setup.WithAggQuery(search.AggQuery{ + qpr, _, err := env.Search(`service:"nginx-quantile"`, 1024, setup.WithAggQuery(search.AggQuery{ Field: "level", Func: seq.AggFuncQuantile, Quantiles: []float64{0.5}, @@ -889,7 +889,7 @@ func (s *IntegrationTestSuite) TestTimeseries() { t.Run("unique_count", func(t *testing.T) { bulkDataset("nginx-unique-count", func(i int) int { return i % nextBin }) - qpr, _, _, err := env.Search(`service:"nginx-unique-count"`, 1024, setup.WithAggQuery(search.AggQuery{ + qpr, _, err := env.Search(`service:"nginx-unique-count"`, 1024, setup.WithAggQuery(search.AggQuery{ Field: "level", GroupBy: "service", Func: seq.AggFuncUniqueCount, @@ -956,7 +956,7 @@ func (s *IntegrationTestSuite) TestAggNoTotal() { env.WaitIdle() - searchNoTotal := func(agg string, interval time.Duration) (*seq.QPR, [][]byte, time.Duration, error) { + searchNoTotal := func(agg string, interval time.Duration) (*seq.QPR, [][]byte, error) { options := []setup.SearchOption{setup.WithInterval(interval), setup.NoFetch(), setup.WithTotal(false)} if agg != "" { options = append(options, setup.WithAggQuery(agg)) @@ -964,7 +964,7 @@ func (s *IntegrationTestSuite) TestAggNoTotal() { return env.Search(`service:x*`, size, options...) } - searchWithTotal := func(agg string, interval time.Duration) (*seq.QPR, [][]byte, time.Duration, error) { + searchWithTotal := func(agg string, interval time.Duration) (*seq.QPR, [][]byte, error) { options := []setup.SearchOption{setup.WithInterval(interval), setup.NoFetch()} if agg != "" { options = append(options, setup.WithAggQuery(agg)) @@ -974,18 +974,18 @@ func (s *IntegrationTestSuite) TestAggNoTotal() { test := func(t *testing.T) { // search - qpr, _, _, err := searchWithTotal("", 0) + qpr, _, err := searchWithTotal("", 0) require.NoError(t, err, "should be no errors") assert.Equal(t, uint64(allDocsNum), qpr.Total, "we must scann all docs in withTotal=true mode") assert.Equal(t, size, len(qpr.IDs), "we must get only size ids") - qpr, _, _, err = searchNoTotal("", 0) + qpr, _, err = searchNoTotal("", 0) require.NoError(t, err, "should be no errors") assert.Equal(t, uint64(0), qpr.Total, "we must get Total = 0 in withTotal=false mode") assert.Equal(t, size, len(qpr.IDs), "we must get only size ids") // aggregation - qpr, _, _, err = searchWithTotal("service", 0) + qpr, _, err = searchWithTotal("service", 0) require.NoError(t, err, "should be no errors") assert.Equal(t, uint64(allDocsNum), qpr.Total, "we must scan all docs in withTotal=true mode") assert.Equal(t, size, len(qpr.IDs), "we must get only size ids") @@ -994,7 +994,7 @@ func (s *IntegrationTestSuite) TestAggNoTotal() { assert.Equal(t, int(aggCnt), int(qpr.Aggs[0].SamplesByBin[seq.AggBin{Token: k}].Total), "we expect 1/%d of all documents", parts) } - qpr, _, _, err = searchNoTotal("service", 0) + qpr, _, err = searchNoTotal("service", 0) require.NoError(t, err, "should be no errors") assert.Equal(t, uint64(0), qpr.Total, "we must get Total = 0 in withTotal=false mode") assert.Equal(t, size, len(qpr.IDs), "we must get only size ids") @@ -1004,7 +1004,7 @@ func (s *IntegrationTestSuite) TestAggNoTotal() { } // histogram - qpr, _, _, err = searchWithTotal("", histInterval) + qpr, _, err = searchWithTotal("", histInterval) require.NoError(t, err, "should be no errors") assert.Equal(t, uint64(allDocsNum), qpr.Total, "we must scann all docs in withTotal=true mode") assert.Equal(t, size, len(qpr.IDs), "we must get only size ids") @@ -1015,7 +1015,7 @@ func (s *IntegrationTestSuite) TestAggNoTotal() { } assert.Equal(t, uint64(allDocsNum), histSum, "the sum of the histogram should be equal to the number of all documents") - qpr, _, _, err = searchNoTotal("", histInterval) + qpr, _, err = searchNoTotal("", histInterval) require.NoError(t, err, "should be no errors") assert.Equal(t, uint64(0), qpr.Total, "we must get Total = 0 in withTotal=false mode") assert.Equal(t, size, len(qpr.IDs), "we must get only size ids") @@ -1074,7 +1074,7 @@ func (s *IntegrationTestSuite) TestSeal() { env.WaitIdle() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`status:200`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`status:200`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(result, withTotal), qpr.Total, "wrong doc count") } @@ -1083,7 +1083,7 @@ func (s *IntegrationTestSuite) TestSeal() { env.SealAll() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`status:200`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`status:200`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(result, withTotal), qpr.Total, "wrong doc count") } @@ -1096,7 +1096,7 @@ func (s *IntegrationTestSuite) TestSeal() { defer env.StopAll() for _, withTotal := range []bool{true, false} { - qpr, _, _, err := env.Search(`status:200`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) + qpr, _, err := env.Search(`status:200`, 10, setup.NoFetch(), setup.WithTotal(withTotal)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), getTotal(result, withTotal), qpr.Total, "wrong doc count") } @@ -1114,7 +1114,7 @@ func (s *IntegrationTestSuite) TestQueryErr() { setup.Bulk(s.T(), env.IngestorBulkAddr(), origDocs) for _, withTotal := range []bool{true, false} { - _, _, _, err := env.Search(`service:a:`, 1000, setup.NoFetch(), setup.WithTotal(withTotal)) + _, _, err := env.Search(`service:a:`, 1000, setup.NoFetch(), setup.WithTotal(withTotal)) assert.True(s.T(), err != nil, "should be an error") } } @@ -1140,7 +1140,7 @@ func (s *IntegrationTestSuite) TestConnectionRefused() { return next, nil }) }() - _, _, _, err := env.Search(`service:a`, 1000, setup.NoFetch()) + _, _, err := env.Search(`service:a`, 1000, setup.NoFetch()) if assert.True(s.T(), err != nil, "should be an error") { assert.True(s.T(), strings.Contains(err.Error(), "connection refused"), "error should be connection refused") @@ -1235,7 +1235,7 @@ func (s *IntegrationTestSuite) TestBulkBadTimestamp() { for _, o := range []seq.DocsOrder{seq.DocsOrderAsc, seq.DocsOrderDesc} { for _, withTotal := range []bool{true, false} { - qpr, docs, _, err := env.Search(`service:a`, 1000, setup.WithTotal(withTotal), setup.WithOrder(o)) + qpr, docs, err := env.Search(`service:a`, 1000, setup.WithTotal(withTotal), setup.WithOrder(o)) assert.NoError(s.T(), err, "should be no errors") if o.IsReverse() { @@ -1409,7 +1409,7 @@ func (s *IntegrationTestSuite) TestDocuments() { env.WaitIdle() for _, o := range []seq.DocsOrder{seq.DocsOrderAsc, seq.DocsOrderDesc} { - qpr, _, _, err := env.Search(`service:a`, n, setup.WithTotal(true), setup.NoFetch(), setup.WithOrder(o)) + qpr, _, err := env.Search(`service:a`, n, setup.WithTotal(true), setup.NoFetch(), setup.WithOrder(o)) s.Assert().NoError(err) s.Assert().Equal(getTotal(len(origDocs), true), qpr.Total, "wrong doc count") @@ -1420,11 +1420,10 @@ func (s *IntegrationTestSuite) TestDocuments() { actualDocs := []string{} actualIDs := []seq.ID{} - for doc, err := docsStream.Next(); err == nil; doc, err = docsStream.Next() { + for doc := range search.DocsIteratorSeq(docsStream) { actualIDs = append(actualIDs, doc.ID) actualDocs = append(actualDocs, string(doc.Data)) } - s.Assert().Equal(qpr.IDs.IDs(), actualIDs) copyDocs := copySlice(origDocs) @@ -1476,7 +1475,7 @@ func (s *IntegrationTestSuite) TestSearchFieldsWithMultipleTypes() { test := func(tc testCase) func(t *testing.T) { return func(t *testing.T) { - qpr, _, _, err := env.Search(tc.request, 100, setup.WithTotal(true)) + qpr, _, err := env.Search(tc.request, 100, setup.WithTotal(true)) require.NoError(t, err) assert.Len(t, qpr.IDs, tc.cnt) assert.Equal(t, tc.cnt, int(qpr.Total)) @@ -1518,7 +1517,7 @@ func (s *IntegrationTestSuite) TestAggregateFieldsWithMultipleTypes() { setup.Bulk(s.T(), env.IngestorBulkAddr(), docs) env.WaitIdle() - qpr, _, _, err := env.Search( + qpr, _, err := env.Search( "level:error", 100, setup.WithAggQuery(search.AggQuery{Field: "message.keyword", Func: seq.AggFuncCount}), @@ -1763,7 +1762,7 @@ func (s *IntegrationTestSuite) TestPaginationWithOffsetAndSize() { for _, order := range []seq.DocsOrder{seq.DocsOrderDesc, seq.DocsOrderAsc} { for { - qpr, docs, _, err := env.Search(`service:*`, pageSize, setup.WithOffset(offset), setup.WithOrder(order)) + qpr, docs, err := env.Search(`service:*`, pageSize, setup.WithOffset(offset), setup.WithOrder(order)) r.NoError(err, "search failed") if len(qpr.IDs) == 0 { @@ -1815,7 +1814,7 @@ func (s *IntegrationTestSuite) TestPaginationWithOffsetId() { fetchedDocs := make(map[string]bool) var offsetId string for { - qpr, docs, _, err := env.Search(`service:*`, pageSize, setup.WithOffsetId(offsetId), setup.WithOrder(order)) + qpr, docs, err := env.Search(`service:*`, pageSize, setup.WithOffsetId(offsetId), setup.WithOrder(order)) r.NoError(err, "search failed") if len(qpr.IDs) == 0 { @@ -1860,7 +1859,7 @@ func (s *IntegrationTestSuite) TestSkipMaskManager() { setup.Bulk(t, env.IngestorBulkAddr(), docs) // save hidden doc ids to test fetch later - qpr, _, _, err := env.Search(`service:hidden`, 10, setup.WithTotal(true)) + qpr, _, err := env.Search(`service:hidden`, 10, setup.WithTotal(true)) r.NoError(err) hiddenDocIDs := qpr.IDs.IDs() @@ -1895,11 +1894,11 @@ func (s *IntegrationTestSuite) TestSkipMaskManager() { // test search - qpr, _, _, err = env.Search(`service:hidden`, 10, setup.WithTotal(true)) + qpr, _, err = env.Search(`service:hidden`, 10, setup.WithTotal(true)) r.NoError(err) r.Equal(uint64(0), qpr.Total) - qpr, _, _, err = env.Search(`service:*`, 10, setup.WithTotal(true)) + qpr, _, err = env.Search(`service:*`, 10, setup.WithTotal(true)) r.NoError(err) r.Equal(uint64(4), qpr.Total) @@ -1922,7 +1921,7 @@ func (s *IntegrationTestSuite) TestSkipMaskManager() { return checkSkipMasksStatus(env.HotStores) && checkSkipMasksStatus(env.ColdStores) }, 5*time.Second, 100*time.Millisecond) - qpr, _, _, err = env.Search(`service:hidden`, 10, setup.WithTotal(true)) + qpr, _, err = env.Search(`service:hidden`, 10, setup.WithTotal(true)) r.NoError(err) r.Equal(uint64(0), qpr.Total) } diff --git a/tests/integration_tests/single_test.go b/tests/integration_tests/single_test.go index 54bdeca94..9aa5008a9 100644 --- a/tests/integration_tests/single_test.go +++ b/tests/integration_tests/single_test.go @@ -124,7 +124,7 @@ func (s *SingleTestSuite) TestSearchAgg() { assertAgg := func(query string, aggQ []any, expected []map[string]uint64) { r := s.Require() - qpr, _, _, err := s.Env.Search(query, math.MaxInt32, setup.WithAggQuery(aggQ...), setup.WithTotal(false)) + qpr, _, err := s.Env.Search(query, math.MaxInt32, setup.WithAggQuery(aggQ...), setup.WithTotal(false)) r.NoError(err) r.Equal(len(expected), len(qpr.Aggs)) for i := range expected { @@ -210,7 +210,7 @@ func (s *SingleTestSuite) TestFetchHints() { sort.Sort(&ExampleDocSorting{sample: docsSample, docStrs: docStrs}) - qpr, _, _, err := s.Env.Search("_all_:*", math.MaxInt32, setup.WithTotal(true), setup.NoFetch()) + qpr, _, err := s.Env.Search("_all_:*", math.MaxInt32, setup.WithTotal(true), setup.NoFetch()) s.Assert().NoError(err) origIDs := qpr.IDs diff --git a/tests/integration_tests/sub_search_test.go b/tests/integration_tests/sub_search_test.go index 202b098ab..cfb66cdd0 100644 --- a/tests/integration_tests/sub_search_test.go +++ b/tests/integration_tests/sub_search_test.go @@ -113,7 +113,7 @@ func (s *IntegrationTestSuite) TestSubSearch() { expectedCount = expectedTotal } - qpr, _, _, err := env.Search("service:*", limit, setup.NoFetch(), setup.WithTotal(false), setup.WithTimeRange(f, t)) + qpr, _, err := env.Search("service:*", limit, setup.NoFetch(), setup.WithTotal(false), setup.WithTimeRange(f, t)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), expectedCount, len(qpr.IDs), "wrong doc count in range [%s, %s]", f, t) } @@ -132,7 +132,7 @@ func (s *IntegrationTestSuite) TestSubSearch() { expectedCount = expectedTotal } - qpr, _, _, err := env.Search("service:*", limit, setup.NoFetch(), setup.WithTotal(true), setup.WithTimeRange(f, t)) + qpr, _, err := env.Search("service:*", limit, setup.NoFetch(), setup.WithTotal(true), setup.WithTimeRange(f, t)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), expectedCount, len(qpr.IDs), "wrong doc count in range [%s, %s]", f, t) assert.Equal(s.T(), expectedTotal, int(qpr.Total), "wrong doc count in range [%s, %s]", f, t) @@ -153,7 +153,7 @@ func (s *IntegrationTestSuite) TestSubSearch() { } interval := 3 * time.Minute - qpr, _, _, err := env.Search("service:*", limit, setup.NoFetch(), setup.WithTotal(false), setup.WithInterval(interval), setup.WithTimeRange(f, t)) + qpr, _, err := env.Search("service:*", limit, setup.NoFetch(), setup.WithTotal(false), setup.WithInterval(interval), setup.WithTimeRange(f, t)) assert.NoError(s.T(), err, "should be no errors") assert.Equal(s.T(), expectedCount, len(qpr.IDs), "wrong doc count in range [%s, %s]", f, t) assert.Equal(s.T(), makeHist(sub, interval), qpr.Histogram, "wrong doc count in range [%s, %s]", f, t) diff --git a/tests/setup/env.go b/tests/setup/env.go index 86d1703a8..086f73ee3 100644 --- a/tests/setup/env.go +++ b/tests/setup/env.go @@ -618,15 +618,15 @@ func (t *TestingEnv) HTTPSearch(tt *testing.T, q string, size int, options ...Se }) } -func (t *TestingEnv) Search(q string, size int, options ...SearchOption) (*seq.QPR, [][]byte, time.Duration, error) { +func (t *TestingEnv) Search(q string, size int, options ...SearchOption) (*seq.QPR, [][]byte, error) { sr := t.buildRequest(q, size, options...) var docs [][]byte - qpr, docsStream, duration, err := t.Ingestor().SearchIngestor.Search(context.Background(), sr, nil) + qpr, docsStream, _, err := t.Ingestor().SearchIngestor.Search(context.Background(), sr, nil) if docsStream != nil { docs = search.ReadAll(docsStream) } - return qpr, docs, duration, err + return qpr, docs, err } func (t *TestingEnv) Fetch(ids []seq.ID) ([][]byte, error) { diff --git a/tests/suites/single.go b/tests/suites/single.go index 469782199..0bad1dca1 100644 --- a/tests/suites/single.go +++ b/tests/suites/single.go @@ -36,11 +36,11 @@ func (s *Single) Bulk(docs []string) { } func (s *Single) SearchDocs(query string, size int, order seq.DocsOrder) []string { - _, docs1, _, err := s.Env.Search(query, size, setup.WithOrder(order)) + _, docs1, err := s.Env.Search(query, size, setup.WithOrder(order)) s.Require().NoError(err) r1 := common.ToStringSlice(docs1) - _, docs2, _, err := s.Env.Search(query, size, setup.WithTotal(false), setup.WithOrder(order)) + _, docs2, err := s.Env.Search(query, size, setup.WithTotal(false), setup.WithOrder(order)) s.Require().NoError(err) r2 := common.ToStringSlice(docs2)