diff --git a/config/frac_version.go b/config/frac_version.go index d6bf41d6c..352601246 100644 --- a/config/frac_version.go +++ b/config/frac_version.go @@ -24,6 +24,9 @@ const ( // BinaryDataV5 - token blocks have zone maps (eng letters presense) and doc frequencies for heavy tokens BinaryDataV5 + + // BinaryDataV6 - keep offsets separate in token block + BinaryDataV6 ) -const CurrentFracVersion = BinaryDataV5 +const CurrentFracVersion = BinaryDataV6 diff --git a/frac/active_token_list.go b/frac/active_token_list.go index c6867417f..a37eb4d20 100644 --- a/frac/active_token_list.go +++ b/frac/active_token_list.go @@ -53,6 +53,21 @@ func (tp *activeTokenProvider) FindContains(needle []byte) ([]uint32, error) { return tids, nil } +// FindSuffix finds tids of tokens which end with a provided suffix. +func (tp *activeTokenProvider) FindSuffix(suffix []byte) ([]uint32, error) { + if len(suffix) == 0 { + return nil, nil + } + var tids []uint32 + for tid := tp.FirstTID(); tid <= tp.LastTID(); tid++ { + token := tp.GetToken(tid) + if len(token) >= len(suffix) && bytes.Equal(token[len(token)-len(suffix):], suffix) { + tids = append(tids, tid) + } + } + return tids, nil +} + // FindToken finds tids of tokens which suffice a provided searcher (predicate). func (tp *activeTokenProvider) FindToken(searcher pattern.Searcher) ([]uint32, error) { firstTID := searcher.FirstTID() diff --git a/frac/sealed/token/block_loader.go b/frac/sealed/token/block_loader.go index 266315725..24345f7dc 100644 --- a/frac/sealed/token/block_loader.go +++ b/frac/sealed/token/block_loader.go @@ -24,6 +24,9 @@ type Block struct { Offsets []uint32 FreqIndexes []uint16 // indexes of tokens which have doc freqs (frequencies) Freqs []uint32 // frequencies of certain tokens (how many docs have this token included at least once) + + // TODO(cheb0) delete this field and convert V0..V5 to a new in-memory format when data all clusters have V6 fractions + FracVer config.BinaryDataVersion } func (b *Block) Size() int { @@ -46,6 +49,8 @@ func (b Block) Pack(dst []byte, buf []uint32) []byte { dst = binary.LittleEndian.AppendUint32(dst, uint32(len(b.Payload))) dst = append(dst, b.Payload...) + dst = packer.CompressDeltaBitpackUint32(dst, b.Offsets, buf) + if len(b.FreqIndexes) > 0 { dst = packer.CompressDeltaBitpackUint16(dst, b.FreqIndexes, buf) dst = packer.CompressDeltaBitpackUint32(dst, b.Freqs, buf) @@ -55,6 +60,13 @@ func (b Block) Pack(dst []byte, buf []uint32) []byte { } func (b *Block) Unpack(data []byte, fracVer config.BinaryDataVersion, unpackBuf *UnpackBuffer) error { + b.FracVer = fracVer + + if fracVer >= config.BinaryDataV6 { + unpackBuf.Reset(fracVer) + return b.unpackV6(data, unpackBuf) + } + if fracVer >= config.BinaryDataV5 { unpackBuf.Reset(fracVer) return b.unpackV5(data, unpackBuf) @@ -62,9 +74,39 @@ func (b *Block) Unpack(data []byte, fracVer config.BinaryDataVersion, unpackBuf return b.unpackV1(data) } -func (b *Block) unpackV1(data []byte) error { - b.Payload = append([]byte{}, data...) - return b.parseTokenPayload(b.Payload) +func (b *Block) unpackV6(data []byte, buf *UnpackBuffer) error { + if len(data) < util.SizeOfUint32 { + return fmt.Errorf("token block too short: %d bytes", len(data)) + } + flags := data[0] + data = data[1:] + + // token payload + payloadLen := binary.LittleEndian.Uint32(data[:util.SizeOfUint32]) + data = data[util.SizeOfUint32:] + if uint32(len(data)) < payloadLen { + return fmt.Errorf("invalid token block payload length: %d, data len %d", payloadLen, len(data)) + } + + payload := data[:payloadLen] + data = data[payloadLen:] + + b.Payload = append(b.Payload[:0], payload...) + + // offsets + var err error + data, buf.decompressedUint32, err = packer.DecompressDeltaBitpackUint32(data, buf.decompressedUint32, buf.compressed) + if err != nil { + return err + } + b.Offsets = append(b.Offsets, buf.decompressedUint32...) + + err = b.unpackFreqs(data, buf, flags) + if err != nil { + return err + } + + return nil } func (b *Block) unpackV5(data []byte, buf *UnpackBuffer) error { @@ -85,11 +127,22 @@ func (b *Block) unpackV5(data []byte, buf *UnpackBuffer) error { b.Payload = append(b.Payload[:0], payload...) - if err := b.parseTokenPayload(payload); err != nil { + if err := b.parseTokenPayloadV5(payload); err != nil { + return err + } + + err := b.unpackFreqs(data, buf, flags) + if err != nil { return err } + return nil +} + +func (b *Block) unpackFreqs(data []byte, buf *UnpackBuffer, flags byte) error { if flags&1 > 0 { + buf.decompressedUint32 = buf.decompressedUint32[:0] + var err error data, buf.decompressedUint16, err = packer.DecompressDeltaBitpackUint16(data, buf.decompressedUint16, buf.compressed) if err != nil { @@ -103,11 +156,16 @@ func (b *Block) unpackV5(data []byte, buf *UnpackBuffer) error { } b.Freqs = append(b.Freqs, buf.decompressedUint32...) } - return nil } -func (b *Block) parseTokenPayload(data []byte) error { +func (b *Block) unpackV1(data []byte) error { + b.Payload = append([]byte{}, data...) + return b.parseTokenPayloadV5(b.Payload) +} + +// parseTokenPayloadV5 derives offsets from tokens payload. Only used for v1...v5 legacy fractions. +func (b *Block) parseTokenPayloadV5(data []byte) error { b.Offsets = b.Offsets[:0] var offset uint32 @@ -129,6 +187,10 @@ func (b *Block) parseTokenPayload(data []byte) error { } func (b *Block) Len() int { + if b.FracVer >= config.BinaryDataV6 { + return len(b.Offsets) - 1 + } + return len(b.Offsets) } @@ -147,6 +209,14 @@ func (b *Block) GetFreq(index int) uint32 { } func (b *Block) GetToken(index int) []byte { + if b.FracVer >= config.BinaryDataV6 { + return b.Payload[b.Offsets[index]:b.Offsets[index+1]] + } + + return b.getTokenV5(index) +} + +func (b *Block) getTokenV5(index int) []byte { offset := b.Offsets[index] l := binary.LittleEndian.Uint32(b.Payload[offset:]) offset += uint32(util.SizeOfUint32) // skip val length @@ -162,15 +232,79 @@ func (b *Block) LettersBitset() util.LettersBitset { } func (b *Block) contains(from, to int, needle []byte) ([]int, error) { + if b.FracVer >= config.BinaryDataV6 { + return b.containsV6(from, to, needle) + } + + return b.containsV5(from, to, needle) +} + +func (b *Block) containsV6(from, to int, needle []byte) ([]int, error) { + indexes := make([]int, 0) + offsets := b.Offsets[from : to+2] + for i := 1; i < len(offsets); i++ { + tok := b.Payload[offsets[i-1]:offsets[i]] + if bytes.Contains(tok, needle) { + indexes = append(indexes, from+i-1) + } + } + return indexes, nil +} + +// TODO(cheb0) delete when Block.FracVer is deleted +func (b *Block) containsV5(from, to int, needle []byte) ([]int, error) { indexes := make([]int, 0) for i := from; i <= to; i++ { - if bytes.Contains(b.GetToken(i), needle) { + if bytes.Contains(b.getTokenV5(i), needle) { indexes = append(indexes, i) } } return indexes, nil } +func (b *Block) suffix(from, to int, suffix []byte) ([]int, error) { + if b.FracVer >= config.BinaryDataV6 { + return b.suffixV6(from, to, suffix), nil + } + + return b.suffixV5(from, to, suffix), nil +} + +// TODO(cheb0) delete this when Block.FracVer is deleted +func (b *Block) suffixV5(from, to int, suffix []byte) []int { + indexes := make([]int, 0) + suffixLen := len(suffix) + + for i := from; i <= to; i++ { + token := b.GetToken(i) + if len(token) >= suffixLen && bytes.Equal(token[len(token)-suffixLen:], suffix) { + indexes = append(indexes, i) + } + } + + return indexes +} + +func (b *Block) suffixV6(from, to int, suffix []byte) []int { + indexes := make([]int, 0) + suffixLen := uint32(len(suffix)) + + offsets := b.Offsets[from : to+2] + + for i := 1; i < len(offsets); i++ { + endPos := offsets[i] + tokLen := endPos - offsets[i-1] + if tokLen >= suffixLen { + tokSuffix := b.Payload[endPos-suffixLen : endPos] + if bytes.Equal(tokSuffix, suffix) { + indexes = append(indexes, from+i-1) + } + } + } + + return indexes +} + func (b *Block) find(from, to int, searcher pattern.Searcher) ([]int, error) { indexes := make([]int, 0) for i := from; i <= to; i++ { diff --git a/frac/sealed/token/block_loader_test.go b/frac/sealed/token/block_loader_test.go index 6f45e02ad..1e6b052f7 100644 --- a/frac/sealed/token/block_loader_test.go +++ b/frac/sealed/token/block_loader_test.go @@ -1,7 +1,6 @@ package token import ( - "encoding/binary" "testing" "github.com/stretchr/testify/assert" @@ -11,14 +10,12 @@ import ( ) func TestBlock_PackUnpack_NoFreq(t *testing.T) { - src := Block{ - Payload: packTokenPayload([]byte("foo"), []byte("bar")), - } + src := buildTokenPayload([]byte("foo"), []byte("bar")) var buf []uint32 packed := src.Pack(nil, buf) var dst Block - require.NoError(t, dst.Unpack(packed, config.BinaryDataV5, &UnpackBuffer{})) + require.NoError(t, dst.Unpack(packed, config.BinaryDataV6, &UnpackBuffer{})) assert.Equal(t, 2, dst.Len()) assert.Equal(t, []byte("foo"), dst.GetToken(0)) @@ -29,16 +26,14 @@ func TestBlock_PackUnpack_NoFreq(t *testing.T) { } func TestBlock_PackUnpack_WithFreq(t *testing.T) { - src := Block{ - Payload: packTokenPayload([]byte("dog"), []byte("cat"), []byte("horse"), []byte("duck")), - FreqIndexes: []uint16{0, 2}, - Freqs: []uint32{100, 200}, - } + src := buildTokenPayload([]byte("dog"), []byte("cat"), []byte("horse"), []byte("duck")) + src.FreqIndexes = []uint16{0, 2} + src.Freqs = []uint32{100, 200} var buf []uint32 packed := src.Pack(nil, buf) var dst Block - require.NoError(t, dst.Unpack(packed, config.BinaryDataV5, &UnpackBuffer{})) + require.NoError(t, dst.Unpack(packed, config.BinaryDataV6, &UnpackBuffer{})) assert.Equal(t, src.Payload, dst.Payload) @@ -48,24 +43,10 @@ func TestBlock_PackUnpack_WithFreq(t *testing.T) { assert.Equal(t, uint32(0), dst.GetFreq(3)) } -func TestBlock_Unpack_Legacy(t *testing.T) { - legacy := packTokenPayload([]byte("legacy")) - - var dst Block - require.NoError(t, dst.Unpack(legacy, config.BinaryDataV4, &UnpackBuffer{})) - - assert.Equal(t, legacy, dst.Payload) - assert.Equal(t, []uint32{0}, dst.Offsets) - assert.Empty(t, dst.FreqIndexes) - assert.Empty(t, dst.Freqs) -} - func TestBlock_UnpackBufferReuse(t *testing.T) { - src := Block{ - Payload: packTokenPayload([]byte("a"), []byte("b")), - FreqIndexes: []uint16{1}, - Freqs: []uint32{64}, - } + src := buildTokenPayload([]byte("a"), []byte("b")) + src.FreqIndexes = []uint16{1} + src.Freqs = []uint32{64} var packBuf []uint32 packed := src.Pack(nil, packBuf) @@ -73,8 +54,8 @@ func TestBlock_UnpackBufferReuse(t *testing.T) { unpackBuf := &UnpackBuffer{} var dst1, dst2 Block - require.NoError(t, dst1.Unpack(packed, config.BinaryDataV5, unpackBuf)) - require.NoError(t, dst2.Unpack(packed, config.BinaryDataV5, unpackBuf)) + require.NoError(t, dst1.Unpack(packed, config.BinaryDataV6, unpackBuf)) + require.NoError(t, dst2.Unpack(packed, config.BinaryDataV6, unpackBuf)) assert.Equal(t, dst1.FreqIndexes, dst2.FreqIndexes) assert.Equal(t, dst1.Freqs, dst2.Freqs) @@ -83,11 +64,16 @@ func TestBlock_UnpackBufferReuse(t *testing.T) { assert.Equal(t, uint32(64), dst2.GetFreq(1)) } -func packTokenPayload(tokens ...[]byte) []byte { +func buildTokenPayload(tokens ...[]byte) Block { var payload []byte + var offsets []uint32 + offsets = append(offsets, 0) for _, tok := range tokens { - payload = binary.LittleEndian.AppendUint32(payload, uint32(len(tok))) + offsets = append(offsets, offsets[len(offsets)-1]+uint32(len(tok))) payload = append(payload, tok...) } - return payload + return Block{ + Payload: payload, + Offsets: offsets, + } } diff --git a/frac/sealed/token/provider.go b/frac/sealed/token/provider.go index d3428c51e..561f5f6c7 100644 --- a/frac/sealed/token/provider.go +++ b/frac/sealed/token/provider.go @@ -74,6 +74,20 @@ func (tp *Provider) FindContains(needle []byte) ([]uint32, error) { }) } +func (tp *Provider) FindSuffix(suffix []byte) ([]uint32, error) { + requiredLetters := util.NewLettersBitset(suffix) + + return tp.findInBlocks( + tp.FirstTID(), + tp.LastTID(), + func(e *TableEntry) bool { + return e.Letters.IsNil() || e.Letters.ContainsAll(requiredLetters) + }, + func(b *Block, firstIndex, lastIndex int) ([]int, error) { + return b.suffix(firstIndex, lastIndex, suffix) + }) +} + func (tp *Provider) FindToken(searcher pattern.Searcher) ([]uint32, error) { return tp.findInBlocks( searcher.FirstTID(), diff --git a/indexwriter/blocks.go b/indexwriter/blocks.go index a05dc2628..1353a9371 100644 --- a/indexwriter/blocks.go +++ b/indexwriter/blocks.go @@ -1,11 +1,11 @@ package indexwriter import ( - "encoding/binary" "iter" "math" "unsafe" + "github.com/ozontech/seq-db/config" "github.com/ozontech/seq-db/frac/sealed/lids" "github.com/ozontech/seq-db/frac/sealed/seqids" "github.com/ozontech/seq-db/frac/sealed/token" @@ -56,6 +56,7 @@ func tokenBlock( blockIdx uint32 blockSize int ) + block.payload.FracVer = config.BinaryDataV6 var ( currentTID uint32 @@ -123,9 +124,13 @@ func tokenBlock( } } - tokenIndex := uint32(len(block.payload.Offsets)) - block.payload.Offsets = append(block.payload.Offsets, uint32(len(block.payload.Payload))) - block.payload.Payload = binary.LittleEndian.AppendUint32(block.payload.Payload, uint32(len(tok))) + offsets := block.payload.Offsets + if len(offsets) == 0 { + offsets = append(offsets, 0) + } + tokenIndex := uint32(len(offsets) - 1) + offsets = append(offsets, offsets[len(offsets)-1]+uint32(len(tok))) + block.payload.Offsets = offsets block.payload.Payload = append(block.payload.Payload, tok...) if len(tlids) >= tokenFreqAbsThreshold { diff --git a/pattern/pattern.go b/pattern/pattern.go index 504d3929d..ebdf52593 100644 --- a/pattern/pattern.go +++ b/pattern/pattern.go @@ -18,6 +18,7 @@ import ( type tokenProvider interface { GetToken(uint32) []byte FindContains(needle []byte) ([]uint32, error) + FindSuffix(suffix []byte) ([]uint32, error) FindToken(searcher Searcher) ([]uint32, error) FirstTID() uint32 LastTID() uint32 @@ -424,6 +425,18 @@ func isSimpleWildcardContains(token parser.Token) (needle []byte, ok bool) { return []byte(lit.Terms[1].Data), true } +// isSimpleWildcardSuffix checks if this AST token is simple wildcard like '*abc' +func isSimpleWildcardSuffix(token parser.Token) (suffix []byte, ok bool) { + lit, ok := token.(*parser.Literal) + if !ok || len(lit.Terms) != 2 { + return nil, false + } + if !lit.Terms[0].IsWildcard() || lit.Terms[1].Kind != parser.TermText { + return nil, false + } + return []byte(lit.Terms[1].Data), true +} + func Search(ctx context.Context, t parser.Token, tp tokenProvider) ([]uint32, error) { if util.IsCancelled(ctx) { return nil, ctx.Err() @@ -431,6 +444,9 @@ func Search(ctx context.Context, t parser.Token, tp tokenProvider) ([]uint32, er if needle, ok := isSimpleWildcardContains(t); ok { return tp.FindContains(needle) } + if suffix, ok := isSimpleWildcardSuffix(t); ok { + return tp.FindSuffix(suffix) + } s := newSearcher(t, tp) return tp.FindToken(s) } diff --git a/pattern/pattern_test.go b/pattern/pattern_test.go index f89766e96..3cdf9f563 100644 --- a/pattern/pattern_test.go +++ b/pattern/pattern_test.go @@ -113,6 +113,20 @@ func (tp *simpleTokenProvider) FindContains(needle []byte) ([]uint32, error) { return tids, nil } +func (tp *simpleTokenProvider) FindSuffix(suffix []byte) ([]uint32, error) { + if len(suffix) == 0 { + return nil, nil + } + var tids []uint32 + for t := tp.FirstTID(); t <= tp.LastTID(); t++ { + token := tp.GetToken(t) + if len(token) >= len(suffix) && bytes.Equal(token[len(token)-len(suffix):], suffix) { + tids = append(tids, t) + } + } + return tids, nil +} + func (tp *simpleTokenProvider) FindToken(searcher Searcher) ([]uint32, error) { firstTID := searcher.FirstTID() lastTID := searcher.LastTID() @@ -335,6 +349,28 @@ func TestPatternSuffix2(t *testing.T) { testAll(t, tp, tests) } +func TestPatternSuffixOnly(t *testing.T) { + tp := newTestTokenProvider([]string{ + "abc", + "xabc", + "xyabc", + "xyzabc", + "abcx", + "xabcx", + "notabc", + "nothing", + }) + + tests := []testCase{ + {"*abc", []string{"abc", "notabc", "xabc", "xyabc", "xyzabc"}}, + {"*x", []string{"abcx", "xabcx"}}, + {"*ng", []string{"nothing"}}, + {"*g", []string{"nothing"}}, + } + + testAll(t, tp, tests) +} + func TestPatternMiddle(t *testing.T) { tp := newTestTokenProvider([]string{ "a:b:a",