diff --git a/plugin/action/transform/README.md b/plugin/action/transform/README.md index 7373931e8..db90633f4 100755 --- a/plugin/action/transform/README.md +++ b/plugin/action/transform/README.md @@ -180,9 +180,9 @@ are required; named arguments are optional and fall back to their defaults: .msg = "code is " + string(.code) ``` -+ `capture(value, pattern)` — matches the string against a regular expression - and returns an object of its named groups `(?P...)`, or `null` when the - value does not match (unnamed groups are ignored): ++ `capture(value, pattern, numeric_groups: false)` — matches the string against a + regular expression and returns an object of its named groups `(?P...)`, + or `null` when the value does not match (unnamed groups are ignored): ``` m = capture(.log, r'^(?P\S+)\s+(?P.+)$') if m != null { @@ -190,6 +190,15 @@ are required; named arguments are optional and fall back to their defaults: .message = m.message } ``` + With `numeric_groups: true` every group is additionally keyed by its index as a + string — `"0"` is the whole match — which is how a pattern without named groups + is read: + ``` + m = capture(.message, r'(\w+):.*', numeric_groups: true) + if m != null { + .level = m["1"] + } + ``` + `after(value, separator)` — returns everything after the first occurrence of `separator`; the value is returned unchanged when the separator is not found. @@ -209,4 +218,35 @@ are required; named arguments are optional and fall back to their defaults: .shard = between(.log, "[", "]") ``` ++ `find_all(value, pattern, group: 0, limit: -1)` — returns every match of + `pattern` as an array, or an empty array when nothing matches. `group` selects + a capture group (`0` is the whole match) and a negative `limit` collects all + occurrences: + ``` + .ids = find_all(.log, r'id=(\w+)', group: 1) + ``` + ++ `join(value, separator)` — joins an array of strings into one string. Only + strings are joined; convert other values with `string()` first: + ``` + .extracted = join(find_all(.message, r're\d+', limit: 2), ",") # "re1,re2" + ``` + ++ `trim(value, cutset)`, `trim_left(value, cutset)`, + `trim_right(value, cutset)` — strip characters from both ends, the start or + the end. `cutset` is a *set of characters*, not a substring: `trim_right(v, "ms")` + removes every trailing `m` and `s`. + ``` + .message = trim_right(.message, "\n") + ``` + ++ `slice(value, start, end: null)` — returns the part of the string between two + positions, counted in characters. Both positions may be negative to count from + the end, and positions outside the string are clamped rather than raising an + error. `end` defaults to the end of the string: + ``` + .head = slice(.message, 0, end: 10) # first 10 characters + .tail = slice(.message, -5) # last 5 characters + ``` +
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/action/transform/doc_examples_test.go b/plugin/action/transform/doc_examples_test.go index 6d228e4ee..de7c9c4c8 100644 --- a/plugin/action/transform/doc_examples_test.go +++ b/plugin/action/transform/doc_examples_test.go @@ -58,6 +58,15 @@ func TestDocExamplesCompile(t *testing.T) { `.message = after(.log, " - ")`, `.level = before(.log, " ")`, `.shard = between(.log, "[", "]")`, + `m = capture(.message, r'(\w+):.*', numeric_groups: true) + if m != null { + .level = m["1"] + }`, + `.ids = find_all(.log, r'id=(\w+)', group: 1)`, + `.extracted = join(find_all(.message, r're\d+', limit: 2), ",")`, + `.message = trim_right(.message, "\n")`, + `.head = slice(.message, 0, end: 10) + .tail = slice(.message, -5)`, } for i, src := range snippets { diff --git a/plugin/action/transform/stdlib/capture.go b/plugin/action/transform/stdlib/capture.go index cadba51d7..85e86baa3 100644 --- a/plugin/action/transform/stdlib/capture.go +++ b/plugin/action/transform/stdlib/capture.go @@ -1,6 +1,8 @@ package stdlib import ( + "strconv" + "github.com/ozontech/file.d/plugin/action/transform/core" ) @@ -8,6 +10,10 @@ import ( // its named capture groups (keyed by group name). Unnamed groups are ignored. // When value does not match, it returns null so callers can guard the result // with `if m != null { ... }`. +// +// With numeric_groups enabled every group is additionally keyed by its index as +// a string - "0" is the whole match, "1" the first group and so on - which is +// how a pattern without named groups is read: m["1"]. type capture struct{} func (capture) Name() string { return "capture" } @@ -24,12 +30,19 @@ func (capture) Params() []Parameter { Description: "The regular expression; named groups (?P...) become object keys.", AcceptedKinds: []core.ValueKind{core.KindRegex}, }, + { + Name: "numeric_groups", + Description: `Also key every group by its index as a string: "0" is the whole match, "1" the first group.`, + Default: core.BoolValue{V: false}, + AcceptedKinds: []core.ValueKind{core.KindBool}, + }, } } func (capture) Call(args map[string]core.Value) (core.Value, error) { value := args["value"].(core.StringValue) re := args["pattern"].(core.RegexValue).V + numericGroups := args["numeric_groups"].(core.BoolValue).V match := re.FindStringSubmatch(value.V) if match == nil { @@ -39,6 +52,9 @@ func (capture) Call(args map[string]core.Value) (core.Value, error) { names := re.SubexpNames() groups := make(map[string]core.Value) for i, name := range names { + if numericGroups { + groups[strconv.Itoa(i)] = core.StringValue{V: match[i]} + } // names[0] is the whole match (always unnamed); unnamed groups have "". if i == 0 || name == "" { continue diff --git a/plugin/action/transform/stdlib/capture_test.go b/plugin/action/transform/stdlib/capture_test.go index a6ebde8d8..303a1c249 100644 --- a/plugin/action/transform/stdlib/capture_test.go +++ b/plugin/action/transform/stdlib/capture_test.go @@ -10,11 +10,15 @@ import ( ) func callCapture(value, pattern string) (core.Value, error) { + return callCaptureNamed(value, pattern, nil) +} + +func callCaptureNamed(value, pattern string, named map[string]core.Value) (core.Value, error) { re := regexp.MustCompile(pattern) - return capture{}.Call(map[string]core.Value{ - "value": core.StringValue{V: value}, - "pattern": core.RegexValue{V: re}, - }) + return callFn(capture{}, []core.Value{ + core.StringValue{V: value}, + core.RegexValue{V: re}, + }, named) } func TestCapture(t *testing.T) { @@ -77,3 +81,61 @@ func TestCapture(t *testing.T) { assert.Len(t, obj.V, 1, "unnamed group must not appear in the result") }) } + +func TestCaptureNumericGroups(t *testing.T) { + t.Parallel() + + enabled := map[string]core.Value{"numeric_groups": core.BoolValue{V: true}} + + t.Run("positional_groups_are_keyed_by_index", func(t *testing.T) { + t.Parallel() + + // modify README: ${message|re("service=(\S+) exec took (\d+\.?\d*(?:ms|s|m|h))",-1,[2],",")} + got, err := callCaptureNamed( + "service=service-test-1 exec took 200ms", + `service=(\S+) exec took (\d+\.?\d*(?:ms|s|m|h))`, + enabled, + ) + require.NoError(t, err) + obj, ok := got.(core.ObjectValue) + require.True(t, ok) + + assert.Equal(t, core.StringValue{V: "service=service-test-1 exec took 200ms"}, obj.V["0"], + `"0" is the whole match`) + assert.Equal(t, core.StringValue{V: "service-test-1"}, obj.V["1"]) + assert.Equal(t, core.StringValue{V: "200ms"}, obj.V["2"]) + }) + + t.Run("named_groups_are_keyed_both_ways", func(t *testing.T) { + t.Parallel() + + got, err := callCaptureNamed("abc123", `(?P[a-z]+)(\d+)`, enabled) + require.NoError(t, err) + obj, ok := got.(core.ObjectValue) + require.True(t, ok) + + assert.Equal(t, core.StringValue{V: "abc"}, obj.V["letters"]) + assert.Equal(t, core.StringValue{V: "abc"}, obj.V["1"]) + assert.Equal(t, core.StringValue{V: "123"}, obj.V["2"], "unnamed groups become reachable") + }) + + t.Run("off_by_default", func(t *testing.T) { + t.Parallel() + + got, err := callCapture("abc123", `(?P[a-z]+)(\d+)`) + require.NoError(t, err) + obj, ok := got.(core.ObjectValue) + require.True(t, ok) + + assert.NotContains(t, obj.V, "0") + assert.NotContains(t, obj.V, "1") + }) + + t.Run("no_match_still_returns_null", func(t *testing.T) { + t.Parallel() + + got, err := callCaptureNamed("nothing here", `^(INFO)$`, enabled) + require.NoError(t, err) + assert.Equal(t, core.NullValue{}, got) + }) +} diff --git a/plugin/action/transform/stdlib/find_all.go b/plugin/action/transform/stdlib/find_all.go new file mode 100644 index 000000000..648dbc077 --- /dev/null +++ b/plugin/action/transform/stdlib/find_all.go @@ -0,0 +1,80 @@ +package stdlib + +import ( + "fmt" + + "github.com/ozontech/file.d/plugin/action/transform/core" +) + +// find_all collects every match of a pattern in a string and returns them as an +// array. It is the multi-occurrence counterpart of capture: capture describes +// one match by its groups, find_all describes many matches by a single group. +// +// find_all(.log, r'\d+') -> ["1", "42"] +// join(find_all(.log, r're\d+', limit: 2), ",") -> "re1,re2" +// +// When nothing matches an empty array is returned, so the result is always safe +// to iterate or join. +type findAll struct{} + +func (findAll) Name() string { return "find_all" } + +func (findAll) Params() []Parameter { + return []Parameter{ + { + Name: "value", + Description: "The string to search.", + AcceptedKinds: []core.ValueKind{core.KindString}, + }, + { + Name: "pattern", + Description: "The regular expression to search for.", + AcceptedKinds: []core.ValueKind{core.KindRegex}, + }, + { + Name: "group", + Description: "Index of the capture group to collect; 0 is the whole match.", + Default: core.IntegerValue{V: 0}, + AcceptedKinds: []core.ValueKind{core.KindInteger}, + }, + { + Name: "limit", + Description: "Maximum number of matches to collect; a negative limit collects all of them.", + Default: core.IntegerValue{V: -1}, + AcceptedKinds: []core.ValueKind{core.KindInteger}, + }, + } +} + +func (findAll) Call(args map[string]core.Value) (core.Value, error) { + value := args["value"].(core.StringValue).V + re := args["pattern"].(core.RegexValue).V + group := int(args["group"].(core.IntegerValue).V) + limit := int(args["limit"].(core.IntegerValue).V) + + if group < 0 || group > re.NumSubexp() { + return core.NullValue{}, fmt.Errorf( + "group %d is out of range: pattern has %d capture groups", group, re.NumSubexp()) + } + + // A negative limit means "all matches"; Go spells that -1 exactly. + if limit < 0 { + limit = -1 + } + + // The index form is used rather than FindAllStringSubmatch so that a group + // that did not participate in a match (-1) can be told apart from a group + // that matched an empty string, and skipped. + matches := re.FindAllStringSubmatchIndex(value, limit) + + result := make([]core.Value, 0, len(matches)) + for _, match := range matches { + start, end := match[group*2], match[group*2+1] + if start == -1 || end == -1 { + continue + } + result = append(result, core.StringValue{V: value[start:end]}) + } + + return core.ArrayValue{V: result}, nil +} diff --git a/plugin/action/transform/stdlib/find_all_test.go b/plugin/action/transform/stdlib/find_all_test.go new file mode 100644 index 000000000..d6b253330 --- /dev/null +++ b/plugin/action/transform/stdlib/find_all_test.go @@ -0,0 +1,89 @@ +package stdlib + +import ( + "regexp" + "testing" + + "github.com/ozontech/file.d/plugin/action/transform/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func callFindAll(t *testing.T, value, pattern string, named map[string]core.Value) []string { + t.Helper() + + got, err := callFn(findAll{}, []core.Value{ + core.StringValue{V: value}, + core.RegexValue{V: regexp.MustCompile(pattern)}, + }, named) + require.NoError(t, err) + + arr, ok := got.(core.ArrayValue) + require.True(t, ok, "expected an array, got %s", got.Kind()) + + out := make([]string, len(arr.V)) + for i, el := range arr.V { + out[i] = el.(core.StringValue).V + } + return out +} + +func TestFindAll(t *testing.T) { + t.Parallel() + + t.Run("whole_matches", func(t *testing.T) { + t.Parallel() + assert.Equal(t, []string{"re1", "re2", "re3", "re4"}, + callFindAll(t, "re1 re2 re3 re4", `re\d+`, nil)) + }) + + t.Run("capture_group", func(t *testing.T) { + t.Parallel() + assert.Equal(t, []string{"1", "2"}, + callFindAll(t, "re1 re2", `re(\d+)`, map[string]core.Value{"group": core.IntegerValue{V: 1}})) + }) + + t.Run("limit", func(t *testing.T) { + t.Parallel() + + assert.Equal(t, []string{"re1", "re2"}, + callFindAll(t, "re1 re2 re3 re4", `re\d+`, map[string]core.Value{"limit": core.IntegerValue{V: 2}}), + "a positive limit caps the number of matches") + + assert.Empty(t, callFindAll(t, "re1 re2", `re\d+`, map[string]core.Value{"limit": core.IntegerValue{V: 0}}), + "limit 0 extracts nothing, as in the modify plugin") + + assert.Equal(t, []string{"re1", "re2"}, + callFindAll(t, "re1 re2", `re\d+`, map[string]core.Value{"limit": core.IntegerValue{V: -5}}), + "any negative limit means all matches") + }) + + t.Run("no_match", func(t *testing.T) { + t.Parallel() + assert.Empty(t, callFindAll(t, "nothing here", `re\d+`, nil)) + assert.Empty(t, callFindAll(t, "", `re\d+`, nil), "empty input") + }) + + t.Run("skips_groups_that_did_not_participate", func(t *testing.T) { + t.Parallel() + // Group 1 participates only in the "a" match, not in the "b" one. + assert.Equal(t, []string{"a"}, + callFindAll(t, "a b", `(a)|(b)`, map[string]core.Value{"group": core.IntegerValue{V: 1}})) + }) + + t.Run("group_out_of_range", func(t *testing.T) { + t.Parallel() + + call := func(group int64) error { + _, err := callFn(findAll{}, []core.Value{ + core.StringValue{V: "re1"}, + core.RegexValue{V: regexp.MustCompile(`re(\d+)`)}, + }, map[string]core.Value{"group": core.IntegerValue{V: group}}) + return err + } + + require.ErrorContains(t, call(2), "out of range") + require.ErrorContains(t, call(-1), "out of range") + require.NoError(t, call(1), "the last valid group is accepted") + }) +} diff --git a/plugin/action/transform/stdlib/join.go b/plugin/action/transform/stdlib/join.go new file mode 100644 index 000000000..7addeeb51 --- /dev/null +++ b/plugin/action/transform/stdlib/join.go @@ -0,0 +1,51 @@ +package stdlib + +import ( + "fmt" + "strings" + + "github.com/ozontech/file.d/plugin/action/transform/core" +) + +// join concatenates an array of strings into one string: +// +// join(find_all(.log, r're\d+'), ",") -> "re1,re2" +// +// Only strings are joined; like the + operator, other kinds must be converted +// with string() first, so a surprising array does not silently produce a +// surprising field. +type join struct{} + +func (join) Name() string { return "join" } + +func (join) Params() []Parameter { + return []Parameter{ + { + Name: "value", + Description: "The array of strings to join.", + AcceptedKinds: []core.ValueKind{core.KindArray}, + }, + { + Name: "separator", + Description: `Placed between elements; pass "" to concatenate them directly.`, + AcceptedKinds: []core.ValueKind{core.KindString}, + }, + } +} + +func (join) Call(args map[string]core.Value) (core.Value, error) { + elements := args["value"].(core.ArrayValue).V + separator := args["separator"].(core.StringValue).V + + parts := make([]string, len(elements)) + for i, el := range elements { + s, ok := el.(core.StringValue) + if !ok { + return core.NullValue{}, fmt.Errorf( + "element %d: expected string, got %s", i, el.Kind()) + } + parts[i] = s.V + } + + return core.StringValue{V: strings.Join(parts, separator)}, nil +} diff --git a/plugin/action/transform/stdlib/join_test.go b/plugin/action/transform/stdlib/join_test.go new file mode 100644 index 000000000..f8d9e51b2 --- /dev/null +++ b/plugin/action/transform/stdlib/join_test.go @@ -0,0 +1,42 @@ +package stdlib + +import ( + "testing" + + "github.com/ozontech/file.d/plugin/action/transform/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJoin(t *testing.T) { + t.Parallel() + + call := func(elements []core.Value, separator string) (core.Value, error) { + return callFn(join{}, []core.Value{core.ArrayValue{V: elements}, str(separator)}, nil) + } + mustCall := func(t *testing.T, elements []core.Value, separator string) string { + t.Helper() + got, err := call(elements, separator) + require.NoError(t, err) + return got.(core.StringValue).V + } + + t.Run("separator", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "re1,re2", mustCall(t, []core.Value{str("re1"), str("re2")}, ",")) + assert.Equal(t, "re1re2", mustCall(t, []core.Value{str("re1"), str("re2")}, "")) + }) + + t.Run("edge_cases", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "", mustCall(t, nil, ","), "empty array") + assert.Equal(t, "only", mustCall(t, []core.Value{str("only")}, ","), "no separator for one element") + assert.Equal(t, ",", mustCall(t, []core.Value{str(""), str("")}, ","), "empty elements are kept") + }) + + t.Run("rejects_non_strings", func(t *testing.T) { + t.Parallel() + _, err := call([]core.Value{str("a"), core.IntegerValue{V: 1}}, ",") + require.ErrorContains(t, err, "element 1: expected string, got integer") + }) +} diff --git a/plugin/action/transform/stdlib/registry.go b/plugin/action/transform/stdlib/registry.go index 1f97fc9f2..42060fa06 100644 --- a/plugin/action/transform/stdlib/registry.go +++ b/plugin/action/transform/stdlib/registry.go @@ -21,6 +21,12 @@ func init() { registry.mustRegister(after{}) registry.mustRegister(before{}) registry.mustRegister(between{}) + registry.mustRegister(findAll{}) + registry.mustRegister(join{}) + registry.mustRegister(trim{}) + registry.mustRegister(trimLeft{}) + registry.mustRegister(trimRight{}) + registry.mustRegister(slice{}) } func GetRegistry() *Registry { diff --git a/plugin/action/transform/stdlib/resolve_test.go b/plugin/action/transform/stdlib/resolve_test.go index 123c1f0fd..19b40d94b 100644 --- a/plugin/action/transform/stdlib/resolve_test.go +++ b/plugin/action/transform/stdlib/resolve_test.go @@ -35,6 +35,21 @@ func resolveArgs(t *testing.T, fn Function, positional []core.Value, named map[s return c.Resolve(positional, named) } +// callFn invokes fn the way the interpreter does: arguments are bound through +// the compiled signature, so omitted named parameters get their declared +// defaults instead of being absent from the map. +func callFn(fn Function, positional []core.Value, named map[string]core.Value) (core.Value, error) { + c, err := compile(fn) + if err != nil { + return nil, err + } + resolved, err := c.Resolve(positional, named) + if err != nil { + return nil, err + } + return c.Call(resolved) +} + func TestJoinKinds(t *testing.T) { t.Parallel() diff --git a/plugin/action/transform/stdlib/slice.go b/plugin/action/transform/stdlib/slice.go new file mode 100644 index 000000000..f99ffa245 --- /dev/null +++ b/plugin/action/transform/stdlib/slice.go @@ -0,0 +1,71 @@ +package stdlib + +import ( + "github.com/ozontech/file.d/plugin/action/transform/core" +) + +// slice returns the part of a string between two positions, counted in +// characters rather than bytes so multi-byte text is never cut in half: +// +// slice(.message, 0, end: 10) -> the first 10 characters +// slice(.message, -5) -> the last 5 characters +// +// start is inclusive, end is exclusive, and both may be negative to count from +// the right. Positions outside the string are clamped instead of raising an +// error, so a shorter-than-expected line passes through whole. +type slice struct{} + +func (slice) Name() string { return "slice" } + +func (slice) Params() []Parameter { + return []Parameter{ + { + Name: "value", + Description: "The string to slice.", + AcceptedKinds: []core.ValueKind{core.KindString}, + }, + { + Name: "start", + Description: "Inclusive start position; negative counts from the end of the string.", + AcceptedKinds: []core.ValueKind{core.KindInteger}, + }, + { + Name: "end", + Description: "Exclusive end position; negative counts from the end. Defaults to the end of the string.", + Default: core.NullValue{}, + AcceptedKinds: []core.ValueKind{core.KindInteger, core.KindNull}, + }, + } +} + +func (slice) Call(args map[string]core.Value) (core.Value, error) { + runes := []rune(args["value"].(core.StringValue).V) + length := len(runes) + + start := clampIndex(int(args["start"].(core.IntegerValue).V), length) + + end := length + if v, ok := args["end"].(core.IntegerValue); ok { + end = clampIndex(int(v.V), length) + } + + if end <= start { + return core.StringValue{V: ""}, nil + } + return core.StringValue{V: string(runes[start:end])}, nil +} + +// clampIndex resolves a possibly negative position against length and pins the +// result to [0, length]. +func clampIndex(i, length int) int { + if i < 0 { + i += length + } + if i < 0 { + return 0 + } + if i > length { + return length + } + return i +} diff --git a/plugin/action/transform/stdlib/slice_test.go b/plugin/action/transform/stdlib/slice_test.go new file mode 100644 index 000000000..81f5ee5db --- /dev/null +++ b/plugin/action/transform/stdlib/slice_test.go @@ -0,0 +1,55 @@ +package stdlib + +import ( + "testing" + + "github.com/ozontech/file.d/plugin/action/transform/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSlice(t *testing.T) { + t.Parallel() + + call := func(value string, start int64, end ...int64) string { + named := map[string]core.Value{} + if len(end) > 0 { + named["end"] = core.IntegerValue{V: end[0]} + } + got, err := callFn(slice{}, []core.Value{str(value), core.IntegerValue{V: start}}, named) + require.NoError(t, err) + return got.(core.StringValue).V + } + + t.Run("modify_cut_equivalents", func(t *testing.T) { + t.Parallel() + // modify README: ${message|cut("first",10)} and ${message|cut("last",5)} + assert.Equal(t, "some loooo", call("some looooooooooooong data", 0, 10)) + assert.Equal(t, " data", call("some looooooooooooong data", -5)) + }) + + t.Run("positions", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "califrag", call("Supercalifragilistic", 5, 13)) + assert.Equal(t, "listic", call("Supercalifragilistic", -6)) + assert.Equal(t, "fragilistic", call("Supercalifragilistic", 9)) + assert.Equal(t, "Supercalifragilistic", call("Supercalifragilistic", 0)) + }) + + t.Run("out_of_range_is_clamped", func(t *testing.T) { + t.Parallel() + assert.Equal(t, "short", call("short", 0, 100), "end past the string returns what is there") + assert.Equal(t, "short", call("short", -100), "start before the string starts at 0") + assert.Equal(t, "", call("short", 100), "start past the string yields nothing") + assert.Equal(t, "", call("short", 3, 1), "end before start yields nothing") + assert.Equal(t, "", call("short", 2, 2), "empty range") + assert.Equal(t, "", call("", 0, 5), "empty input") + }) + + t.Run("counts_characters_not_bytes", func(t *testing.T) { + t.Parallel() + // The modify plugin's cut counts bytes and would split these runes. + assert.Equal(t, "привет", call("привет мир", 0, 6)) + assert.Equal(t, "мир", call("привет мир", -3)) + }) +} diff --git a/plugin/action/transform/stdlib/trim.go b/plugin/action/transform/stdlib/trim.go new file mode 100644 index 000000000..e789ef876 --- /dev/null +++ b/plugin/action/transform/stdlib/trim.go @@ -0,0 +1,70 @@ +package stdlib + +import ( + "strings" + + "github.com/ozontech/file.d/plugin/action/transform/core" +) + +// The trim family strips characters from the ends of a string: +// +// trim(.message, " ") -> both ends +// trim_left(.message, " ") -> leading only +// trim_right(.message, "\n") -> trailing only +// +// cutset is a *set of characters*, not a substring: trim_right(v, "ms") removes +// every trailing "m" and "s", not the suffix "ms". Side is part of the function +// name rather than a mode argument so that a typo is caught when the program is +// compiled instead of on the first event that reaches it. + +type trim struct{} + +func (trim) Name() string { return "trim" } + +func (trim) Params() []Parameter { return trimParams("both ends") } + +func (trim) Call(args map[string]core.Value) (core.Value, error) { + value, cutset := trimArgs(args) + return core.StringValue{V: strings.Trim(value, cutset)}, nil +} + +type trimLeft struct{} + +func (trimLeft) Name() string { return "trim_left" } + +func (trimLeft) Params() []Parameter { return trimParams("the start") } + +func (trimLeft) Call(args map[string]core.Value) (core.Value, error) { + value, cutset := trimArgs(args) + return core.StringValue{V: strings.TrimLeft(value, cutset)}, nil +} + +type trimRight struct{} + +func (trimRight) Name() string { return "trim_right" } + +func (trimRight) Params() []Parameter { return trimParams("the end") } + +func (trimRight) Call(args map[string]core.Value) (core.Value, error) { + value, cutset := trimArgs(args) + return core.StringValue{V: strings.TrimRight(value, cutset)}, nil +} + +func trimParams(side string) []Parameter { + return []Parameter{ + { + Name: "value", + Description: "The string to trim.", + AcceptedKinds: []core.ValueKind{core.KindString}, + }, + { + Name: "cutset", + Description: "Set of characters to remove from " + side + "; any character in the set is stripped.", + AcceptedKinds: []core.ValueKind{core.KindString}, + }, + } +} + +func trimArgs(args map[string]core.Value) (string, string) { + return args["value"].(core.StringValue).V, args["cutset"].(core.StringValue).V +} diff --git a/plugin/action/transform/stdlib/trim_test.go b/plugin/action/transform/stdlib/trim_test.go new file mode 100644 index 000000000..70cd67113 --- /dev/null +++ b/plugin/action/transform/stdlib/trim_test.go @@ -0,0 +1,54 @@ +package stdlib + +import ( + "testing" + + "github.com/ozontech/file.d/plugin/action/transform/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func trimCaller(t *testing.T, fn Function) func(value, cutset string) string { + t.Helper() + + return func(value, cutset string) string { + got, err := callFn(fn, []core.Value{str(value), str(cutset)}, nil) + require.NoError(t, err) + return got.(core.StringValue).V + } +} + +func TestTrim(t *testing.T) { + t.Parallel() + call := trimCaller(t, trim{}) + + assert.Equal(t, "data", call(" data ", " ")) + assert.Equal(t, "data", call("xxdataxx", "x")) + assert.Equal(t, "", call("aaa", "a"), "everything trimmed away") + assert.Equal(t, "", call("", " "), "empty input") + assert.Equal(t, "data", call("data", ""), "empty cutset changes nothing") +} + +func TestTrimLeft(t *testing.T) { + t.Parallel() + call := trimCaller(t, trimLeft{}) + + assert.Equal(t, "data ", call(" data ", " ")) + assert.Equal(t, "compaction", call("] compaction", "] "), + "cutset is a set of characters, so both ] and space are stripped") + assert.Equal(t, "data", call("data", "x"), "nothing to trim") +} + +func TestTrimRight(t *testing.T) { + t.Parallel() + call := trimCaller(t, trimRight{}) + + // modify README: ${message|trim("right","\n")} + assert.Equal(t, + `{"service":"service-test-1","took":"200ms"}`, + call(`{"service":"service-test-1","took":"200ms"}`+"\n", "\n")) + + assert.Equal(t, " data", call(" data ", " ")) + assert.Equal(t, "took: 200", call("took: 200ms", "ms"), + "cutset is a set of characters, not a suffix") +} diff --git a/plugin/action/transform/transform.go b/plugin/action/transform/transform.go index afdb4719c..d09d0295d 100644 --- a/plugin/action/transform/transform.go +++ b/plugin/action/transform/transform.go @@ -198,9 +198,9 @@ are required; named arguments are optional and fall back to their defaults: .msg = "code is " + string(.code) ``` -+ `capture(value, pattern)` — matches the string against a regular expression - and returns an object of its named groups `(?P...)`, or `null` when the - value does not match (unnamed groups are ignored): ++ `capture(value, pattern, numeric_groups: false)` — matches the string against a + regular expression and returns an object of its named groups `(?P...)`, + or `null` when the value does not match (unnamed groups are ignored): ``` m = capture(.log, r'^(?P\S+)\s+(?P.+)$') if m != null { @@ -208,6 +208,15 @@ are required; named arguments are optional and fall back to their defaults: .message = m.message } ``` + With `numeric_groups: true` every group is additionally keyed by its index as a + string — `"0"` is the whole match — which is how a pattern without named groups + is read: + ``` + m = capture(.message, r'(\w+):.*', numeric_groups: true) + if m != null { + .level = m["1"] + } + ``` + `after(value, separator)` — returns everything after the first occurrence of `separator`; the value is returned unchanged when the separator is not found. @@ -226,6 +235,37 @@ are required; named arguments are optional and fall back to their defaults: ``` .shard = between(.log, "[", "]") ``` + ++ `find_all(value, pattern, group: 0, limit: -1)` — returns every match of + `pattern` as an array, or an empty array when nothing matches. `group` selects + a capture group (`0` is the whole match) and a negative `limit` collects all + occurrences: + ``` + .ids = find_all(.log, r'id=(\w+)', group: 1) + ``` + ++ `join(value, separator)` — joins an array of strings into one string. Only + strings are joined; convert other values with `string()` first: + ``` + .extracted = join(find_all(.message, r're\d+', limit: 2), ",") # "re1,re2" + ``` + ++ `trim(value, cutset)`, `trim_left(value, cutset)`, + `trim_right(value, cutset)` — strip characters from both ends, the start or + the end. `cutset` is a *set of characters*, not a substring: `trim_right(v, "ms")` + removes every trailing `m` and `s`. + ``` + .message = trim_right(.message, "\n") + ``` + ++ `slice(value, start, end: null)` — returns the part of the string between two + positions, counted in characters. Both positions may be negative to count from + the end, and positions outside the string are clamped rather than raising an + error. `end` defaults to the end of the string: + ``` + .head = slice(.message, 0, end: 10) # first 10 characters + .tail = slice(.message, -5) # last 5 characters + ``` }*/ type Plugin struct {