Add functions in transform plugin to cover modify plugin

plugin/action/transform/README.md

@@ -180,16 +180,25 @@ are required; named arguments are optional and fall back to their defaults:
   .msg = "code is " + string(.code)
   ```
 
-+ `capture(value, pattern)` — matches the string against a regular expression
-  and returns an object of its named groups `(?P<name>...)`, or `null` when the
-  value does not match (unnamed groups are ignored):
++ `capture(value, pattern, numeric_groups: false)` — matches the string against a
+  regular expression and returns an object of its named groups `(?P<name>...)`,
+  or `null` when the value does not match (unnamed groups are ignored):
   ```
   m = capture(.log, r'^(?P<level>\S+)\s+(?P<message>.+)$')
   if m != null {
     .level = m.level
     .message = m.message
   }
   ```
+  With `numeric_groups: true` every group is additionally keyed by its index as a
+  string — `"0"` is the whole match — which is how a pattern without named groups
+  is read:
+  ```
+  m = capture(.message, r'(\w+):.*', numeric_groups: true)
+  if m != null {
+    .level = m["1"]
+  }
+  ```
 
 + `after(value, separator)` — returns everything after the first occurrence of
   `separator`; the value is returned unchanged when the separator is not found.
@@ -209,4 +218,35 @@ are required; named arguments are optional and fall back to their defaults:
   .shard = between(.log, "[", "]")
   ```
 
++ `find_all(value, pattern, group: 0, limit: -1)` — returns every match of
+  `pattern` as an array, or an empty array when nothing matches. `group` selects
+  a capture group (`0` is the whole match) and a negative `limit` collects all
+  occurrences:
+  ```
+  .ids = find_all(.log, r'id=(\w+)', group: 1)
+  ```
+
++ `join(value, separator)` — joins an array of strings into one string. Only
+  strings are joined; convert other values with `string()` first:
+  ```
+  .extracted = join(find_all(.message, r're\d+', limit: 2), ",")   # "re1,re2"
+  ```
+
++ `trim(value, cutset)`, `trim_left(value, cutset)`,
+  `trim_right(value, cutset)` — strip characters from both ends, the start or
+  the end. `cutset` is a *set of characters*, not a substring: `trim_right(v, "ms")`
+  removes every trailing `m` and `s`.
+  ```
+  .message = trim_right(.message, "\n")
+  ```
+
++ `slice(value, start, end: null)` — returns the part of the string between two
+  positions, counted in characters. Both positions may be negative to count from
+  the end, and positions outside the string are clamped rather than raising an
+  error. `end` defaults to the end of the string:
+  ```
+  .head = slice(.message, 0, end: 10)   # first 10 characters
+  .tail = slice(.message, -5)           # last 5 characters
+  ```
+
 <br>*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)*

plugin/action/transform/doc_examples_test.go

@@ -58,6 +58,15 @@ func TestDocExamplesCompile(t *testing.T) {
 		`.message = after(.log, " - ")`,
 		`.level = before(.log, " ")`,
 		`.shard = between(.log, "[", "]")`,
+		`m = capture(.message, r'(\w+):.*', numeric_groups: true)
+		if m != null {
+		  .level = m["1"]
+		}`,
+		`.ids = find_all(.log, r'id=(\w+)', group: 1)`,
+		`.extracted = join(find_all(.message, r're\d+', limit: 2), ",")`,
+		`.message = trim_right(.message, "\n")`,
+		`.head = slice(.message, 0, end: 10)
+		.tail = slice(.message, -5)`,
 	}
 
 	for i, src := range snippets {

plugin/action/transform/stdlib/capture.go

@@ -1,13 +1,19 @@
 package stdlib
 
 import (
+	"strconv"
+
 	"github.com/ozontech/file.d/plugin/action/transform/core"
 )
 
 // capture matches value against a regular expression and returns an object of
 // its named capture groups (keyed by group name). Unnamed groups are ignored.
 // When value does not match, it returns null so callers can guard the result
 // with `if m != null { ... }`.
+//
+// With numeric_groups enabled every group is additionally keyed by its index as
+// a string - "0" is the whole match, "1" the first group and so on - which is
+// how a pattern without named groups is read: m["1"].
 type capture struct{}
 
 func (capture) Name() string { return "capture" }
@@ -24,12 +30,19 @@ func (capture) Params() []Parameter {
 			Description:   "The regular expression; named groups (?P<name>...) become object keys.",
 			AcceptedKinds: []core.ValueKind{core.KindRegex},
 		},
+		{
+			Name:          "numeric_groups",
+			Description:   `Also key every group by its index as a string: "0" is the whole match, "1" the first group.`,
+			Default:       core.BoolValue{V: false},
+			AcceptedKinds: []core.ValueKind{core.KindBool},
+		},
 	}
 }
 
 func (capture) Call(args map[string]core.Value) (core.Value, error) {
 	value := args["value"].(core.StringValue)
 	re := args["pattern"].(core.RegexValue).V
+	numericGroups := args["numeric_groups"].(core.BoolValue).V
 
 	match := re.FindStringSubmatch(value.V)
 	if match == nil {
@@ -39,6 +52,9 @@ func (capture) Call(args map[string]core.Value) (core.Value, error) {
 	names := re.SubexpNames()
 	groups := make(map[string]core.Value)
 	for i, name := range names {
+		if numericGroups {
+			groups[strconv.Itoa(i)] = core.StringValue{V: match[i]}
+		}
 		// names[0] is the whole match (always unnamed); unnamed groups have "".
 		if i == 0 || name == "" {
 			continue

plugin/action/transform/stdlib/capture_test.go

@@ -10,11 +10,15 @@ import (
 )
 
 func callCapture(value, pattern string) (core.Value, error) {
+	return callCaptureNamed(value, pattern, nil)
+}
+
+func callCaptureNamed(value, pattern string, named map[string]core.Value) (core.Value, error) {
 	re := regexp.MustCompile(pattern)
-	return capture{}.Call(map[string]core.Value{
-		"value":   core.StringValue{V: value},
-		"pattern": core.RegexValue{V: re},
-	})
+	return callFn(capture{}, []core.Value{
+		core.StringValue{V: value},
+		core.RegexValue{V: re},
+	}, named)
 }
 
 func TestCapture(t *testing.T) {
@@ -77,3 +81,61 @@ func TestCapture(t *testing.T) {
 		assert.Len(t, obj.V, 1, "unnamed group must not appear in the result")
 	})
 }
+
+func TestCaptureNumericGroups(t *testing.T) {
+	t.Parallel()
+
+	enabled := map[string]core.Value{"numeric_groups": core.BoolValue{V: true}}
+
+	t.Run("positional_groups_are_keyed_by_index", func(t *testing.T) {
+		t.Parallel()
+
+		// modify README: ${message|re("service=(\S+) exec took (\d+\.?\d*(?:ms|s|m|h))",-1,[2],",")}
+		got, err := callCaptureNamed(
+			"service=service-test-1 exec took 200ms",
+			`service=(\S+) exec took (\d+\.?\d*(?:ms|s|m|h))`,
+			enabled,
+		)
+		require.NoError(t, err)
+		obj, ok := got.(core.ObjectValue)
+		require.True(t, ok)
+
+		assert.Equal(t, core.StringValue{V: "service=service-test-1 exec took 200ms"}, obj.V["0"],
+			`"0" is the whole match`)
+		assert.Equal(t, core.StringValue{V: "service-test-1"}, obj.V["1"])
+		assert.Equal(t, core.StringValue{V: "200ms"}, obj.V["2"])
+	})
+
+	t.Run("named_groups_are_keyed_both_ways", func(t *testing.T) {
+		t.Parallel()
+
+		got, err := callCaptureNamed("abc123", `(?P<letters>[a-z]+)(\d+)`, enabled)
+		require.NoError(t, err)
+		obj, ok := got.(core.ObjectValue)
+		require.True(t, ok)
+
+		assert.Equal(t, core.StringValue{V: "abc"}, obj.V["letters"])
+		assert.Equal(t, core.StringValue{V: "abc"}, obj.V["1"])
+		assert.Equal(t, core.StringValue{V: "123"}, obj.V["2"], "unnamed groups become reachable")
+	})
+
+	t.Run("off_by_default", func(t *testing.T) {
+		t.Parallel()
+
+		got, err := callCapture("abc123", `(?P<letters>[a-z]+)(\d+)`)
+		require.NoError(t, err)
+		obj, ok := got.(core.ObjectValue)
+		require.True(t, ok)
+
+		assert.NotContains(t, obj.V, "0")
+		assert.NotContains(t, obj.V, "1")
+	})
+
+	t.Run("no_match_still_returns_null", func(t *testing.T) {
+		t.Parallel()
+
+		got, err := callCaptureNamed("nothing here", `^(INFO)$`, enabled)
+		require.NoError(t, err)
+		assert.Equal(t, core.NullValue{}, got)
+	})
+}

plugin/action/transform/stdlib/find_all.go

@@ -0,0 +1,80 @@
+package stdlib
+
+import (
+	"fmt"
+
+	"github.com/ozontech/file.d/plugin/action/transform/core"
+)
+
+// find_all collects every match of a pattern in a string and returns them as an
+// array. It is the multi-occurrence counterpart of capture: capture describes
+// one match by its groups, find_all describes many matches by a single group.
+//
+//	find_all(.log, r'\d+')                     -> ["1", "42"]
+//	join(find_all(.log, r're\d+', limit: 2), ",") -> "re1,re2"
+//
+// When nothing matches an empty array is returned, so the result is always safe
+// to iterate or join.
+type findAll struct{}
+
+func (findAll) Name() string { return "find_all" }
+
+func (findAll) Params() []Parameter {
+	return []Parameter{
+		{
+			Name:          "value",
+			Description:   "The string to search.",
+			AcceptedKinds: []core.ValueKind{core.KindString},
+		},
+		{
+			Name:          "pattern",
+			Description:   "The regular expression to search for.",
+			AcceptedKinds: []core.ValueKind{core.KindRegex},
+		},
+		{
+			Name:          "group",
+			Description:   "Index of the capture group to collect; 0 is the whole match.",
+			Default:       core.IntegerValue{V: 0},
+			AcceptedKinds: []core.ValueKind{core.KindInteger},
+		},
+		{
+			Name:          "limit",
+			Description:   "Maximum number of matches to collect; a negative limit collects all of them.",
+			Default:       core.IntegerValue{V: -1},
+			AcceptedKinds: []core.ValueKind{core.KindInteger},
+		},
+	}
+}
+
+func (findAll) Call(args map[string]core.Value) (core.Value, error) {
+	value := args["value"].(core.StringValue).V
+	re := args["pattern"].(core.RegexValue).V
+	group := int(args["group"].(core.IntegerValue).V)
+	limit := int(args["limit"].(core.IntegerValue).V)
+
+	if group < 0 || group > re.NumSubexp() {
+		return core.NullValue{}, fmt.Errorf(
+			"group %d is out of range: pattern has %d capture groups", group, re.NumSubexp())
+	}
+
+	// A negative limit means "all matches"; Go spells that -1 exactly.
+	if limit < 0 {
+		limit = -1
+	}
+
+	// The index form is used rather than FindAllStringSubmatch so that a group
+	// that did not participate in a match (-1) can be told apart from a group
+	// that matched an empty string, and skipped.
+	matches := re.FindAllStringSubmatchIndex(value, limit)
+
+	result := make([]core.Value, 0, len(matches))
+	for _, match := range matches {
+		start, end := match[group*2], match[group*2+1]
+		if start == -1 || end == -1 {
+			continue
+		}
+		result = append(result, core.StringValue{V: value[start:end]})
+	}
+
+	return core.ArrayValue{V: result}, nil
+}

plugin/action/transform/stdlib/find_all_test.go

@@ -0,0 +1,89 @@
+package stdlib
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/ozontech/file.d/plugin/action/transform/core"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func callFindAll(t *testing.T, value, pattern string, named map[string]core.Value) []string {
+	t.Helper()
+
+	got, err := callFn(findAll{}, []core.Value{
+		core.StringValue{V: value},
+		core.RegexValue{V: regexp.MustCompile(pattern)},
+	}, named)
+	require.NoError(t, err)
+
+	arr, ok := got.(core.ArrayValue)
+	require.True(t, ok, "expected an array, got %s", got.Kind())
+
+	out := make([]string, len(arr.V))
+	for i, el := range arr.V {
+		out[i] = el.(core.StringValue).V
+	}
+	return out
+}
+
+func TestFindAll(t *testing.T) {
+	t.Parallel()
+
+	t.Run("whole_matches", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, []string{"re1", "re2", "re3", "re4"},
+			callFindAll(t, "re1 re2 re3 re4", `re\d+`, nil))
+	})
+
+	t.Run("capture_group", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, []string{"1", "2"},
+			callFindAll(t, "re1 re2", `re(\d+)`, map[string]core.Value{"group": core.IntegerValue{V: 1}}))
+	})
+
+	t.Run("limit", func(t *testing.T) {
+		t.Parallel()
+
+		assert.Equal(t, []string{"re1", "re2"},
+			callFindAll(t, "re1 re2 re3 re4", `re\d+`, map[string]core.Value{"limit": core.IntegerValue{V: 2}}),
+			"a positive limit caps the number of matches")
+
+		assert.Empty(t, callFindAll(t, "re1 re2", `re\d+`, map[string]core.Value{"limit": core.IntegerValue{V: 0}}),
+			"limit 0 extracts nothing, as in the modify plugin")
+
+		assert.Equal(t, []string{"re1", "re2"},
+			callFindAll(t, "re1 re2", `re\d+`, map[string]core.Value{"limit": core.IntegerValue{V: -5}}),
+			"any negative limit means all matches")
+	})
+
+	t.Run("no_match", func(t *testing.T) {
+		t.Parallel()
+		assert.Empty(t, callFindAll(t, "nothing here", `re\d+`, nil))
+		assert.Empty(t, callFindAll(t, "", `re\d+`, nil), "empty input")
+	})
+
+	t.Run("skips_groups_that_did_not_participate", func(t *testing.T) {
+		t.Parallel()
+		// Group 1 participates only in the "a" match, not in the "b" one.
+		assert.Equal(t, []string{"a"},
+			callFindAll(t, "a b", `(a)|(b)`, map[string]core.Value{"group": core.IntegerValue{V: 1}}))
+	})
+
+	t.Run("group_out_of_range", func(t *testing.T) {
+		t.Parallel()
+
+		call := func(group int64) error {
+			_, err := callFn(findAll{}, []core.Value{
+				core.StringValue{V: "re1"},
+				core.RegexValue{V: regexp.MustCompile(`re(\d+)`)},
+			}, map[string]core.Value{"group": core.IntegerValue{V: group}})
+			return err
+		}
+
+		require.ErrorContains(t, call(2), "out of range")
+		require.ErrorContains(t, call(-1), "out of range")
+		require.NoError(t, call(1), "the last valid group is accepted")
+	})
+}