diff --git a/plugin/action/transform/README.md b/plugin/action/transform/README.md index 7373931e8..1084cf55f 100755 --- a/plugin/action/transform/README.md +++ b/plugin/action/transform/README.md @@ -209,4 +209,21 @@ are required; named arguments are optional and fall back to their defaults: .shard = between(.log, "[", "]") ``` ++ `lookup(value, table, default: )` — translates a value through a + table of replacements. It turns enumeration codes into readable names without + a chain of `if`s: + ``` + api_key = {"0": "produce", "1": "fetch", "2": "offsets"} + .kafka_request_api_key = lookup(.kafka_request_api_key, api_key) + ``` + Keys are matched by their string form, so the number `0` and the string `"0"` + are the same key — JSON writes codes both ways. A value that is not in the + table is returned unchanged; pass `default:` to replace it instead: + ``` + .severity = lookup(.status, {"500": "crit", "400": "warn"}, default: "ok") + ``` + A table written as a literal is built once at startup, not per event, so a + large table costs no more than a small one. Keep it in a variable when the + same table is used more than once. +
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/action/transform/compiler/fold.go b/plugin/action/transform/compiler/fold.go new file mode 100644 index 000000000..e83287731 --- /dev/null +++ b/plugin/action/transform/compiler/fold.go @@ -0,0 +1,141 @@ +package compiler + +import ( + "github.com/ozontech/file.d/plugin/action/transform/core" +) + +// Constant folding computes the value of constant sub-expressions once, at +// startup, and stores it in a core.ConstExpr that replaces them in the AST. +// +// It exists so that data prepared for a function - a lookup table written as an +// object literal, most of all - is built a single time instead of on every +// event: core.ObjectExpr.Eval allocates a fresh map on every evaluation. +// +// Folding runs inside the validation walk (see validateExpr) rather than in a +// pass of its own, because that walk already visits every node and already +// prepares nodes in place: it compiles regex literals and parses timestamp +// literals. Order matters - a node is folded only after the walk has visited +// it, so RegexLit.Compiled and TimestampLit.Parsed are populated by then. +// +// An expression whose evaluation fails is never folded. It stays in the AST and +// fails per event exactly as it does today, so folding cannot turn a runtime +// error into a startup error. + +// tryFold returns a core.ConstExpr holding the value of expr when that value +// can be computed at startup, and expr itself otherwise. +func tryFold(expr core.Expr) core.Expr { + if _, ok := expr.(*core.ConstExpr); ok { + return expr + } + v, ok := foldConst(expr) + if !ok { + return expr + } + return &core.ConstExpr{Node: core.NewNode(expr.Pos()), V: v} +} + +// foldConst computes the value of expr, reporting whether it is constant. +// A composite is constant only when every part of it is. +func foldConst(expr core.Expr) (core.Value, bool) { + switch e := expr.(type) { + case *core.ConstExpr: + return e.V, true + + case *core.IntLit: + return core.IntegerValue{V: e.Value}, true + case *core.FloatLit: + return core.FloatValue{V: e.Value}, true + case *core.StringLit: + return core.StringValue{V: e.Value}, true + case *core.BoolLit: + return core.BoolValue{V: e.Value}, true + case *core.NullLit: + return core.NullValue{}, true + + case *core.RegexLit: + // Compiled by the validation walk; nil means this node has not been + // visited yet, so leave it to be evaluated at runtime. + if e.Compiled == nil { + return nil, false + } + return core.RegexValue{V: e.Compiled}, true + + case *core.TimestampLit: + // Parsed by the validation walk. A zero time means the node has not + // been visited; t'0001-01-01T00:00:00Z' simply misses the optimization. + if e.Parsed.IsZero() { + return nil, false + } + return core.TimestampValue{V: e.Parsed}, true + + case *core.ArrayExpr: + elements := make([]core.Value, len(e.Elements)) + for i, el := range e.Elements { + v, ok := foldConst(el) + if !ok { + return nil, false + } + elements[i] = v + } + return core.ArrayValue{V: elements}, true + + case *core.ObjectExpr: + // Duplicate keys are rejected by validateExpr before we get here. + pairs := make(map[string]core.Value, len(e.Pairs)) + for _, kv := range e.Pairs { + v, ok := foldConst(kv.Value) + if !ok { + return nil, false + } + pairs[kv.Key] = v + } + return core.ObjectValue{V: pairs}, true + + case *core.UnaryExpr: + operand, ok := foldConst(e.Operand) + if !ok { + return nil, false + } + return evalConst(&core.UnaryExpr{ + Node: core.NewNode(e.Pos()), + Op: e.Op, + Operand: constNode(e.Operand, operand), + }) + + case *core.BinaryExpr: + left, ok := foldConst(e.Left) + if !ok { + return nil, false + } + right, ok := foldConst(e.Right) + if !ok { + return nil, false + } + return evalConst(&core.BinaryExpr{ + Node: core.NewNode(e.Pos()), + Op: e.Op, + Left: constNode(e.Left, left), + Right: constNode(e.Right, right), + }) + } + + return nil, false +} + +func constNode(from core.Expr, v core.Value) core.Expr { + return &core.ConstExpr{Node: core.NewNode(from.Pos()), V: v} +} + +// evalConst evaluates an operator node whose operands are already core.ConstExpr. +// Those ignore the evaluation context, and UnaryExpr/BinaryExpr touch the context +// only to evaluate their operands, so a nil context is never dereferenced. +// +// A failing expression - `1 / 0`, `"a" + 1` - is reported as not constant so it +// keeps failing at runtime instead of breaking pipeline startup. +func evalConst(expr core.Expr) (core.Value, bool) { + v, err := expr.Eval(nil) + if err != nil { + return nil, false + } + return v, true +} diff --git a/plugin/action/transform/compiler/fold_test.go b/plugin/action/transform/compiler/fold_test.go new file mode 100644 index 000000000..b5d38c7a6 --- /dev/null +++ b/plugin/action/transform/compiler/fold_test.go @@ -0,0 +1,324 @@ +package compiler + +import ( + "strings" + "testing" + + "github.com/ozontech/file.d/plugin/action/transform/core" + "github.com/ozontech/file.d/plugin/action/transform/stdlib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// prepare compiles src and runs the validation walk, which is where folding +// happens. It returns the prepared AST. +func prepare(t *testing.T, src string) []core.Expr { + t.Helper() + + exprs := compileN(t, src) + require.NoError(t, stdlibValidate(exprs)) + return exprs +} + +func stdlibValidate(exprs []core.Expr) error { + return ValidateCalls(exprs, stdlib.GetRegistry()) +} + +// dumpPrepared renders the AST after preparation, so folded nodes show up as Const(...). +func dumpPrepared(t *testing.T, src string) string { + t.Helper() + + dumps := make([]string, 0) + for _, e := range prepare(t, src) { + dumps = append(dumps, core.DumpAST(e, 0)) + } + return strings.Join(dumps, "\n") +} + +// assertNotFolded checks that the value of the single assignment in src was +// left in the AST. It reaches into a call argument when argIndex is not -1. +func assertNotFolded(t *testing.T, src string, argIndex int) { + t.Helper() + + exprs := prepare(t, src) + require.Len(t, exprs, 1) + + assign, ok := exprs[0].(*core.AssignExpr) + require.True(t, ok, "expected an assignment, got %T", exprs[0]) + + target := assign.Value + if argIndex >= 0 { + call, ok := target.(*core.CallExpr) + require.True(t, ok, "expected a call, got %T", target) + require.Greater(t, len(call.Args), argIndex) + target = call.Args[argIndex].Value + } + + _, folded := target.(*core.ConstExpr) + assert.False(t, folded, "expression must not be folded: %s", core.DumpAST(exprs[0], 0)) +} + +// A failing constant expression must stay in the AST so it keeps failing per +// event. Folding must never turn a runtime error into a startup error. +func TestFoldSkipsFailingExpressions(t *testing.T) { + tests := []struct { + name string + src string + arg int + }{ + {name: "division by zero", src: `x = 1 / 0`, arg: -1}, + {name: "modulo by zero", src: `x = 1 % 0`, arg: -1}, + {name: "string plus integer", src: `x = "a" + 1`, arg: -1}, + {name: "negate a string", src: `x = -"a"`, arg: -1}, + {name: "failing element in an array", src: `x = [1, 1 / 0]`, arg: -1}, + {name: "failing value in an object", src: `x = {a: 1 / 0}`, arg: -1}, + {name: "failing argument", src: `x = upcase(1 / 0)`, arg: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assertNotFolded(t, tt.src, tt.arg) + }) + } +} + +func TestFoldConstantArguments(t *testing.T) { + tests := []goldenCase{ + { + name: "object literal argument", + src: `.x = lookup(.x, {"0": "produce", "1": "fetch"})`, + want: ` +Assign + Path(.x) + Call(lookup) + Path(.x) + Const({"0": "produce", "1": "fetch"})`, + }, + { + name: "nested composite literal", + src: `.x = lookup(.x, {a: [1, {b: "c"}]})`, + want: ` +Assign + Path(.x) + Call(lookup) + Path(.x) + Const({"a": [1, {"b": "c"}]})`, + }, + { + name: "negative numbers do not block folding", + src: `.x = lookup(.x, {a: -1, b: 2 + 3})`, + want: ` +Assign + Path(.x) + Call(lookup) + Path(.x) + Const({"a": -1, "b": 5})`, + }, + { + name: "named argument", + src: `.x = lookup(.x, {}, default: "unknown")`, + want: ` +Assign + Path(.x) + Call(lookup) + Path(.x) + Const({}) + named(default:) + Const("unknown")`, + }, + { + name: "regex literal folds after it is compiled", + src: `m = capture(.log, r'(?P\d+)')`, + want: ` +Assign + Ident(m) + Call(capture) + Path(.log) + Const(r'(?P\d+)')`, + }, + { + name: "timestamp literal folds after it is parsed", + src: `.x = lookup(.x, {}, default: t'2024-01-15T10:30:00Z')`, + want: ` +Assign + Path(.x) + Call(lookup) + Path(.x) + Const({}) + named(default:) + Const(2024-01-15T10:30:00Z)`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, strings.TrimSpace(tt.want), dumpPrepared(t, tt.src)) + }) + } +} + +// The `table = { ... }` idiom must not rebuild the object on every event. +func TestFoldAssignmentValue(t *testing.T) { + tests := []goldenCase{ + { + name: "object literal", + src: `t = {"0": "produce"}`, + want: ` +Assign + Ident(t) + Const({"0": "produce"})`, + }, + { + name: "array literal", + src: `t = [1, "two", true]`, + want: ` +Assign + Ident(t) + Const([1, "two", true])`, + }, + { + name: "field write", + src: `.x = 1 + 2`, + want: ` +Assign + Path(.x) + Const(3)`, + }, + } + + runFoldGolden(t, tests) +} + +// Anything that depends on the event or on per-event state is not constant. +func TestFoldSkipsNonConstant(t *testing.T) { + tests := []struct { + name string + src string + arg int + }{ + {name: "event field", src: `t = .table`, arg: -1}, + {name: "variable", src: `t = other`, arg: -1}, + {name: "object holding a field read", src: `t = {a: .x}`, arg: -1}, + {name: "array holding a variable", src: `t = [1, other]`, arg: -1}, + // A call is never folded, even with constant arguments: functions are + // not required to be pure. + {name: "function call", src: `t = upcase("a")`, arg: -1}, + {name: "table argument read from the event", src: `.x = lookup(.x, .table)`, arg: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assertNotFolded(t, tt.src, tt.arg) + }) + } +} + +// Folding reaches into nested blocks, since the validation walk does. +func TestFoldInsideBlocks(t *testing.T) { + tests := []goldenCase{ + { + name: "if branches", + src: `if .a { + t = {a: 1} +} else { + t = {b: 2} +}`, + want: ` +If + condition: + Path(.a) + then: + Assign + Ident(t) + Const({"a": 1}) + else: + Assign + Ident(t) + Const({"b": 2})`, + }, + { + name: "for body", + src: `for _, item in .items { + t = {a: 1} +}`, + want: ` +For(index="", item="item") + iter: + Path(.items) + body: + Assign + Ident(t) + Const({"a": 1})`, + }, + } + + runFoldGolden(t, tests) +} + +func runFoldGolden(t *testing.T, tests []goldenCase) { + t.Helper() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, strings.TrimSpace(tt.want), dumpPrepared(t, tt.src)) + }) + } +} + +// A folded value is shared by every event and every processor goroutine, so it +// must survive being written through. +func TestFoldedValueIsNotMutatedByAssignment(t *testing.T) { + exprs := prepare(t, ` +t = {a: 1} +t.a = 2 +`) + require.Len(t, exprs, 2) + + assign, ok := exprs[0].(*core.AssignExpr) + require.True(t, ok) + folded, ok := assign.Value.(*core.ConstExpr) + require.True(t, ok, "the table must have been folded") + + table, ok := folded.V.(core.ObjectValue) + require.True(t, ok) + + // Run the program twice against the same AST; the second run must see the + // original table, not the one the first run wrote to. + for range 2 { + ctx := newFoldTestContext() + for _, e := range exprs { + _, err := e.Eval(ctx) + require.NoError(t, err) + } + assert.Equal(t, core.IntegerValue{V: 1}, table.V["a"], "the folded table was mutated") + } +} + +// foldTestContext is the smallest EvalContext that supports variables; the +// mutation test needs no event target. +type foldTestContext struct { + vars map[string]core.Value +} + +func newFoldTestContext() *foldTestContext { + return &foldTestContext{vars: make(map[string]core.Value)} +} + +func (c *foldTestContext) GetVar(name string) (core.Value, bool) { + v, ok := c.vars[name] + return v, ok +} +func (c *foldTestContext) SetVar(name string, v core.Value) { c.vars[name] = v } +func (c *foldTestContext) DeleteVar(name string) { delete(c.vars, name) } +func (c *foldTestContext) GetTarget() core.Target { return nil } +func (c *foldTestContext) CallFunc(_ core.Position, _ string, _ []core.Value, _ map[string]core.Value) (core.Value, error) { + return core.NullValue{}, nil +} diff --git a/plugin/action/transform/compiler/validate.go b/plugin/action/transform/compiler/validate.go index 88e9f397c..9a3e86699 100644 --- a/plugin/action/transform/compiler/validate.go +++ b/plugin/action/transform/compiler/validate.go @@ -12,6 +12,10 @@ import ( // ValidateCalls walks the AST and checks that every function call refers to // a function that exists in the registry. // This is a lightweight static check - argument types are validated at runtime +// +// The walk also prepares nodes in place, so it must run before the program is +// evaluated: regex literals are compiled, timestamp literals are parsed, and +// constant sub-expressions are folded into core.ConstExpr (see fold.go). func ValidateCalls(exprs []core.Expr, registry *stdlib.Registry) error { for _, expr := range exprs { if err := validateExpr(expr, registry); err != nil { @@ -36,6 +40,11 @@ func validateExpr(expr core.Expr, registry *stdlib.Registry) error { return err } } + // Fold after the walk: an argument holding a regex or timestamp literal + // is only constant once validateExpr has prepared it. + for i := range e.Args { + e.Args[i].Value = tryFold(e.Args[i].Value) + } case *core.BinaryExpr: if err := validateExpr(e.Left, registry); err != nil { return err @@ -44,7 +53,12 @@ func validateExpr(expr core.Expr, registry *stdlib.Registry) error { case *core.UnaryExpr: return validateExpr(e.Operand, registry) case *core.AssignExpr: - return validateExpr(e.Value, registry) + if err := validateExpr(e.Value, registry); err != nil { + return err + } + // Folding the right-hand side keeps `table = { ... }` at the top of a + // program from rebuilding the object on every event. + e.Value = tryFold(e.Value) case *core.IndexExpr: if err := validateExpr(e.Object, registry); err != nil { return err diff --git a/plugin/action/transform/core/ast.go b/plugin/action/transform/core/ast.go index c2e9a7f1b..f326893a7 100644 --- a/plugin/action/transform/core/ast.go +++ b/plugin/action/transform/core/ast.go @@ -3,6 +3,7 @@ package core import ( "fmt" "regexp" + "sort" "strings" "time" ) @@ -72,6 +73,11 @@ type TimestampLit struct { Parsed time.Time } +type ConstExpr struct { + Node + V Value +} + type IdentExpr struct { Node Name string @@ -166,6 +172,40 @@ type ForExpr struct { Body []Expr } +// dumpValue renders a value for debug output, sorting object keys so that the +// dump of a folded constant is stable across runs. +// +// Value.String() deliberately does not sort: it sits on the runtime path of +// string(value), and debug output is the only place that needs a stable order. +func dumpValue(v Value) string { + switch t := v.(type) { + case StringValue: + return fmt.Sprintf("%q", t.V) + + case ArrayValue: + parts := make([]string, len(t.V)) + for i, el := range t.V { + parts[i] = dumpValue(el) + } + return "[" + strings.Join(parts, ", ") + "]" + + case ObjectValue: + keys := make([]string, 0, len(t.V)) + for k := range t.V { + keys = append(keys, k) + } + sort.Strings(keys) + + parts := make([]string, 0, len(t.V)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%q: %s", k, dumpValue(t.V[k]))) + } + return "{" + strings.Join(parts, ", ") + "}" + } + + return v.String() +} + // DumpAST returns a human-readable representation of the AST. // Use only for debug func DumpAST(expr Expr, depth int) string { @@ -187,6 +227,8 @@ func DumpAST(expr Expr, depth int) string { return fmt.Sprintf("%sRegexLit(%q)", pad, e.Pattern) case *TimestampLit: return fmt.Sprintf("%sTimestampLit(%q)", pad, e.Value) + case *ConstExpr: + return fmt.Sprintf("%sConst(%s)", pad, dumpValue(e.V)) case *IdentExpr: return fmt.Sprintf("%sIdent(%s)", pad, e.Name) diff --git a/plugin/action/transform/core/eval.go b/plugin/action/transform/core/eval.go index 426ac9a8c..2621276bb 100644 --- a/plugin/action/transform/core/eval.go +++ b/plugin/action/transform/core/eval.go @@ -52,6 +52,10 @@ func (e *TimestampLit) Eval(_ EvalContext) (Value, error) { return TimestampValue{V: e.Parsed}, nil } +func (e *ConstExpr) Eval(_ EvalContext) (Value, error) { + return e.V, nil +} + func (e *IdentExpr) Eval(ctx EvalContext) (Value, error) { if val, ok := ctx.GetVar(e.Name); ok { return val, nil @@ -189,9 +193,9 @@ func (e *BinaryExpr) Eval(ctx EvalContext) (Value, error) { switch e.Op { case "==": - return BoolValue{V: left.Equal(right)}, nil + return BoolValue{V: resolve(left).Equal(resolve(right))}, nil case "!=": - return BoolValue{V: !left.Equal(right)}, nil + return BoolValue{V: !resolve(left).Equal(resolve(right))}, nil case "+": return evalAdd(e.Pos(), resolve(left), resolve(right)) case "-", "*", "/", "%": diff --git a/plugin/action/transform/core/value.go b/plugin/action/transform/core/value.go index b1bca102b..c7cc1422c 100644 --- a/plugin/action/transform/core/value.go +++ b/plugin/action/transform/core/value.go @@ -162,8 +162,13 @@ func (v TimestampValue) Equal(other Value) bool { return ok && v.V.Equal(o.V) } +// Equal compares the decoded node, not its text. Comparing String() forms made +// an event field never equal an object or array literal, because a node renders +// as raw JSON ({"a":1}) while an ObjectValue renders with spaces ({"a": 1}). +// Resolving both sides also keeps equality type-aware, so the number 1 does not +// equal the string "1". func (v JSONNodeValue) Equal(other Value) bool { - return v.String() == other.String() + return resolve(v).Equal(resolve(other)) } func (NullValue) String() string { return "null" } @@ -202,6 +207,9 @@ func (v ArrayValue) String() string { return "[" + strings.Join(parts, ", ") + "]" } +// Key order follows Go map iteration and is therefore not stable. This is on +// the runtime path of string(value), so it does not pay for a sort; debug +// output that needs a stable rendering uses dumpValue instead. func (v ObjectValue) String() string { parts := make([]string, 0, len(v.V)) for k, val := range v.V { diff --git a/plugin/action/transform/core/value_test.go b/plugin/action/transform/core/value_test.go new file mode 100644 index 000000000..044c4acce --- /dev/null +++ b/plugin/action/transform/core/value_test.go @@ -0,0 +1,140 @@ +package core + +import ( + "testing" + + insaneJSON "github.com/ozontech/insane-json" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Debug output must not depend on Go map iteration order, so that the dump of a +// folded constant is reproducible. Value.String() is deliberately unsorted - +// it sits on the runtime path of string(value) - so the sort lives in dumpValue. +func TestDumpConstIsStable(t *testing.T) { + t.Parallel() + + expr := &ConstExpr{V: ObjectValue{V: map[string]Value{ + "z": StringValue{V: "z"}, + "a": IntegerValue{V: 1}, + "m": ArrayValue{V: []Value{BoolValue{V: true}}}, + "n": ObjectValue{V: map[string]Value{"b": IntegerValue{V: 2}, "a": IntegerValue{V: 1}}}, + }}} + + want := `Const({"a": 1, "m": [true], "n": {"a": 1, "b": 2}, "z": "z"})` + for range 20 { + assert.Equal(t, want, DumpAST(expr, 0)) + } +} + +// A JSON node is compared by its decoded value, not by its text. Comparing +// String() forms made a field never equal an object or array literal, since a +// node renders as raw JSON while an ObjectValue renders with spaces. +func TestJSONNodeEquality(t *testing.T) { + t.Parallel() + + root, err := insaneJSON.DecodeString( + `{"num":1,"num2":1,"str":"1","obj":{"a":1},"arr":[1,2],"flt":1.0,"yes":true,"nul":null}`) + require.NoError(t, err) + defer insaneJSON.Release(root) + + node := func(field string) JSONNodeValue { + return JSONNodeValue{N: root.Dig(field)} + } + + tests := []struct { + name string + left Value + right Value + want bool + }{ + { + name: "object field equals an object literal", + left: node("obj"), + right: ObjectValue{V: map[string]Value{"a": IntegerValue{V: 1}}}, + want: true, + }, + { + name: "object field differs from another object", + left: node("obj"), + right: ObjectValue{V: map[string]Value{"a": IntegerValue{V: 2}}}, + want: false, + }, + { + name: "array field equals an array literal", + left: node("arr"), + right: ArrayValue{V: []Value{IntegerValue{V: 1}, IntegerValue{V: 2}}}, + want: true, + }, + { + name: "array order matters", + left: node("arr"), + right: ArrayValue{V: []Value{IntegerValue{V: 2}, IntegerValue{V: 1}}}, + want: false, + }, + { + name: "number equals an integer literal", + left: node("num"), + right: IntegerValue{V: 1}, + want: true, + }, + { + // Integers and floats compare numerically, as documented. + name: "integer equals a float", + left: node("num"), + right: FloatValue{V: 1.0}, + want: true, + }, + { + // Equality is type-aware; it used to be true via the string forms. + name: "number does not equal the same digits as a string", + left: node("num"), + right: StringValue{V: "1"}, + want: false, + }, + { + name: "string equals a string literal", + left: node("str"), + right: StringValue{V: "1"}, + want: true, + }, + { + name: "two nodes of equal value", + left: node("num"), + right: node("num2"), + want: true, + }, + { + name: "two nodes of different type", + left: node("num"), + right: node("str"), + want: false, + }, + { + name: "bool field", + left: node("yes"), + right: BoolValue{V: true}, + want: true, + }, + { + name: "null field", + left: node("nul"), + right: NullValue{}, + want: true, + }, + { + name: "missing field is null", + left: JSONNodeValue{N: nil}, + right: NullValue{}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, tt.left.Equal(tt.right)) + }) + } +} diff --git a/plugin/action/transform/doc_examples_test.go b/plugin/action/transform/doc_examples_test.go index 6d228e4ee..f88e5f58f 100644 --- a/plugin/action/transform/doc_examples_test.go +++ b/plugin/action/transform/doc_examples_test.go @@ -58,6 +58,9 @@ func TestDocExamplesCompile(t *testing.T) { `.message = after(.log, " - ")`, `.level = before(.log, " ")`, `.shard = between(.log, "[", "]")`, + `api_key = {"0": "produce", "1": "fetch", "2": "offsets"} + .kafka_request_api_key = lookup(.kafka_request_api_key, api_key)`, + `.severity = lookup(.status, {"500": "crit", "400": "warn"}, default: "ok")`, } for i, src := range snippets { diff --git a/plugin/action/transform/stdlib/lookup.go b/plugin/action/transform/stdlib/lookup.go new file mode 100644 index 000000000..e3370839f --- /dev/null +++ b/plugin/action/transform/stdlib/lookup.go @@ -0,0 +1,81 @@ +package stdlib + +import ( + "github.com/ozontech/file.d/plugin/action/transform/core" +) + +// lookup translates a value through a table: +// +// api = {"0": "produce", "1": "fetch", "2": "offsets"} +// .kafka_request_api_key = lookup(.kafka_request_api_key, api) +// +// The table is an ordinary object, so a constant one is built once at startup by +// the compiler's constant folding and a call costs a single map access. +// +// Keys are compared by their canonical string form, which makes the numeric 0 +// and the string "0" the same key - JSON writes enumeration codes both ways. +// +// A value that is not in the table is returned unchanged, the same forgiving +// behavior as the substring family, so an unexpected code passes through +// instead of being dropped. Pass `default:` to override it. +type lookup struct{} + +// keepInput is the default of the `default` parameter. A named parameter is +// made optional by having a non-nil default, and "return the input unchanged" +// is not expressible as an ordinary value - null already means an explicit +// null. This sentinel never leaves Call. +type keepInput struct{} + +// paramDefault is the name of the miss-replacement parameter, used both in the +// signature and to read the resolved argument. +const paramDefault = "default" + +func (keepInput) Kind() core.ValueKind { return core.KindNull } +func (keepInput) AsBool() bool { return false } +func (keepInput) Equal(other core.Value) bool { + _, ok := other.(keepInput) + return ok +} +func (keepInput) String() string { return "" } + +func (lookup) Name() string { return "lookup" } + +func (lookup) Params() []Parameter { + return []Parameter{ + { + Name: "value", + Description: "The value to translate; it is matched against the table keys by its string form.", + AcceptedKinds: []core.ValueKind{ + core.KindNull, + core.KindBool, + core.KindInteger, + core.KindFloat, + core.KindString, + }, + }, + { + Name: "table", + Description: "An object mapping keys to their replacements.", + AcceptedKinds: []core.ValueKind{core.KindObject}, + }, + { + Name: paramDefault, + Description: "Returned when the value is not in the table; by default the value is returned unchanged.", + Default: keepInput{}, + }, + } +} + +func (lookup) Call(args map[string]core.Value) (core.Value, error) { + value := args["value"] + table := args["table"].(core.ObjectValue) + + if mapped, ok := table.V[value.String()]; ok { + return mapped, nil + } + + if _, unchanged := args[paramDefault].(keepInput); unchanged { + return value, nil + } + return args[paramDefault], nil +} diff --git a/plugin/action/transform/stdlib/lookup_test.go b/plugin/action/transform/stdlib/lookup_test.go new file mode 100644 index 000000000..24dcfc48d --- /dev/null +++ b/plugin/action/transform/stdlib/lookup_test.go @@ -0,0 +1,195 @@ +package stdlib + +import ( + "testing" + + "github.com/ozontech/file.d/plugin/action/transform/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func kafkaAPIKeys() core.ObjectValue { + return core.ObjectValue{V: map[string]core.Value{ + "0": core.StringValue{V: "produce"}, + "1": core.StringValue{V: "fetch"}, + "2": core.StringValue{V: "offsets"}, + }} +} + +// callLookup resolves the arguments the way the interpreter does, so defaults +// and kind checks are exercised rather than bypassed. +func callLookup(t *testing.T, positional []core.Value, named map[string]core.Value) (core.Value, error) { + t.Helper() + + args, err := resolveArgs(t, lookup{}, positional, named) + if err != nil { + return nil, err + } + return lookup{}.Call(args) +} + +func TestLookup(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + value core.Value + want core.Value + }{ + { + name: "string key hits", + value: core.StringValue{V: "1"}, + want: core.StringValue{V: "fetch"}, + }, + { + // JSON writes enumeration codes both ways; both must hit. + name: "integer key hits the same entry", + value: core.IntegerValue{V: 1}, + want: core.StringValue{V: "fetch"}, + }, + { + name: "zero is not confused with a miss", + value: core.IntegerValue{V: 0}, + want: core.StringValue{V: "produce"}, + }, + { + name: "miss returns the value unchanged", + value: core.StringValue{V: "77"}, + want: core.StringValue{V: "77"}, + }, + { + name: "missing field stays null", + value: core.NullValue{}, + want: core.NullValue{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := callLookup(t, []core.Value{tt.value, kafkaAPIKeys()}, nil) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestLookupDefault(t *testing.T) { + t.Parallel() + + t.Run("replaces a miss", func(t *testing.T) { + t.Parallel() + + got, err := callLookup(t, + []core.Value{core.StringValue{V: "77"}, kafkaAPIKeys()}, + map[string]core.Value{"default": core.StringValue{V: "unknown"}}, + ) + require.NoError(t, err) + assert.Equal(t, core.StringValue{V: "unknown"}, got) + }) + + t.Run("does not replace a hit", func(t *testing.T) { + t.Parallel() + + got, err := callLookup(t, + []core.Value{core.StringValue{V: "1"}, kafkaAPIKeys()}, + map[string]core.Value{"default": core.StringValue{V: "unknown"}}, + ) + require.NoError(t, err) + assert.Equal(t, core.StringValue{V: "fetch"}, got) + }) + + // An explicit null must be distinguishable from an omitted default, which + // is why the omitted case uses a sentinel rather than null. + t.Run("explicit null differs from omitted", func(t *testing.T) { + t.Parallel() + + got, err := callLookup(t, + []core.Value{core.StringValue{V: "77"}, kafkaAPIKeys()}, + map[string]core.Value{"default": core.NullValue{}}, + ) + require.NoError(t, err) + assert.Equal(t, core.NullValue{}, got) + }) + + t.Run("a null entry is returned as null", func(t *testing.T) { + t.Parallel() + + table := core.ObjectValue{V: map[string]core.Value{"a": core.NullValue{}}} + got, err := callLookup(t, []core.Value{core.StringValue{V: "a"}, table}, nil) + require.NoError(t, err) + assert.Equal(t, core.NullValue{}, got) + }) +} + +func TestLookupTableValuesOfAnyType(t *testing.T) { + t.Parallel() + + table := core.ObjectValue{V: map[string]core.Value{ + "a": core.IntegerValue{V: 7}, + "b": core.ArrayValue{V: []core.Value{core.StringValue{V: "x"}}}, + }} + + got, err := callLookup(t, []core.Value{core.StringValue{V: "a"}, table}, nil) + require.NoError(t, err) + assert.Equal(t, core.IntegerValue{V: 7}, got) + + got, err = callLookup(t, []core.Value{core.StringValue{V: "b"}, table}, nil) + require.NoError(t, err) + assert.Equal(t, core.ArrayValue{V: []core.Value{core.StringValue{V: "x"}}}, got) +} + +func TestLookupRejectsBadArguments(t *testing.T) { + t.Parallel() + + t.Run("table must be an object", func(t *testing.T) { + t.Parallel() + + _, err := callLookup(t, + []core.Value{core.StringValue{V: "1"}, core.StringValue{V: "not a table"}}, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "expected object") + assert.Contains(t, err.Error(), "got string") + }) + + t.Run("composite values cannot be keys", func(t *testing.T) { + t.Parallel() + + _, err := callLookup(t, + []core.Value{core.ArrayValue{V: nil}, kafkaAPIKeys()}, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "got array") + }) + + t.Run("table is required", func(t *testing.T) { + t.Parallel() + + _, err := callLookup(t, []core.Value{core.StringValue{V: "1"}}, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing required argument") + }) + + t.Run("unknown named argument", func(t *testing.T) { + t.Parallel() + + _, err := callLookup(t, + []core.Value{core.StringValue{V: "1"}, kafkaAPIKeys()}, + map[string]core.Value{"fallback": core.StringValue{V: "x"}}, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown argument") + }) +} + +// The table is shared across events, so a lookup must never write to it. +func TestLookupDoesNotMutateTable(t *testing.T) { + t.Parallel() + + table := kafkaAPIKeys() + + _, err := callLookup(t, []core.Value{core.StringValue{V: "77"}, table}, nil) + require.NoError(t, err) + + assert.Len(t, table.V, 3) +} diff --git a/plugin/action/transform/stdlib/registry.go b/plugin/action/transform/stdlib/registry.go index 1f97fc9f2..f7e6f9740 100644 --- a/plugin/action/transform/stdlib/registry.go +++ b/plugin/action/transform/stdlib/registry.go @@ -21,6 +21,7 @@ func init() { registry.mustRegister(after{}) registry.mustRegister(before{}) registry.mustRegister(between{}) + registry.mustRegister(lookup{}) } func GetRegistry() *Registry { diff --git a/plugin/action/transform/transform.go b/plugin/action/transform/transform.go index afdb4719c..b863e4e5f 100644 --- a/plugin/action/transform/transform.go +++ b/plugin/action/transform/transform.go @@ -226,6 +226,23 @@ are required; named arguments are optional and fall back to their defaults: ``` .shard = between(.log, "[", "]") ``` + ++ `lookup(value, table, default: )` — translates a value through a + table of replacements. It turns enumeration codes into readable names without + a chain of `if`s: + ``` + api_key = {"0": "produce", "1": "fetch", "2": "offsets"} + .kafka_request_api_key = lookup(.kafka_request_api_key, api_key) + ``` + Keys are matched by their string form, so the number `0` and the string `"0"` + are the same key — JSON writes codes both ways. A value that is not in the + table is returned unchanged; pass `default:` to replace it instead: + ``` + .severity = lookup(.status, {"500": "crit", "400": "warn"}, default: "ok") + ``` + A table written as a literal is built once at startup, not per event, so a + large table costs no more than a small one. Keep it in a variable when the + same table is used more than once. }*/ type Plugin struct { diff --git a/plugin/action/transform/transform_test.go b/plugin/action/transform/transform_test.go index f4307ee27..030991b08 100644 --- a/plugin/action/transform/transform_test.go +++ b/plugin/action/transform/transform_test.go @@ -433,6 +433,77 @@ func TestLanguage(t *testing.T) { }, }, }, + { + name: "equality_against_event_fields", + source: ` + .obj_eq = .obj == {a: 1} + .arr_eq = .arr == [1, 2] + .num_eq = .num == 1 + .flt_eq = .num == 1.0 + .mixed = .num == "1" + .nodes = .num == .num2 + .differ = .obj == {a: 2} + `, + events: []eventCase{ + { + in: `{"obj":{"a":1},"arr":[1,2],"num":1,"num2":1}`, + fields: map[string]string{ + // Composite literals used to never match a field, because + // the two sides were compared as text in different formats. + "obj_eq": "true", + "arr_eq": "true", + "num_eq": "true", + "flt_eq": "true", + // Equality is type-aware. + "mixed": "false", + "nodes": "true", + "differ": "false", + }, + }, + }, + }, + { + name: "func_lookup", + source: ` + api_key = {"0": "produce", "1": "fetch", "2": "offsets"} + .kafka_request_api_key = lookup(.kafka_request_api_key, api_key) + .severity = lookup(.status, {"500": "crit", "400": "warn"}, default: "ok") + `, + events: []eventCase{ + { + // A numeric code matches the string key of the table. + in: `{"kafka_request_api_key":1,"status":500}`, + fields: map[string]string{ + "kafka_request_api_key": "fetch", + "severity": "crit", + }, + }, + { + // Same table, a second event: the folded table survives reuse. + in: `{"kafka_request_api_key":"0","status":200}`, + fields: map[string]string{ + "kafka_request_api_key": "produce", + "severity": "ok", + }, + }, + { + // An unmapped code passes through unchanged. + in: `{"kafka_request_api_key":77,"status":400}`, + fields: map[string]string{ + "kafka_request_api_key": "77", + "severity": "warn", + }, + }, + { + // A missing field stays missing rather than becoming a default. + in: `{"status":400}`, + fields: map[string]string{ + "kafka_request_api_key": "null", + "severity": "warn", + }, + }, + }, + }, } for _, tc := range tests {