From 7e839635cc26dc547abadcd12b9e60aefd47ca5e Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Tue, 11 Aug 2026 16:36:51 -0600 Subject: [PATCH 1/4] ci: trigger workflows for feature/script-analysis branch (#3142) --- .github/workflows/build.yml | 2 +- .github/workflows/ruff-format.yml | 2 +- .github/workflows/tests.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d51f404463..1f79c85780 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,7 +2,7 @@ name: build on: pull_request: - branches: [ master ] + branches: [ master, feature/script-analysis ] paths-ignore: - 'web/**' - 'doc/**' diff --git a/.github/workflows/ruff-format.yml b/.github/workflows/ruff-format.yml index 76b05d61c6..f81eff8b56 100644 --- a/.github/workflows/ruff-format.yml +++ b/.github/workflows/ruff-format.yml @@ -2,7 +2,7 @@ name: ruff auto-format on: pull_request: - branches: [ master ] + branches: [ master, feature/script-analysis ] paths-ignore: - 'web/**' - 'doc/**' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 510b044d25..7d394d092c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,13 +6,13 @@ name: CI on: push: - branches: [ master ] + branches: [ master, feature/script-analysis ] paths-ignore: - 'web/**' - 'doc/**' - '**.md' pull_request: - branches: [ master ] + branches: [ master, feature/script-analysis ] paths-ignore: - 'web/**' - 'doc/**' From 01a9bf03029043520ebce48b5e357a6be040f603 Mon Sep 17 00:00:00 2001 From: Saniya Fatima Date: Wed, 12 Aug 2026 20:56:15 +0530 Subject: [PATCH 2/4] Add support for script analysis (#3102) Adds static analysis for script files (C#, Python, ASPX, HTML embedded scripts) using Tree-Sitter feature extractors. Revives and completes work originally introduced in #1080. Co-authored-by: Edoardo Allegrini Co-authored-by: Adam Storek --- .github/mypy/mypy.ini | 3 + ...-capa.features.extractors.ts.signatures.py | 20 + CHANGELOG.md | 1 + capa/capabilities/static.py | 2 +- capa/features/address.py | 24 + capa/features/common.py | 10 +- capa/features/extractors/common.py | 4 + capa/features/extractors/script.py | 55 + capa/features/extractors/ts/__init__.py | 0 capa/features/extractors/ts/autodetect.py | 80 ++ capa/features/extractors/ts/engine.py | 300 ++++ capa/features/extractors/ts/extractor.py | 134 ++ capa/features/extractors/ts/file.py | 34 + capa/features/extractors/ts/function.py | 194 +++ capa/features/extractors/ts/global_.py | 40 + capa/features/extractors/ts/query.py | 257 ++++ .../extractors/ts/signatures/__init__.py | 0 .../features/extractors/ts/signatures/cs.json | 93 ++ .../features/extractors/ts/signatures/py.json | 47 + capa/features/extractors/ts/tools.py | 297 ++++ capa/features/freeze/__init__.py | 13 + capa/features/freeze/features.py | 13 + capa/helpers.py | 7 +- capa/loader.py | 11 + capa/main.py | 5 + capa/render/verbose.py | 6 + capa/rules/__init__.py | 3 + pyproject.toml | 9 + requirements.txt | 6 + tests/fixtures/__init__.py | 261 +++- tests/test_freeze_static.py | 9 +- tests/test_helpers.py | 7 + tests/test_ts.py | 1241 +++++++++++++++++ 33 files changed, 3181 insertions(+), 5 deletions(-) create mode 100644 .github/pyinstaller/hooks/hook-capa.features.extractors.ts.signatures.py create mode 100644 capa/features/extractors/script.py create mode 100644 capa/features/extractors/ts/__init__.py create mode 100644 capa/features/extractors/ts/autodetect.py create mode 100644 capa/features/extractors/ts/engine.py create mode 100644 capa/features/extractors/ts/extractor.py create mode 100644 capa/features/extractors/ts/file.py create mode 100644 capa/features/extractors/ts/function.py create mode 100644 capa/features/extractors/ts/global_.py create mode 100644 capa/features/extractors/ts/query.py create mode 100644 capa/features/extractors/ts/signatures/__init__.py create mode 100644 capa/features/extractors/ts/signatures/cs.json create mode 100644 capa/features/extractors/ts/signatures/py.json create mode 100644 capa/features/extractors/ts/tools.py create mode 100644 tests/test_ts.py diff --git a/.github/mypy/mypy.ini b/.github/mypy/mypy.ini index 4d513719a1..3e8ae35f1b 100644 --- a/.github/mypy/mypy.ini +++ b/.github/mypy/mypy.ini @@ -86,3 +86,6 @@ ignore_missing_imports = True [mypy-ghidra.*] ignore_missing_imports = True + +[mypy-tree_sitter.*] +ignore_missing_imports = True diff --git a/.github/pyinstaller/hooks/hook-capa.features.extractors.ts.signatures.py b/.github/pyinstaller/hooks/hook-capa.features.extractors.ts.signatures.py new file mode 100644 index 0000000000..7c6b19efd8 --- /dev/null +++ b/.github/pyinstaller/hooks/hook-capa.features.extractors.ts.signatures.py @@ -0,0 +1,20 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from PyInstaller.utils.hooks import collect_data_files + + +# Tree-sitter signature lookups use importlib.resources, so PyInstaller must +# bundle the JSON files alongside the package. +datas = collect_data_files("capa.features.extractors.ts.signatures") diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f3adba73b..93d1fbb0b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -148,6 +148,7 @@ This release includes Ghidra PyGhidra support, performance improvements, depende ### New Features +- Tree-Sitter Script Analysis @adamstorek #1080 @EdoardoAllegrini #2931 @saniyafatima07 #3102 - ghidra: support PyGhidra @mike-hunhoff #2788 - vmray: extract number features from whitelisted void_ptr parameters (hKey, hKeyRoot) @adeboyedn #2835 diff --git a/capa/capabilities/static.py b/capa/capabilities/static.py index 68b5245c2d..5300f6a937 100644 --- a/capa/capabilities/static.py +++ b/capa/capabilities/static.py @@ -205,7 +205,7 @@ def find_static_capabilities( match_count += len(matches_) logger.debug( - "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs", + "analyzed function %s and extracted %d features, %d matches in %0.02fs", f.address, code_capabilities.feature_count, match_count, diff --git a/capa/features/address.py b/capa/features/address.py index 83822b01f1..da3a4c0cbb 100644 --- a/capa/features/address.py +++ b/capa/features/address.py @@ -159,6 +159,30 @@ def __hash__(self): return int.__hash__(self) +class FileOffsetRangeAddress(Address): + """an address range relative to the start of a file""" + + def __init__(self, start_byte, end_byte): + self.start_byte = start_byte + self.end_byte = end_byte + + def __eq__(self, other): + if not isinstance(other, FileOffsetRangeAddress): + return NotImplemented + return (self.start_byte, self.end_byte) == (other.start_byte, other.end_byte) + + def __lt__(self, other): + if not isinstance(other, FileOffsetRangeAddress): + return NotImplemented + return (self.start_byte, self.end_byte) < (other.start_byte, other.end_byte) + + def __hash__(self): + return hash((self.start_byte, self.end_byte)) + + def __repr__(self): + return f"file(0x{self.start_byte:x}, 0x{self.end_byte:x})" + + class DNTokenAddress(int, Address): """a .NET token""" diff --git a/capa/features/common.py b/capa/features/common.py index fa2e29f926..6db1c93d9f 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -487,10 +487,17 @@ def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True): return Result(False, self, []) +class ScriptLanguage(Feature): + def __init__(self, value: str, description=None): + super().__init__(value, description=description) + self.name = "script language" + + FORMAT_PE = "pe" FORMAT_ELF = "elf" FORMAT_DOTNET = "dotnet" -VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET) +FORMAT_SCRIPT = "script" +VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET, FORMAT_SCRIPT) # internal only, not to be used in rules FORMAT_AUTO = "auto" FORMAT_SC32 = "sc32" @@ -508,6 +515,7 @@ def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True): FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET, + FORMAT_SCRIPT, FORMAT_FREEZE, FORMAT_RESULT, FORMAT_BINEXPORT2, diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index 86c7e649f0..06fccd06dc 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -36,6 +36,7 @@ VALID_ARCH, FORMAT_FREEZE, FORMAT_RESULT, + FORMAT_SCRIPT, Arch, Format, String, @@ -43,6 +44,7 @@ ) from capa.features.freeze import is_freeze from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress +from capa.features.extractors.ts.autodetect import is_script logger = logging.getLogger(__name__) @@ -78,6 +80,8 @@ def extract_format(buf: bytes) -> Iterator[tuple[Feature, Address]]: # we don't know what it is exactly, but may support it (e.g. a dynamic CAPE sandbox report) # skip verdict here and let subsequent code analyze this further return + elif is_script(buf): + yield Format(FORMAT_SCRIPT), NO_ADDRESS else: # we likely end up here: # 1. handling a file format (e.g. macho) diff --git a/capa/features/extractors/script.py b/capa/features/extractors/script.py new file mode 100644 index 0000000000..0d68ec49c5 --- /dev/null +++ b/capa/features/extractors/script.py @@ -0,0 +1,55 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple, Iterator + +from capa.features.common import OS, OS_ANY, ARCH_ANY, FORMAT_SCRIPT, Arch, Format, Feature, ScriptLanguage +from capa.features.address import NO_ADDRESS, Address, FileOffsetRangeAddress + +# Can be used to instantiate tree_sitter Language objects (see ts/query.py) +LANG_CS = "c_sharp" +LANG_HTML = "html" +LANG_JS = "javascript" +LANG_PY = "python" +LANG_TEM = "embedded_template" + +EXT_ASPX = (".aspx", ".aspx_") +EXT_CS = (".cs", ".cs_") +EXT_HTML = (".html", ".html_") +EXT_PY = (".py", ".py_") + + +LANGUAGE_FEATURE_FORMAT = { + LANG_CS: "C#", + LANG_HTML: "HTML", + LANG_JS: "JavaScript", + LANG_PY: "Python", + LANG_TEM: "Embedded Template", +} + + +def extract_arch() -> Iterator[Tuple[Feature, Address]]: + yield Arch(ARCH_ANY), NO_ADDRESS + + +def extract_language(language: str, addr: FileOffsetRangeAddress) -> Iterator[Tuple[Feature, Address]]: + yield ScriptLanguage(LANGUAGE_FEATURE_FORMAT[language]), addr + + +def extract_os() -> Iterator[Tuple[Feature, Address]]: + yield OS(OS_ANY), NO_ADDRESS + + +def extract_format() -> Iterator[Tuple[Feature, Address]]: + yield Format(FORMAT_SCRIPT), NO_ADDRESS diff --git a/capa/features/extractors/ts/__init__.py b/capa/features/extractors/ts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/capa/features/extractors/ts/autodetect.py b/capa/features/extractors/ts/autodetect.py new file mode 100644 index 0000000000..99dfa4fc02 --- /dev/null +++ b/capa/features/extractors/ts/autodetect.py @@ -0,0 +1,80 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional +from pathlib import Path + +from tree_sitter import Node, Tree, Query, Parser, Language, QueryCursor + +from capa.features.extractors.script import EXT_CS, EXT_PY, LANG_CS, LANG_PY, EXT_ASPX, EXT_HTML, LANG_TEM, LANG_HTML +from capa.features.extractors.ts.query import TS_LANGUAGES + + +def is_script(buf: bytes) -> bool: + try: + return bool(get_language_ts(buf)) + except ValueError: + return False + + +def _parse(ts_language: Language, buf: bytes) -> Optional[Tree]: + try: + parser = Parser(ts_language) + return parser.parse(buf) + except ValueError: + return None + + +def _contains_errors(ts_language, node: Node) -> bool: + query = Query(ts_language, "(ERROR) @error") + return bool(QueryCursor(query).captures(node)) + + +def get_language_ts(buf: bytes) -> str: + for language, ts_language in TS_LANGUAGES.items(): + tree = _parse(ts_language, buf) + if tree and not _contains_errors(ts_language, tree.root_node): + return language + raise ValueError("failed to parse the language") + + +def get_template_language_ts(buf: bytes) -> str: + for language, ts_language in TS_LANGUAGES.items(): + if language in [LANG_TEM, LANG_HTML]: + continue + tree = _parse(ts_language, buf) + if tree and not _contains_errors(ts_language, tree.root_node): + return language + raise ValueError("failed to parse the language") + + +def get_language_from_ext(path: str) -> str: + if path.endswith(EXT_ASPX): + return LANG_TEM + if path.endswith(EXT_CS): + return LANG_CS + if path.endswith(EXT_HTML): + return LANG_HTML + if path.endswith(EXT_PY): + return LANG_PY + raise ValueError(f"{path} has an unrecognized or an unsupported extension.") + + +def get_language(path: Path) -> str: + try: + return get_language_from_ext(str(path)) + except ValueError: + with path.open("rb") as f: + buf = f.read() + return get_language_ts(buf) diff --git a/capa/features/extractors/ts/engine.py b/capa/features/extractors/ts/engine.py new file mode 100644 index 0000000000..a6e69c3b2e --- /dev/null +++ b/capa/features/extractors/ts/engine.py @@ -0,0 +1,300 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import List, Tuple, Iterator, Optional + +from tree_sitter import Node, Tree, Parser, QueryCursor + +import capa.features.extractors.ts.autodetect +from capa.features.address import FileOffsetRangeAddress +from capa.features.extractors.script import LANG_CS, LANG_JS, LANG_TEM, LANG_HTML +from capa.features.extractors.ts.query import ( + BINDINGS, + QueryBinding, + HTMLQueryBinding, + ScriptQueryBinding, + TemplateQueryBinding, +) +from capa.features.extractors.ts.tools import LANGUAGE_TOOLKITS, BaseNamespace, CSharpNamespace, LanguageToolkit + +_RE_CSHARP_PAGE = re.compile(rb'@ .*Page Language\s*=\s*"C#".*', re.IGNORECASE) +_RE_ASPX_IMPORT_DIRECTIVE = re.compile(rb"@\s*Import Namespace=", re.IGNORECASE) +_RE_ASPX_NAMESPACE = re.compile(rb'@\s*Import namespace="(.*?)"', re.IGNORECASE) +_RE_RUNAT_SERVER = re.compile(rb'runat\s*=\s*"server"') + + +class TreeSitterBaseEngine: + buf: bytes + language: str + query: QueryBinding + tree: Tree + + def __init__(self, language: str, buf: bytes): + self.language = language + self.query = BINDINGS[language] + self.buf = buf + self.tree = self.parse() + + def parse(self) -> Tree: + parser = Parser(self.query.language) + return parser.parse(self.buf) + + def get_byte_range(self, node: Node) -> bytes: + return self.buf[node.start_byte : node.end_byte] + + def get_str(self, node: Node) -> str: + return self.get_byte_range(node).decode("utf-8") + + def get_address(self, node: Node) -> FileOffsetRangeAddress: + return FileOffsetRangeAddress(node.start_byte, node.end_byte) + + def get_default_address(self) -> FileOffsetRangeAddress: + return self.get_address(self.tree.root_node) + + @staticmethod + def get_node_sort_key(node: Node) -> Tuple[int, int]: + return node.start_byte, node.end_byte + + @staticmethod + def get_node_capture_sort_key(capture: Tuple[Node, str]) -> Tuple[int, int]: + node, _ = capture + return TreeSitterBaseEngine.get_node_sort_key(node) + + @staticmethod + def get_captured_nodes(cursor: QueryCursor, node: Node) -> Iterator[Node]: + captured_nodes: List[Node] = [] + for nodes in cursor.captures(node).values(): + captured_nodes.extend(nodes) + + yield from sorted(captured_nodes, key=TreeSitterBaseEngine.get_node_sort_key) + + +class TreeSitterExtractorEngine(TreeSitterBaseEngine): + query: ScriptQueryBinding + language_toolkit: LanguageToolkit + buf_offset: int + namespaces: set[BaseNamespace] + + def __init__( + self, + language: str, + buf: bytes, + buf_offset: int = 0, + additional_namespaces: set[BaseNamespace] | None = None, + ): + super().__init__(language, buf) + self.buf_offset = buf_offset + self.language_toolkit = LANGUAGE_TOOLKITS[language] + + if additional_namespaces is None: + additional_namespaces = set() + + self.namespaces = set(self.get_processed_namespaces()) + self.namespaces = self.namespaces.union(additional_namespaces) + + def get_address(self, node: Node) -> FileOffsetRangeAddress: + return FileOffsetRangeAddress(self.buf_offset + node.start_byte, self.buf_offset + node.end_byte) + + def get_new_object_names(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.new_object_name) + yield from self.get_captured_nodes(cursor, node) + + def get_property_names(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.property_name) + yield from self.get_captured_nodes(cursor, node) + + def get_processed_property_names(self, node: Node) -> Iterator[Tuple[Node, str]]: + """Generates captured property name nodes and their associated proper names (see process_property + for details), e.g.: [(node0, "StartInfo"), (node1, "RedirectStandardOutput")].""" + for pt_node in self.get_property_names(node): + pt_name = self.language_toolkit.process_property(pt_node, self.get_str(pt_node)) + if pt_name: + yield pt_node, pt_name + + def get_function_definitions(self, node: Optional[Node] = None) -> Iterator[Node]: + node = self.tree.root_node if node is None else node + cursor = QueryCursor(self.query.function_definition) + yield from self.get_captured_nodes(cursor, node) + + def get_function_definition_name(self, node: Node) -> Node | None: + return node.child_by_field_name(self.query.function_definition_field_name) + + def get_function_definition_names(self, node: Node) -> Iterator[Node]: + for fd_node in self.get_function_definitions(node): + name_node = self.get_function_definition_name(fd_node) + if name_node is not None: + yield name_node + + def get_function_call_names(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.function_call_name) + yield from self.get_captured_nodes(cursor, node) + + def get_imported_constants(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.imported_constant_name) + yield from self.get_captured_nodes(cursor, node) + + def get_processed_imported_constants(self, node: Node) -> Iterator[Tuple[Node, str]]: + """Generates captured imported constant nodes and their associated proper names (see process_imported_constant + for details), e.g.: [(node0, "ssl.CERT_NONE"), (node1, "win32con.FILE_ATTRIBUTE_HIDDEN")].""" + for ic_node in self.get_imported_constants(node): + ic_name = self.language_toolkit.process_imported_constant(ic_node, self.get_str(ic_node)) + if ic_name: + yield ic_node, ic_name + + def get_string_literals(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.string_literal) + yield from self.get_captured_nodes(cursor, node) + + def get_integer_literals(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.integer_literal) + yield from self.get_captured_nodes(cursor, node) + + def get_namespaces(self, node: Optional[Node] = None) -> List[Tuple[Node, str]]: + target_node = self.tree.root_node if node is None else node + cursor = QueryCursor(self.query.namespace) + namespace_captures: List[Tuple[Node, str]] = [] + + for query_name, nodes in cursor.captures(target_node).items(): + for namespace_node in nodes: + namespace_captures.append((namespace_node, query_name)) + + return sorted(namespace_captures, key=self.get_node_capture_sort_key) + + def get_processed_namespaces(self, node: Optional[Node] = None) -> Iterator[BaseNamespace]: + for ns_node, query_name in self.get_namespaces(node): + yield from self.language_toolkit.process_namespace(ns_node, query_name, self.get_str) + + def get_global_statements(self) -> Iterator[Node]: + cursor = QueryCursor(self.query.global_statement) + yield from self.get_captured_nodes(cursor, self.tree.root_node) + + def get_direct_method_call(self, node: Node) -> Optional[Node]: + cursor = QueryCursor(self.query.direct_method_call) + captures = cursor.captures(node) + for nodes in captures.values(): + if nodes: + return nodes[0] + return None + + +class TreeSitterTemplateEngine(TreeSitterBaseEngine): + query: TemplateQueryBinding + language_toolkit: LanguageToolkit + embedded_language: str + namespaces: set[BaseNamespace] + + def __init__(self, buf: bytes): + super().__init__(LANG_TEM, buf) + self.embedded_language = self.identify_language() + self.language_toolkit = LANGUAGE_TOOLKITS[self.embedded_language] + self.namespaces = set(self.get_namespaces()) + + def get_code_sections(self) -> Iterator[Node]: + cursor = QueryCursor(self.query.code) + yield from self.get_captured_nodes(cursor, self.tree.root_node) + + def get_parsed_code_sections(self) -> Iterator[TreeSitterExtractorEngine]: + for node in self.get_code_sections(): + # TODO(EdoardoAllegrini): support JS + # https://github.com/mandiant/capa/issues/1092 + if self.embedded_language == LANG_CS: + yield TreeSitterExtractorEngine( + self.embedded_language, + self.get_byte_range(node), + node.start_byte, + self.namespaces, + ) + else: + raise ValueError(f"parsing of {self.embedded_language} is not supported") + + def get_content_sections(self) -> Iterator[Node]: + cursor = QueryCursor(self.query.content) + yield from self.get_captured_nodes(cursor, self.tree.root_node) + + def identify_language(self) -> str: + for node in self.get_code_sections(): + if self.is_c_sharp(node): + return LANG_CS + try: + return capa.features.extractors.ts.autodetect.get_template_language_ts(self.get_byte_range(node)) + except ValueError: + continue + raise ValueError("failed to identify the template language") + + def get_imported_namespaces(self) -> Iterator[BaseNamespace]: + for node in self.get_code_sections(): + if self.is_aspx_import_directive(node): + namespace = self.get_aspx_namespace(node) + if namespace is not None: + yield namespace + + def get_namespaces(self) -> Iterator[BaseNamespace]: + yield from self.language_toolkit.get_default_namespaces(True) + yield from self.get_imported_namespaces() + + def is_c_sharp(self, node: Node) -> bool: + return bool(_RE_CSHARP_PAGE.match(self.get_byte_range(node))) + + def is_aspx_import_directive(self, node: Node) -> bool: + return bool(_RE_ASPX_IMPORT_DIRECTIVE.match(self.get_byte_range(node))) + + def get_aspx_namespace(self, node: Node) -> Optional[BaseNamespace]: + match = _RE_ASPX_NAMESPACE.search(self.get_byte_range(node)) + return CSharpNamespace(match.group(1).decode("utf-8"), node) if match is not None else None + + +class TreeSitterHTMLEngine(TreeSitterBaseEngine): + query: HTMLQueryBinding + namespaces: set[BaseNamespace] + + def __init__(self, buf: bytes, namespaces: set[BaseNamespace] | None = None): + super().__init__(LANG_HTML, buf) + self.namespaces = namespaces if namespaces is not None else set() + + def get_scripts(self) -> Iterator[Node]: + cursor = QueryCursor(self.query.script_element) + for nodes in cursor.captures(self.tree.root_node).values(): + yield from nodes + + def get_attributes(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.attribute) + for nodes in cursor.captures(node).values(): + yield from nodes + + def get_identified_scripts(self) -> Iterator[Tuple[Node, str]]: + for node in self.get_scripts(): + for content_node in self.get_script_contents(node): + yield content_node, self.identify_language(node) + + def get_script_contents(self, node: Node) -> Iterator[Node]: + cursor = QueryCursor(self.query.script_content) + for nodes in cursor.captures(node).values(): + yield from nodes + + def get_parsed_code_sections(self) -> Iterator[TreeSitterExtractorEngine]: + for node, language in self.get_identified_scripts(): + # TODO(EdoardoAllegrini): support JS + # https://github.com/mandiant/capa/issues/1092 + if language == LANG_CS: + yield TreeSitterExtractorEngine(language, self.get_byte_range(node), node.start_byte, self.namespaces) + + def identify_language(self, node: Node) -> str: + for att_node in self.get_attributes(node): + if self.is_server_side_c_sharp(att_node): + return LANG_CS + return LANG_JS + + def is_server_side_c_sharp(self, node: Node) -> bool: + return bool(_RE_RUNAT_SERVER.search(self.get_byte_range(node))) diff --git a/capa/features/extractors/ts/extractor.py b/capa/features/extractors/ts/extractor.py new file mode 100644 index 0000000000..b3b4b5739c --- /dev/null +++ b/capa/features/extractors/ts/extractor.py @@ -0,0 +1,134 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Tuple, Union, Iterator +from pathlib import Path + +import capa.features.extractors.script +import capa.features.extractors.ts.file +import capa.features.extractors.ts.global_ +import capa.features.extractors.ts.function +import capa.features.extractors.ts.autodetect +from capa.exceptions import UnsupportedFormatError +from capa.features.common import Namespace +from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, FileOffsetRangeAddress +from capa.features.extractors.script import LANG_TEM, LANG_HTML +from capa.features.extractors.ts.tools import BaseNamespace +from capa.features.extractors.ts.engine import TreeSitterHTMLEngine, TreeSitterTemplateEngine, TreeSitterExtractorEngine +from capa.features.extractors.ts.function import PSEUDO_MAIN, TSFunctionInner +from capa.features.extractors.base_extractor import ( + Feature, + BBHandle, + InsnHandle, + SampleHashes, + FunctionHandle, + StaticFeatureExtractor, +) + + +class TreeSitterFeatureExtractor(StaticFeatureExtractor): + engines: List[TreeSitterExtractorEngine] + template_engine: TreeSitterTemplateEngine + language: str + path: Path + + def __init__(self, path: Path): + self.path = path + + with self.path.open("rb") as f: + buf = f.read() + + super().__init__(hashes=SampleHashes.from_bytes(buf)) + + try: + self.language = capa.features.extractors.ts.autodetect.get_language(path) + self.template_engine = self.get_template_engine(buf) + self.engines = self.get_engines(buf) + except ValueError as e: + raise UnsupportedFormatError(e) + + def get_template_engine(self, buf: bytes): + if self.language == LANG_TEM: + return TreeSitterTemplateEngine(buf) + + def get_engines(self, buf: bytes) -> List[TreeSitterExtractorEngine]: + if self.language == LANG_TEM and self.template_engine: + return self.extract_code_from_template() + if self.language == LANG_HTML: + return self.extract_code_from_html(buf) + return [TreeSitterExtractorEngine(self.language, buf)] + + def extract_code_from_template(self) -> List[TreeSitterExtractorEngine]: + engines = list(self.template_engine.get_parsed_code_sections()) + for node in self.template_engine.get_content_sections(): + section_buf = self.template_engine.get_byte_range(node) + engines.extend(self.extract_code_from_html(section_buf, self.template_engine.namespaces)) + return engines + + def extract_code_from_html( + self, buf: bytes, namespaces: set[BaseNamespace] | None = None + ) -> List[TreeSitterExtractorEngine]: + if namespaces is None: + namespaces = set() + return list(TreeSitterHTMLEngine(buf, namespaces).get_parsed_code_sections()) + + def get_base_address(self) -> Union[AbsoluteVirtualAddress, capa.features.address._NoAddress]: + return NO_ADDRESS + + def extract_template_namespaces(self) -> Iterator[Tuple[Feature, Address]]: + for ns in self.template_engine.get_namespaces(): + address = NO_ADDRESS if ns.node is None else FileOffsetRangeAddress(ns.node.start_byte, ns.node.end_byte) + yield Namespace(ns.name), address + + def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]: + for engine in self.engines: + yield from capa.features.extractors.script.extract_language(engine.language, engine.get_default_address()) + yield from capa.features.extractors.ts.global_.extract_features() + + def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]: + if self.language == LANG_TEM: + yield from self.extract_template_namespaces() + for engine in self.engines: + yield from capa.features.extractors.ts.file.extract_features(engine) + + def get_pseudo_main_function_inner(self, engine: TreeSitterExtractorEngine) -> TSFunctionInner: + return TSFunctionInner(engine.tree.root_node, PSEUDO_MAIN, engine) + + def get_pseudo_main_function(self, engine: TreeSitterExtractorEngine) -> FunctionHandle: + return FunctionHandle(engine.get_default_address(), self.get_pseudo_main_function_inner(engine)) + + def get_functions(self) -> Iterator[FunctionHandle]: + for engine in self.engines: + yield self.get_pseudo_main_function(engine) + for node in engine.get_function_definitions(): + name_node = engine.get_function_definition_name(node) + name = engine.get_str(name_node) if name_node is not None else "" + yield FunctionHandle(engine.get_address(node), TSFunctionInner(node, name, engine)) + + def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]: + yield from capa.features.extractors.ts.function.extract_features(f, f.inner.engine) + + def get_basic_blocks(self, f: FunctionHandle) -> Iterator[BBHandle]: + yield from [] + + def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]: + yield from [] + + def get_instructions(self, f: FunctionHandle, bb: BBHandle) -> Iterator[InsnHandle]: + yield from [] + + def extract_insn_features( + self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle + ) -> Iterator[Tuple[Feature, Address]]: + yield from [] diff --git a/capa/features/extractors/ts/file.py b/capa/features/extractors/ts/file.py new file mode 100644 index 0000000000..66a2e34115 --- /dev/null +++ b/capa/features/extractors/ts/file.py @@ -0,0 +1,34 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple, Iterator + +from capa.features.common import Feature, Namespace +from capa.features.address import Address +from capa.features.extractors.ts.engine import TreeSitterExtractorEngine + + +def extract_namespaces(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for namespace in engine.get_processed_namespaces(): + if namespace.node is not None: + yield Namespace(namespace.name), engine.get_address(namespace.node) + + +def extract_features(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for file_handler in FILE_HANDLERS: + for feature, addr in file_handler(engine): + yield feature, addr + + +FILE_HANDLERS = (extract_namespaces,) diff --git a/capa/features/extractors/ts/function.py b/capa/features/extractors/ts/function.py new file mode 100644 index 0000000000..f9f004ab96 --- /dev/null +++ b/capa/features/extractors/ts/function.py @@ -0,0 +1,194 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple, Iterator +from dataclasses import dataclass + +from tree_sitter import Node + +from capa.features.insn import API, Number, Property +from capa.features.common import Class, String, Feature, Namespace +from capa.features.address import Address +from capa.features.extractors.ts.tools import BaseNamespace +from capa.features.extractors.ts.engine import TreeSitterExtractorEngine +from capa.features.extractors.base_extractor import FunctionHandle + +PSEUDO_MAIN = "PSEUDO MAIN" # all global statements in one function scope + + +@dataclass +class TSFunctionInner: + node: Node + name: str + engine: TreeSitterExtractorEngine + + +def is_pseudo_main_function(fh: FunctionHandle, engine: TreeSitterExtractorEngine) -> bool: + return ( + fh.address == engine.get_default_address() + and fh.inner.node == engine.tree.root_node + and fh.inner.name == PSEUDO_MAIN + ) + + +def extract_strings(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_string_literals(fn_node): + yield String(engine.language_toolkit.parse_string(engine.get_str(node))), engine.get_address(node) + + +def extract_integers(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_integer_literals(fn_node): + try: + yield Number(engine.language_toolkit.parse_integer(engine.get_str(node))), engine.get_address(node) + except ValueError: + continue + + +def get_possible_full_names(name: str, namespaces: set[BaseNamespace]) -> Iterator[str]: + yield name + for namespace in namespaces: + yield namespace.join(name) + + +def get_default_constructor(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[str]: + for name_node in engine.get_new_object_names(fn_node): + for full_name in get_possible_full_names(engine.get_str(name_node), engine.namespaces): + if engine.language_toolkit.is_imported_class(full_name): + yield full_name + + +def get_custom_constructor(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[str]: + for name_node in engine.get_function_call_names(fn_node): + for full_name in get_possible_full_names(engine.get_str(name_node), engine.namespaces): + if engine.language_toolkit.is_imported_constructor(full_name): + yield full_name + + +def get_classes(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[str]: + yield from get_default_constructor(fn_node, engine) + yield from get_custom_constructor(fn_node, engine) + + +def _extract_default_constructor(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for name_node in engine.get_new_object_names(fn_node): + for full_name in get_possible_full_names(engine.get_str(name_node), engine.namespaces): + if engine.language_toolkit.is_imported_class(full_name): + yield Namespace(full_name), engine.get_address(name_node) + yield Class(engine.language_toolkit.format_imported_class(full_name)), engine.get_address(name_node) + yield ( + API(engine.language_toolkit.format_imported_default_constructor(full_name)), + engine.get_address(name_node), + ) + + +def _extract_custom_constructor(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for name_node in engine.get_function_call_names(fn_node): + for full_name in get_possible_full_names(engine.get_str(name_node), engine.namespaces): + if engine.language_toolkit.is_imported_constructor(full_name): + yield Namespace(full_name), engine.get_address(name_node) + yield Class(engine.language_toolkit.format_imported_class(full_name)), engine.get_address(name_node) + yield ( + API(engine.language_toolkit.format_imported_custom_constructor(full_name)), + engine.get_address(name_node), + ) + + +def _extract_classes(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + yield from _extract_default_constructor(fn_node, engine) + yield from _extract_custom_constructor(fn_node, engine) + + +def _extract_constants(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for ic_node, ic_name in engine.get_processed_imported_constants(fn_node): + for full_name in get_possible_full_names(ic_name, engine.namespaces): + if engine.language_toolkit.is_imported_constant(full_name): + yield API(engine.language_toolkit.format_imported_constant(full_name)), engine.get_address(ic_node) + + +def _extract_properties( + fn_node: Node, classes: set[BaseNamespace], engine: TreeSitterExtractorEngine +) -> Iterator[Tuple[Feature, Address]]: + for pt_node, pt_name in engine.get_processed_property_names(fn_node): + for full_name in get_possible_full_names(pt_name, classes): + if engine.language_toolkit.is_imported_property(full_name): + yield Property(engine.language_toolkit.format_imported_property(full_name)), engine.get_address(pt_node) + + +def _extract_static_methods(node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + if engine.language_toolkit.is_builtin(engine.get_str(node)): + yield API(engine.language_toolkit.get_builtin_name(engine.get_str(node))), engine.get_address(node) + for full_name in get_possible_full_names(engine.get_str(node), engine.namespaces): + if engine.language_toolkit.is_imported_function(full_name): + yield API(engine.language_toolkit.format_imported_function(full_name)), engine.get_address(node) + + +def _do_extract_instance_methods( + node: Node, classes: set[BaseNamespace], engine: TreeSitterExtractorEngine +) -> Iterator[Tuple[Feature, Address]]: + for full_name in get_possible_full_names( + engine.language_toolkit.get_member_from_name(engine.get_str(node)), classes + ): + if engine.language_toolkit.is_imported_function(full_name): + yield API(engine.language_toolkit.format_imported_function(full_name)), engine.get_address(node) + + +def _extract_instance_methods( + node: Node, classes: set[BaseNamespace], engine: TreeSitterExtractorEngine +) -> Iterator[Tuple[Feature, Address]]: + direct_method_call_node = engine.get_direct_method_call(node) # eg new Foo.Bar().direct_method_call(x, y, 3) + if direct_method_call_node: + yield from _do_extract_instance_methods(direct_method_call_node, classes, engine) + else: + yield from _do_extract_instance_methods(node, classes, engine) + + +def _extract_function_calls( + fn_node: Node, classes: set[BaseNamespace], engine: TreeSitterExtractorEngine +) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_function_call_names(fn_node): + yield from _extract_static_methods(node, engine) + yield from _extract_instance_methods(node, classes, engine) + + +def extract_imports(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + classes = {engine.language_toolkit.create_namespace(cls) for cls in get_classes(fn_node, engine)} + yield from _extract_classes(fn_node, engine) + yield from _extract_constants(fn_node, engine) + yield from _extract_properties(fn_node, classes, engine) + yield from _extract_function_calls(fn_node, classes, engine) + + +def _extract_pseudo_main_features(engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for node in engine.get_global_statements(): + yield from _extract_features(node, engine) + + +def _extract_features(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + for function_handler in FUNCTION_HANDLERS: + for feature, addr in function_handler(fn_node, engine): + yield feature, addr + + +def extract_features(fh: FunctionHandle, engine: TreeSitterExtractorEngine) -> Iterator[Tuple[Feature, Address]]: + if is_pseudo_main_function(fh, engine): + yield from _extract_pseudo_main_features(engine) + else: + yield from _extract_features(fh.inner.node, engine) + + +FUNCTION_HANDLERS = ( + extract_imports, + extract_integers, + extract_strings, +) diff --git a/capa/features/extractors/ts/global_.py b/capa/features/extractors/ts/global_.py new file mode 100644 index 0000000000..6449b8faa3 --- /dev/null +++ b/capa/features/extractors/ts/global_.py @@ -0,0 +1,40 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple, Iterator + +import capa.features.extractors.script +from capa.features.common import Feature +from capa.features.address import Address + + +def extract_arch() -> Iterator[Tuple[Feature, Address]]: + yield from capa.features.extractors.script.extract_arch() + + +def extract_os() -> Iterator[Tuple[Feature, Address]]: + yield from capa.features.extractors.script.extract_os() + + +def extract_file_format() -> Iterator[Tuple[Feature, Address]]: + yield from capa.features.extractors.script.extract_format() + + +def extract_features() -> Iterator[Tuple[Feature, Address]]: + for glob_handler in GLOBAL_HANDLERS: + for feature, addr in glob_handler(): + yield feature, addr + + +GLOBAL_HANDLERS = (extract_arch, extract_os, extract_file_format) diff --git a/capa/features/extractors/ts/query.py b/capa/features/extractors/ts/query.py new file mode 100644 index 0000000000..af904ad39d --- /dev/null +++ b/capa/features/extractors/ts/query.py @@ -0,0 +1,257 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + +import tree_sitter_html +import tree_sitter_python +import tree_sitter_c_sharp +import tree_sitter_javascript +import tree_sitter_embedded_template +from tree_sitter import Query, Language + +from capa.features.extractors.script import ( + LANG_CS, + LANG_JS, + LANG_PY, + LANG_TEM, + LANG_HTML, +) + + +@dataclass +class QueryBinding: + language: Language + + +@dataclass +class ScriptQueryBinding(QueryBinding): + new_object_name: Query + function_definition: Query + function_definition_field_name: str + direct_method_call: Query + function_call_name: Query + property_name: Query + imported_constant_name: Query + string_literal: Query + integer_literal: Query + namespace: Query + global_statement: Query + + +@dataclass +class TemplateQueryBinding(QueryBinding): + code: Query + content: Query + + +@dataclass +class HTMLQueryBinding(QueryBinding): + script_element: Query + script_content: Query + attribute: Query + + +TS_LANGUAGES: dict[str, Language] = { + LANG_CS: Language(tree_sitter_c_sharp.language()), + LANG_PY: Language(tree_sitter_python.language()), + LANG_JS: Language(tree_sitter_javascript.language()), + LANG_TEM: Language(tree_sitter_embedded_template.language()), + LANG_HTML: Language(tree_sitter_html.language()), +} + + +def deserialize(language: str, binding: dict) -> dict: + result = {} + + if "query" in binding: + for name, query in binding["query"].items(): + result[name] = Query(TS_LANGUAGES[language], query) + + if "field_name" in binding: + for name, field in binding["field_name"].items(): + result[f"{name}_field_name"] = field + + return result + + +BINDINGS: dict[str, QueryBinding] = { + LANG_CS: ScriptQueryBinding( + TS_LANGUAGES[LANG_CS], + **deserialize( + LANG_CS, + { + "query": { + # new Foo() + "new_object_name": """ + (object_creation_expression + type: [ + (qualified_name) @new-object + (identifier) @new-object + ]) + """, + # local functions + "function_definition": """ + [ + (local_function_statement) + (method_declaration) + (constructor_declaration) + ] @function-definition + """, + # foo() or obj.foo() + "function_call_name": """ + (invocation_expression + function: [ + (member_access_expression) @function-call + (identifier) @function-call + ]) + """, + # obj.property + "property_name": """ + (member_access_expression) @property + """, + # SomeClass.CONSTANT + "imported_constant_name": """ + (member_access_expression) @constant + """, + "string_literal": """ + (string_literal) @string-literal + """, + "integer_literal": """ + (integer_literal) @integer-literal + """, + # using System.Text; + "namespace": """ + (using_directive + [ + (identifier) @namespace + (qualified_name) @namespace + ]) + """, + # global statements + "global_statement": """ + (global_statement + [ + (if_statement) @global-statement + (expression_statement) @global-statement + (local_declaration_statement) @global-statement + ]) + """, + # new Foo().Bar() + "direct_method_call": """ + (member_access_expression + expression: (object_creation_expression) + name: (identifier) @direct-method-call) + """, + }, + "field_name": { + "function_definition": "name", + }, + }, + ), + ), + LANG_PY: ScriptQueryBinding( + TS_LANGUAGES[LANG_PY], + **deserialize( + LANG_PY, + { + "query": { + # Python: constructor == call + "new_object_name": """ + (call + function: [ + (attribute) @new-object + (identifier) @new-object + ]) + """, + "function_definition": """ + (function_definition) @function-definition + """, + "function_call_name": """ + (call + function: [ + (attribute) @function-call + (identifier) @function-call + ]) + """, + "property_name": """ + (attribute) @property + """, + # obj.CONSTANT + "imported_constant_name": """ + (attribute) @constant + """, + "string_literal": """ + (string) @string-literal + """, + "integer_literal": """ + (integer) @integer-literal + """, + "namespace": """ + [ + (import_statement) @import + (import_from_statement) @import-from + ] + """, + "global_statement": """ + (module + [ + (if_statement) @global-statement + (expression_statement) @global-statement + ]) + """, + "direct_method_call": """ + (attribute + object: (call) + attribute: (identifier) @direct-method-call) + """, + }, + "field_name": { + "function_definition": "name", + }, + }, + ), + ), + LANG_TEM: TemplateQueryBinding( + TS_LANGUAGES[LANG_TEM], + **deserialize( + LANG_TEM, + { + "query": { + "code": "(code) @code", + "content": "(content) @content", + } + }, + ), + ), + LANG_HTML: HTMLQueryBinding( + TS_LANGUAGES[LANG_HTML], + **deserialize( + LANG_HTML, + { + "query": { + "script_element": """ + (script_element) @script-element + """, + "script_content": """ + (raw_text) @script-content + """, + "attribute": """ + (attribute) @attribute + """, + } + }, + ), + ), +} diff --git a/capa/features/extractors/ts/signatures/__init__.py b/capa/features/extractors/ts/signatures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/capa/features/extractors/ts/signatures/cs.json b/capa/features/extractors/ts/signatures/cs.json new file mode 100644 index 0000000000..3da994aed8 --- /dev/null +++ b/capa/features/extractors/ts/signatures/cs.json @@ -0,0 +1,93 @@ +{ + "classes": [ + "System.Data.SqlClient.SqlCommand", + "System.Data.SqlClient.SqlConnection", + "System.Data.SqlClient.SqlDataAdapter", + "System.Diagnostics.Process", + "System.Diagnostics.ProcessStartInfo", + "System.IO.DirectoryInfo", + "System.Security.Cryptography.CryptoStream", + "System.Security.Cryptography.Rijndael", + "System.Security.Cryptography.RijndaelManaged", + "System.Security.Cryptography.RSACryptoServiceProvider", + "System.Security.Cryptography.SHA1", + "System.Security.Cryptography.SHA1CryptoServiceProvider", + "System.Security.Cryptography.SHA256", + "System.Security.Cryptography.SHA256CryptoServiceProvider" + ], + "constructors": [ + "System.Security.Cryptography.Rijndael.Create" + ], + "functions": [ + "System.Convert.FromBase64String", + "System.Convert.ToBase64String", + "System.Data.SqlClient.SqlCommand.ExecuteReader", + "System.Data.SqlClient.SqlConnection.Open", + "System.Diagnostics.Process.Start", + "System.IO.Directory.CreateDirectory", + "System.IO.File.Delete", + "System.IO.File.GetAttributes", + "System.IO.File.GetCreationTime", + "System.IO.File.GetLastAccessTime", + "System.IO.File.GetLastWriteTime", + "System.IO.File.ReadAllBytes", + "System.IO.File.ReadAllBytesAsync", + "System.IO.File.ReadAllLines", + "System.IO.File.ReadAllLinesAsync", + "System.IO.File.ReadAllText", + "System.IO.File.ReadAllTextAsync", + "System.IO.File.ReadLines", + "System.IO.File.ReadLinesAsync", + "System.IO.File.SetCreationTime", + "System.IO.File.SetLastAccessTime", + "System.IO.File.SetLastWriteTime", + "System.IO.File.Write", + "System.IO.File.WriteAllBytes", + "System.IO.File.WriteAllBytesAsync", + "System.IO.File.WriteAllLines", + "System.IO.File.WriteAllLinesAsync", + "System.IO.File.WriteAllText", + "System.IO.File.WriteAllTextAsync", + "System.IO.File.WriteLines", + "System.IO.File.WriteLinesAsync", + "System.IO.Path.GetTempPath", + "System.Security.Cryptography.RijndaelManaged.CreateDecryptor", + "System.Security.Cryptography.RijndaelManaged.CreateEncryptor", + "System.Security.Cryptography.RSACryptoServiceProvider.Encrypt", + "System.Security.Cryptography.SHA1CryptoServiceProvider.ComputeHash", + "System.Security.Cryptography.SHA256CryptoServiceProvider.ComputeHash" + ], + "properties": [ + "System.Diagnostics.Process.StartInfo.Arguments", + "System.Diagnostics.Process.StartInfo.CreateNoWindow", + "System.Diagnostics.Process.StartInfo.FileName", + "System.Diagnostics.Process.StartInfo.RedirectStandardInput", + "System.Diagnostics.Process.StartInfo.RedirectStandardOutput", + "System.Diagnostics.Process.StartInfo.UseShellExecute", + "System.Diagnostics.ProcessStartInfo.Arguments", + "System.Diagnostics.ProcessStartInfo.CreateNoWindow", + "System.Diagnostics.ProcessStartInfo.FileName", + "System.Diagnostics.ProcessStartInfo.RedirectStandardInput", + "System.Diagnostics.ProcessStartInfo.RedirectStandardOutput", + "System.Diagnostics.ProcessStartInfo.UseShellExecute" + ], + "constants": [], + "builtins": [], + "aspx_default_namespaces": [ + "System", + "System.Collections", + "System.Collections.Specialized", + "System.Configuration", + "System.Text", + "System.Text.RegularExpressions", + "System.Web", + "System.Web.Caching", + "System.Web.Profile", + "System.Web.Security", + "System.Web.SessionState", + "System.Web.UI", + "System.Web.UI.HtmlControls", + "System.Web.UI.WebControls", + "System.Web.UI.WebControls.WebParts" + ] +} diff --git a/capa/features/extractors/ts/signatures/py.json b/capa/features/extractors/ts/signatures/py.json new file mode 100644 index 0000000000..bf6cdf09b7 --- /dev/null +++ b/capa/features/extractors/ts/signatures/py.json @@ -0,0 +1,47 @@ +{ + "classes": [ + "socket.error", + "socket.socket", + "urllib2.Request" + ], + "constructors": [ + "ssl.wrap_socket", + "win32com.client.Dispatch" + ], + "functions": [ + "base64.b64decode", + "base64.b64encode", + "base64.encodestring", + "os.chdir", + "os.chmod", + "os.getcwd", + "os.path.dirname", + "os.path.expanduser", + "os.popen", + "os.remove", + "platform.mac_ver", + "shutil.copytree", + "subprocess.Popen", + "time.sleep", + "urllib2.urlopen", + "win32api.SetFileAttributes" + ], + "constants": [ + "os.environ", + "socket.AF_INET", + "socket.SO_REUSEADDR", + "socket.SOCK_STREAM", + "socket.SQL_SOCKET", + "ssl.CERT_NONE", + "ssl.PROTOCOL_TLSv1", + "subprocess.PIPE", + "win32con.FILE_ATTRIBUTE_HIDDEN", + "win32con.FILE_ATTRIBUTE_SYSTEM" + ], + "properties": [], + "builtins": [ + "eval", + "exec", + "open" + ] +} diff --git a/capa/features/extractors/ts/tools.py b/capa/features/extractors/ts/tools.py new file mode 100644 index 0000000000..fb50d914ba --- /dev/null +++ b/capa/features/extractors/ts/tools.py @@ -0,0 +1,297 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import json +import importlib.resources +from typing import Dict, List, Tuple, Union, Callable, Iterator, Optional +from dataclasses import dataclass + +from tree_sitter import Node + +import capa.features.extractors.ts.signatures +from capa.features.extractors.script import LANG_CS, LANG_PY + + +@dataclass(frozen=True) +class BaseNamespace(abc.ABC): + """Abstract class for internal representation of the namespace concept, including aliases.""" + + name: str + node: Node | None = None + alias: str = "" + + def __hash__(self): + return hash(self.name) + + def join(self, name: str) -> str: + raise NotImplementedError() + + +class CSharpNamespace(BaseNamespace): + def join(self, name: str) -> str: + """using System; Diagnostics.ProcessStartInfo => System.Diagnostics.ProcessStartInfo""" + return LANGUAGE_TOOLKITS[LANG_CS].join_names(self.name, name) + + +class PythonImport(BaseNamespace): + def join(self, name: str) -> str: + """import subprocess ; subprocess.Popen => subprocess.Popen + from threading import Timer (threading.Timer) => Timer + """ + toolkit = LANGUAGE_TOOLKITS[LANG_PY] + qualified_names = toolkit.split_name(self.name) + if len(qualified_names) < 2: + return name + + # if plain "import x.y case" - don't prepend + if name == self.name or name.startswith(self.name + "."): + return name + return toolkit.join_names(*(qualified_names[:-1] + [name])) + + +class LanguageToolkit: + signature_file: str + import_signatures: Dict[str, set[str]] + method_call_query_type: str + property_query_type: str + string_delimiters: str + integer_prefixes: List[ + Tuple[Union[str, Tuple[str, ...]], int] + ] # Tends to indicate a number system, e.g. (("0x", "0X"), 16) + integer_suffixes: Tuple[str, ...] # Tends to indicate unsigned (100u) or long (100l) integer literal + + def __init__(self): + self.import_signatures = self.load_import_signatures(self.signature_file) + + def load_import_signatures(self, signature_file: str) -> Dict[str, set[str]]: + ref = importlib.resources.files(capa.features.extractors.ts.signatures) / signature_file + signatures = json.loads(ref.read_text(encoding="utf-8")) + return {category: set(names) for category, names in signatures.items()} + + def get_full_name(self, name: str, namespace: Optional[BaseNamespace] = None) -> str: + if namespace: + if namespace.alias: + return name.replace(namespace.alias, namespace.name) + return namespace.join(name) + return name + + def is_imported_function(self, name: str, namespace: Optional[BaseNamespace] = None) -> bool: + return self.get_full_name(name, namespace) in self.import_signatures["functions"] + + def is_imported_class(self, name: str, namespace: Optional[BaseNamespace] = None) -> bool: + return self.get_full_name(name, namespace) in self.import_signatures["classes"] + + def is_imported_constructor(self, name: str, namespace: Optional[BaseNamespace] = None) -> bool: + return self.get_full_name(name, namespace) in self.import_signatures["constructors"] + + def is_imported_property(self, name: str, namespace: Optional[BaseNamespace] = None) -> bool: + return self.get_full_name(name, namespace) in self.import_signatures["properties"] + + def is_imported_constant(self, name: str, namespace: Optional[BaseNamespace] = None) -> bool: + return self.get_full_name(name, namespace) in self.import_signatures["constants"] + + def is_builtin(self, func: str) -> bool: + return func in self.import_signatures["builtins"] + + def get_builtin_name(self, func: str) -> str: + return self.join_names("builtins", func) + + def join_names(self, *args: str) -> str: + return ".".join(args) + + def split_name(self, name: str) -> List[str]: + return name.split(".") + + def process_property(self, node: Node, name: str) -> str: + if self.is_method_call(node): # yield only p.StartInfo but not p.Start() + return "" + if self.is_recursive_property(node): # yield only Current.Server.ClearError but not Current.Server and Current + return "" + + parts = self.split_name(name) + if len(parts) < 2: + return name + + # PascalCase first segment is considered a type + if parts[0][:1].isupper(): + return name + + return self.join_names(*parts[1:]) + + def process_imported_constant(self, node: Node, name: str) -> Optional[str]: + if self.is_method_call(node): # yield only ssl.CERT_NONE and not ssl.wrap_socket() + return None + if self.is_recursive_property(node): # yield foo.foo.bar and not foo.bar or bar + return None + return name + + def get_namespace_from_name(self, name: str) -> str: + qualified_names = self.split_name(name) + if len(qualified_names) < 2: + return "" + return self.join_names(*qualified_names[:-1]) + + def get_member_from_name(self, name: str) -> str: + qualified_names = self.split_name(name) + if len(qualified_names) < 2: + return qualified_names[0] + return self.join_names(*qualified_names[1:]) + + def format_imported_class(self, name: str) -> str: + return name + + def format_imported_class_members(self, name: str) -> str: + qualified_names = self.split_name(name) + if len(qualified_names) < 2: + raise ValueError(f"{name} does not have an associated class or namespace") + if len(qualified_names) == 2: + classname, membername = qualified_names[0], qualified_names[1] + return f"{classname}::{membername}" + namespace, classname, membername = qualified_names[:-2], qualified_names[-2], qualified_names[-1] + return f"{'.'.join(namespace)}.{classname}::{membername}" + + def format_imported_function(self, name: str) -> str: + return self.format_imported_class_members(name) + + def format_imported_custom_constructor(self, name: str) -> str: + return self.format_imported_class_members(name) + + def format_imported_default_constructor(self, name: str) -> str: + return self.format_imported_function(self.join_names(name, "ctor")) + + def format_imported_property(self, name: str) -> str: + return self.format_imported_class_members(name) + + def format_imported_constant(self, name: str) -> str: + return self.format_imported_class_members(name) + + def parse_integer(self, integer: str) -> int: + integer = integer.lower() + for suffix in self.integer_suffixes: + if integer.endswith(suffix): + integer = integer[: -len(suffix)] + break + for prefix, base in self.integer_prefixes: + if integer.startswith(prefix): + return int(integer, base) + return int(integer) + + def parse_string(self, string: str) -> str: + return string.strip(self.string_delimiters) + + def is_method_call(self, node: Node) -> bool: + if node.parent is None: + return False + return node.parent.type == self.method_call_query_type + + def is_recursive_property(self, node: Node) -> bool: + if node.parent is None: + return False + return node.parent.type == self.property_query_type + + @abc.abstractmethod + def create_namespace(self, name: str) -> BaseNamespace: + raise NotImplementedError() + + @abc.abstractmethod + def process_namespace(self, node: Node, query_name: str, get_str: Callable) -> Iterator[BaseNamespace]: + raise NotImplementedError() + + @abc.abstractmethod + def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: + raise NotImplementedError() + + +class CSharpToolkit(LanguageToolkit): + signature_file: str = "cs.json" + method_call_query_type: str = "invocation_expression" + property_query_type: str = "member_access_expression" + string_delimiters: str = '"' + integer_prefixes: List[Tuple[Union[str, Tuple[str, ...]], int]] = [(("0x", "0X"), 16)] + integer_suffixes: Tuple[str, ...] = ("u", "l") + + def create_namespace(self, name: str) -> BaseNamespace: + return CSharpNamespace(name) + + def process_namespace(self, node: Node, query_name: str, get_str: Callable) -> Iterator[BaseNamespace]: + yield CSharpNamespace(get_str(node), node, "") + + def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: + if embedded: + return {CSharpNamespace(name) for name in self.import_signatures["aspx_default_namespaces"]} + return set() + + +class PythonToolkit(LanguageToolkit): + signature_file: str = "py.json" + method_call_query_type: str = "call" + property_query_type: str = "attribute" + string_delimiters: str = "\"'" + integer_prefixes: List[Tuple[Union[str, Tuple[str, ...]], int]] = [ + (("0b", "0B"), 2), + (("0o", "0O"), 8), + (("0x", "0X"), 16), + ] + integer_suffixes: Tuple[str, ...] = () + + def create_namespace(self, name: str) -> BaseNamespace: + return PythonImport(name) + + def get_import_name(self, name: str, module_name: Optional[str] = None) -> str: + return self.join_names(module_name, name) if module_name else name + + def process_simple_import(self, node: Node, get_str: Callable, module_name: Optional[str] = None) -> BaseNamespace: + return PythonImport(self.get_import_name(get_str(node), module_name), node) + + def process_aliased_import(self, node: Node, get_str: Callable, module_name: Optional[str] = None) -> BaseNamespace: + name = self.get_import_name(get_str(node.child_by_field_name("name")), module_name) + alias = get_str(node.child_by_field_name("alias")) + return PythonImport(name, node, alias) + + def process_imports( + self, nodes: List[Node], get_str: Callable, module_name: Optional[str] = None + ) -> Iterator[BaseNamespace]: + for import_node in nodes: + if import_node.type == "dotted_name": + yield self.process_simple_import(import_node, get_str, module_name) + elif import_node.type == "aliased_import": + yield self.process_aliased_import(import_node, get_str, module_name) + + def get_wildcard_import(self, node: Node) -> Optional[Node]: + for child_node in node.children: + if child_node.type == "wildcard_import": + return child_node + return None + + def process_import_from(self, node: Node, import_nodes: List[Node], get_str: Callable) -> Iterator[BaseNamespace]: + module_name, import_nodes = get_str(import_nodes[0]), import_nodes[1:] + wildcard_import = self.get_wildcard_import(node) + if wildcard_import: + yield self.process_simple_import(wildcard_import, get_str, module_name) + else: + yield from self.process_imports(import_nodes, get_str, module_name) + + def process_namespace(self, node: Node, query_name: str, get_str: Callable) -> Iterator[BaseNamespace]: + import_nodes = [child_node for child_node in node.children if child_node.is_named] + if query_name == "import-from": + yield from self.process_import_from(node, import_nodes, get_str) + elif query_name == "import": + yield from self.process_imports(import_nodes, get_str) + + def get_default_namespaces(self, embedded: bool) -> set[BaseNamespace]: + return set() + + +LANGUAGE_TOOLKITS: dict[str, LanguageToolkit] = {LANG_CS: CSharpToolkit(), LANG_PY: PythonToolkit()} diff --git a/capa/features/freeze/__init__.py b/capa/features/freeze/__init__.py index 728b2b7854..a2d2ee7c71 100644 --- a/capa/features/freeze/__init__.py +++ b/capa/features/freeze/__init__.py @@ -49,6 +49,7 @@ class AddressType(str, Enum): ABSOLUTE = "absolute" RELATIVE = "relative" FILE = "file" + FILE_RANGE = "file range" DN_TOKEN = "dn token" DN_TOKEN_OFFSET = "dn token offset" PROCESS = "process" @@ -79,6 +80,9 @@ def from_capa(cls, a: capa.features.address.Address) -> "Address": elif isinstance(a, capa.features.address.FileOffsetAddress): return cls(type=AddressType.FILE, value=int(a)) + elif isinstance(a, capa.features.address.FileOffsetRangeAddress): + return cls(type=AddressType.FILE_RANGE, value=(a.start_byte, a.end_byte)) + elif isinstance(a, capa.features.address.DNTokenAddress): return cls(type=AddressType.DN_TOKEN, value=int(a)) @@ -116,6 +120,15 @@ def to_capa(self) -> capa.features.address.Address: assert isinstance(self.value, int) return capa.features.address.FileOffsetAddress(self.value) + elif self.type is AddressType.FILE_RANGE: + if isinstance(self.value, (tuple, list)) and len(self.value) >= 2: + start_byte, end_byte = self.value[:2] + elif isinstance(self.value, int): + start_byte = end_byte = self.value + else: + start_byte = end_byte = 0 + return capa.features.address.FileOffsetRangeAddress(start_byte, end_byte) + elif self.type is AddressType.DN_TOKEN: assert isinstance(self.value, int) return capa.features.address.DNTokenAddress(self.value) diff --git a/capa/features/freeze/features.py b/capa/features/freeze/features.py index 151964e55d..9a02748920 100644 --- a/capa/features/freeze/features.py +++ b/capa/features/freeze/features.py @@ -35,6 +35,9 @@ def to_capa(self) -> capa.features.common.Feature: elif isinstance(self, FormatFeature): return capa.features.common.Format(self.format, description=self.description) + elif isinstance(self, ScriptLanguageFeature): + return capa.features.common.ScriptLanguage(self.language, description=self.description) + elif isinstance(self, MatchFeature): return capa.features.common.MatchedRule(self.match, description=self.description) @@ -123,6 +126,9 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature": assert isinstance(f.value, str) return FormatFeature(format=f.value, description=f.description) + elif isinstance(f, capa.features.common.ScriptLanguage): + return ScriptLanguageFeature(language=str(f.value), description=f.description) + elif isinstance(f, capa.features.common.MatchedRule): assert isinstance(f.value, str) return MatchFeature(match=f.value, description=f.description) @@ -232,6 +238,12 @@ class FormatFeature(FeatureModel): description: Optional[str] = None +class ScriptLanguageFeature(FeatureModel): + type: Literal["script language"] = "script language" + language: str + description: Optional[str] = None + + class MatchFeature(FeatureModel): type: Literal["match"] = "match" match: str @@ -359,6 +371,7 @@ class OperandOffsetFeature(FeatureModel): OSFeature, ArchFeature, FormatFeature, + ScriptLanguageFeature, MatchFeature, CharacteristicFeature, ExportFeature, diff --git a/capa/helpers.py b/capa/helpers.py index 6d723c378f..d82b65a0ce 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -27,10 +27,10 @@ from datetime import datetime import msgspec.json -from rich.text import Text from rich.console import Console from rich.progress import ( Task, + Text, Progress, BarColumn, TextColumn, @@ -52,12 +52,14 @@ FORMAT_VMRAY, FORMAT_DOTNET, FORMAT_FREEZE, + FORMAT_SCRIPT, FORMAT_DRAKVUF, FORMAT_UNKNOWN, FORMAT_BINJA_DB, FORMAT_BINEXPORT2, Format, ) +from capa.features.extractors.script import EXT_CS, EXT_PY, EXT_ASPX, EXT_HTML EXTENSIONS_SHELLCODE_32 = (".sc32", ".raw32") EXTENSIONS_SHELLCODE_64 = (".sc64", ".raw64") @@ -69,6 +71,7 @@ EXTENSIONS_ELF = ".elf_" EXTENSIONS_FREEZE = ".frz" EXTENSIONS_BINJA_DB = ".bndb" +EXTENSIONS_SUPPORTED_SCRIPTS = EXT_ASPX + EXT_CS + EXT_HTML + EXT_PY logger = logging.getLogger("capa") @@ -236,6 +239,8 @@ def get_format_from_extension(sample: Path) -> str: format_ = FORMAT_BINEXPORT2 elif sample.name.endswith(EXTENSIONS_BINJA_DB): format_ = FORMAT_BINJA_DB + elif sample.name.endswith(EXTENSIONS_SUPPORTED_SCRIPTS): + return FORMAT_SCRIPT return format_ diff --git a/capa/loader.py b/capa/loader.py index b0895b2524..68953d0540 100644 --- a/capa/loader.py +++ b/capa/loader.py @@ -46,6 +46,7 @@ FORMAT_SC64, FORMAT_VMRAY, FORMAT_DOTNET, + FORMAT_SCRIPT, FORMAT_DRAKVUF, FORMAT_BINJA_DB, FORMAT_BINEXPORT2, @@ -72,6 +73,7 @@ BACKEND_BINEXPORT2 = "binexport2" BACKEND_IDA = "ida" BACKEND_GHIDRA = "ghidra" +BACKEND_SCRIPT = "script" class CorruptFile(ValueError): @@ -485,6 +487,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): import capa.features.extractors.ghidra.extractor return capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor(ctx_manager=cm, tmpdir=tmpdir) + elif backend == BACKEND_SCRIPT: + import capa.features.extractors.ts.extractor + + return capa.features.extractors.ts.extractor.TreeSitterFeatureExtractor(input_path) else: raise ValueError("unexpected backend: " + backend) @@ -555,6 +561,11 @@ def get_file_extractors(input_file: Path, input_format: str) -> list[FeatureExtr elif input_format == FORMAT_BINEXPORT2: file_extractors = _get_binexport2_file_extractors(input_file) + elif input_format == FORMAT_SCRIPT: + import capa.features.extractors.ts.extractor + + file_extractors.append(capa.features.extractors.ts.extractor.TreeSitterFeatureExtractor(input_file)) + return file_extractors diff --git a/capa/main.py b/capa/main.py index 837974f54c..ab3dc2b029 100644 --- a/capa/main.py +++ b/capa/main.py @@ -52,6 +52,7 @@ BACKEND_FREEZE, BACKEND_GHIDRA, BACKEND_PEFILE, + BACKEND_SCRIPT, BACKEND_DRAKVUF, BACKEND_BINEXPORT2, ) @@ -88,6 +89,7 @@ FORMAT_DOTNET, FORMAT_FREEZE, FORMAT_RESULT, + FORMAT_SCRIPT, FORMAT_DRAKVUF, STATIC_FORMATS, DYNAMIC_FORMATS, @@ -598,6 +600,9 @@ def get_backend_from_cli(args, input_format: str) -> str: elif input_format == FORMAT_BINEXPORT2: return BACKEND_BINEXPORT2 + elif input_format == FORMAT_SCRIPT: + return BACKEND_SCRIPT + else: return BACKEND_VIV diff --git a/capa/render/verbose.py b/capa/render/verbose.py index 4afb46386b..72edbc38d5 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -56,6 +56,12 @@ def format_address(address: frz.Address) -> str: elif address.type == frz.AddressType.FILE: assert isinstance(address.value, int) return f"file+{capa.helpers.hex(address.value)}" + elif address.type == frz.AddressType.FILE_RANGE: + assert isinstance(address.value, tuple) + start, end = address.value + assert isinstance(start, int) + assert isinstance(end, int) + return f"file({capa.helpers.hex(start)}, {capa.helpers.hex(end)})" elif address.type == frz.AddressType.DN_TOKEN: assert isinstance(address.value, int) return f"token({capa.helpers.hex(address.value)})" diff --git a/capa/rules/__init__.py b/capa/rules/__init__.py index ef4e372c70..b270237921 100644 --- a/capa/rules/__init__.py +++ b/capa/rules/__init__.py @@ -182,6 +182,7 @@ def from_dict(cls, scopes: dict[str, str]) -> "Scopes": capa.features.common.OS, capa.features.common.Arch, capa.features.common.Format, + capa.features.common.ScriptLanguage, }, Scope.FILE: { capa.features.common.MatchedRule, @@ -445,6 +446,8 @@ def parse_feature(key: str): return capa.features.common.Namespace elif key == "property": return capa.features.insn.Property + elif key == "language": + return capa.features.common.ScriptLanguage elif key.startswith("operand[") and key.endswith("].number"): index = int(key[len("operand[") : -len("].number")]) return functools.partial(capa.features.insn.OperandNumber, index) diff --git a/pyproject.toml b/pyproject.toml index f670e53cf1..2428c94926 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,6 +106,12 @@ dependencies = [ "networkx>=3", "dnfile>=0.17.0", + "tree-sitter>=0.25.0", + "tree-sitter-c-sharp>=0.23.0", + "tree-sitter-embedded-template>=0.25.0", + "tree-sitter-html>=0.23.0", + "tree-sitter-javascript>=0.25.0", + "tree-sitter-python>=0.25.0", ] dynamic = ["version"] @@ -123,6 +129,9 @@ version = {attr = "capa.version.__version__"} include = ["capa*"] namespaces = false +[tool.setuptools.package-data] +"capa.features.extractors.ts.signatures" = ["*.json"] + [project.optional-dependencies] dev = [ # Dev and build dependencies are not relaxed because diff --git a/requirements.txt b/requirements.txt index ec340cb1e9..3a5c2dc18f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,3 +47,9 @@ viv-utils==0.8.0 vivisect==1.3.2 msgspec==0.21.1 bump-my-version==1.5.0 +tree-sitter==0.25.2 +tree-sitter-c-sharp==0.23.1 +tree-sitter-embedded-template==0.25.0 +tree-sitter-html==0.23.2 +tree-sitter-javascript==0.25.0 +tree-sitter-python==0.25.0 diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index d8d8e25dd4..4234c3e428 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -17,7 +17,7 @@ import functools import contextlib import collections -from typing import Union, Literal, Optional +from typing import Tuple, Union, Literal, Iterator, Optional from pathlib import Path from dataclasses import field, dataclass @@ -28,6 +28,7 @@ import capa.render.result_document from capa.features.common import OS_AUTO, FORMAT_AUTO, Feature from capa.features.address import Address +from capa.features.extractors.script import LANG_CS, LANG_PY from capa.features.extractors.base_extractor import ( BBHandle, CallHandle, @@ -45,6 +46,10 @@ CD = _FIXTURES_DIR.parent FIXTURE_MANIFEST_DIR = _FIXTURES_DIR / "features" DNFILE_TESTFILES = CD / "data" / "dotnet" / "dnfile-testfiles" +SOURCE_DIR = CD / "data" / "source" +ASPX_DIR = SOURCE_DIR / "aspx" +CS_DIR = SOURCE_DIR / "cs" +PY_DIR = SOURCE_DIR / "py" def parse_feature_string(s: str) -> Feature | ceng.Range | ceng.Statement: @@ -474,6 +479,69 @@ def get_function(extractor, fva: int) -> FunctionHandle: raise ValueError("function not found") +def get_function_ts(extractor, fid: Union[Tuple[int], str]) -> Iterator[FunctionHandle]: + for fh in extractor.get_functions(): + if isinstance(fid, tuple): + addr = (fh.address.start_byte, fh.address.end_byte) + elif isinstance(fid, str): + addr = fh.inner.name + else: + raise ValueError("invalid fva format") + + if addr == fid: + yield fh + + +def get_function_id_ts(scope): + fid = scope.partition("=")[2] + if fid[0] == "(" and fid[-1] == ")": + fid = tuple(int(x, 16) if x.lstrip().startswith("0x") else int(x) for x in fid[1:-1].split(",")) + return fid + + +def resolve_scope_ts(scope): + if scope == "global": + + def inner_fn(extractor): + return extract_global_features(extractor) + + elif scope == "file": + + def inner_fn(extractor): + features = extract_file_features(extractor) + for k, vs in extract_global_features(extractor).items(): + features[k].update(vs) + return features + + elif scope.startswith("function"): + # like `function=(0xbeef, 0xdead) or function=(123, 456) or function=foo_bar` + def inner_fn(extractor): + fid = get_function_id_ts(scope) + fhs = list(get_function_ts(extractor, fid)) + if not fhs: + raise ValueError("function not found") + features = collections.defaultdict(set) + for fh in fhs: + for k, vs in extract_function_features(extractor, fh).items(): + # print(f"{k}:{vs}") + features[k].update(vs) + for k, vs in extract_file_features(extractor).items(): + features[k].update(vs) + for k, vs in extract_global_features(extractor).items(): + features[k].update(vs) + return features + + else: + raise ValueError("unexpected scope fixture") + inner_fn.__name__ = scope + return inner_fn + + +@pytest.fixture +def scope_ts(request): + return resolve_scope_ts(request.param) + + def get_function_by_token(extractor, token: int) -> FunctionHandle: for fh in extractor.get_functions(): if fh.address == token: @@ -700,6 +768,139 @@ def dynamic_a0000a6_rd(): PMA1601 = CD / "data" / "Practical Malware Analysis Lab 16-01.exe_" +@pytest.fixture +def aspx_4f6fa6_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_4f6fa6"]) + + +@pytest.fixture +def aspx_5f959f_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_5f959f"]) + + +@pytest.fixture +def aspx_10162f_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_10162f"]) + + +@pytest.fixture +def aspx_2b71dd_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_2b71dd"]) + + +@pytest.fixture +def aspx_f2bf20_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_f2bf20"]) + + +@pytest.fixture +def aspx_f39dc0_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_f39dc0"]) + + +@pytest.fixture +def aspx_ea2a01_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_ea2a01"]) + + +@pytest.fixture +def aspx_6f3261_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_6f3261"]) + + +@pytest.fixture +def aspx_1f8f40_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_1f8f40"]) + + +@pytest.fixture +def aspx_2e8c7e_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_2e8c7e"]) + + +@pytest.fixture +def aspx_03bb5c_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_03bb5c"]) + + +@pytest.fixture +def aspx_606dbf_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_606dbf"]) + + +@pytest.fixture +def aspx_f397cb_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_f397cb"]) + + +@pytest.fixture +def aspx_b4bb14_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_b4bb14"]) + + +@pytest.fixture +def aspx_54433d_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_54433d"]) + + +@pytest.fixture +def aspx_a35878_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_a35878"]) + + +@pytest.fixture +def aspx_a5c893_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_a5c893"]) + + +@pytest.fixture +def aspx_15eed4_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_15eed4"]) + + +@pytest.fixture +def aspx_b75f16_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_b75f16"]) + + +@pytest.fixture +def aspx_d460ca_template_engine(): + return get_ts_template_engine(ASPX_DATA_PATH_BY_NAME["aspx_d460ca"]) + + +@pytest.fixture +def cs_138cdc_extractor_engine(): + return get_ts_extractor_engine(LANG_CS, CS_DATA_PATH_BY_NAME["cs_138cdc"].read_bytes()) + + +@pytest.fixture +def py_24e48f_template_engine(): + return get_ts_extractor_engine(LANG_PY, PY_DATA_PATH_BY_NAME["py_24e48f"]) + + +@pytest.fixture +def py_a4d252_template_engine(): + return get_ts_extractor_engine(LANG_PY, PY_DATA_PATH_BY_NAME["py_a4d252"]) + + +def resolve_sample_ts(sample): + if sample.startswith("cs_"): + return CS_DATA_PATH_BY_NAME[sample] + if sample.startswith("py_"): + return PY_DATA_PATH_BY_NAME[sample] + if sample.startswith("aspx_"): + try: + return ASPX_DATA_PATH_BY_NAME[sample] + except KeyError: + raise ValueError(f"unexpected sample fixture: {sample}") + raise ValueError(f"unexpected sample fixture: {sample}") + + +@pytest.fixture +def sample_ts(request): + return resolve_sample_ts(request.param) + + # used by test_viv_features # as well as some fixtures below @functools.lru_cache(maxsize=1) @@ -966,3 +1167,61 @@ def get_binexport_extractor(path): buf = path.read_bytes() return capa.features.extractors.binexport2.extractor.BinExport2FeatureExtractor(be2, buf) + + +@functools.lru_cache(maxsize=1) +def get_ts_extractor_engine(language, buf): + import capa.features.extractors.ts.engine + + return capa.features.extractors.ts.engine.TreeSitterExtractorEngine(language, buf) + + +@functools.lru_cache(maxsize=1) +def get_ts_template_engine(path): + import capa.features.extractors.ts.engine + + with Path(path).open("rb") as f: + buf = f.read() + return capa.features.extractors.ts.engine.TreeSitterTemplateEngine(buf) + + +@functools.lru_cache(maxsize=1) +def get_ts_extractor(path): + import capa.features.extractors.ts.extractor + + return capa.features.extractors.ts.extractor.TreeSitterFeatureExtractor(path) + + +ASPX_DATA_PATH_BY_NAME = { + "aspx_4f6fa6": ASPX_DIR / "4f6fa6a45017397c7e1c9cd5a17235ccb1ff0f5087dfa6b7384552bf507e7fe1.aspx_", + "aspx_5f959f": ASPX_DIR / "5f959f480a66a33d37d9a0ef6c8f7d0059625ca2a8ae9236b49b194733622655.aspx_", + "aspx_10162f": ASPX_DIR / "10162feb5f063ea09c6a3d275f31abf0fe8a9e4e36fded0053b1f8e054da8161.aspx_", + "aspx_2b71dd": ASPX_DIR / "2b71dd245520d9eb5f1e4c633fee61c7d83687591d9f64f9390c26dc95057c3c.aspx_", + "aspx_f2bf20": ASPX_DIR / "f2bf20e7bb482d27da8f19aa0f8bd4927746a65300929b99166867074a38a4b4.aspx_", + "aspx_f39dc0": ASPX_DIR / "f39dc0dfd43477d65c1380a7cff89296ad72bfa7fc3afcfd8e294f195632030e.aspx_", + "aspx_ea2a01": ASPX_DIR / "ea2a01cae57c00df01bff6bb8a72585fdc0abb7a26a869dc1a0131bdff50b400.aspx_", + "aspx_6f3261": ASPX_DIR / "6f3261eaaabf369bd928d179641b73ffd768184dfd4e00124da462a3075d4239.aspx_", + "aspx_1f8f40": ASPX_DIR / "1f8f4054932ed1d5d055e9a92aa1e2abba49af3370506674cb1b2c70146ae81a.aspx_", + "aspx_2e8c7e": ASPX_DIR / "2e8c7eacd739ca3f3dc4112b41a024157035096b8d0c26ba79d8b893136391bc.aspx_", + "aspx_03bb5c": ASPX_DIR / "03bb5cab46b406bb8613ca6e32991ab3e10b5cd759d5c7813191e9e62868ea73.aspx_", + "aspx_606dbf": ASPX_DIR / "606dbfebdc7751ecb6cb9a845853ae1905afd4b8a2cb54e1e4a98c932e268712.aspx_", + "aspx_f397cb": ASPX_DIR / "f397cb676353873cdc8fcfbf0e3a317334353cc63946099e5ea22db6d1eebfb8.aspx_", + "aspx_b4bb14": ASPX_DIR / "b4bb14aeb692f7afc107ee89f86d096f1cd8f9761b6c50788f626a9dccc8b077.aspx_", + "aspx_54433d": ASPX_DIR / "54433dd57414773098a6d3292d262f91a6812855dfcbf8d421695608d1fad638.aspx_", + "aspx_a35878": ASPX_DIR / "a35878e74425cd97ad98e3ec4b2583867bb536f4275d821cd8b82bc19380ba1a.aspx_", + "aspx_a5c893": ASPX_DIR / "a5c8934836f5b36bba3a722eab691a9f1f926c138fefe5bae07e9074e7c49ae3.aspx_", + "aspx_15eed4": ASPX_DIR / "15eed42e4904205b2ef2ff285ff1ce6c8138296c12cf075a2562c69a5fafd1cb.aspx_", + "aspx_b75f16": ASPX_DIR / "b75f163ca9b9240bf4b37ad92bc7556b40a17e27c2b8ed5c8991385fe07d17d0.aspx_", + "aspx_d460ca": ASPX_DIR / "d460cae7d34c51059ef57c5aadb3de099469efbac5fffcf76d0528a511192a28.aspx_", +} + + +CS_DATA_PATH_BY_NAME = { + "cs_138cdc": CS_DIR / "138cdc4b10f3f5ece9c47bb0ec17fde5b70c1f9a90b267794c5e5dfa337fc798.cs_", +} + + +PY_DATA_PATH_BY_NAME = { + "py_24e48f": PY_DIR / "24e48f27083aa14d630ec1aec8dfe8ec869dc8ba48f68154e2b14f493a548d28.py_", + "py_a4d252": PY_DIR / "a4d252752d0558206b3f631fee3d57ae56190fb8203e571506fa058d076fbb96.py_", +} diff --git a/tests/test_freeze_static.py b/tests/test_freeze_static.py index 3d0a7b3db1..3d11e02a0f 100644 --- a/tests/test_freeze_static.py +++ b/tests/test_freeze_static.py @@ -23,10 +23,11 @@ import capa.features.insn import capa.features.common import capa.features.freeze +import capa.features.address import capa.features.basicblock import capa.features.extractors.null import capa.features.freeze.features -from capa.features.address import Address, AbsoluteVirtualAddress +from capa.features.address import Address, AbsoluteVirtualAddress, FileOffsetRangeAddress from capa.features.extractors.base_extractor import ( BBHandle, SampleHashes, @@ -189,6 +190,12 @@ def test_serialize_features(): capa.features.insn.Property("System.IO.FileInfo::Length", access=capa.features.common.FeatureAccess.READ) ) roundtrip_feature(capa.features.insn.Property("System.IO.FileInfo::Length")) + roundtrip_feature(capa.features.common.ScriptLanguage("Python")) + + +def test_freeze_file_range_address_roundtrip(): + addr = FileOffsetRangeAddress(0x10, 0x20) + assert capa.features.freeze.Address.from_capa(addr).to_capa() == addr def test_no_address_lt_irreflexivity(): diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 2f458fc14b..fb87f2c776 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -28,6 +28,7 @@ EXTENSIONS_BINEXPORT2, EXTENSIONS_SHELLCODE_32, EXTENSIONS_SHELLCODE_64, + EXTENSIONS_SUPPORTED_SCRIPTS, get_file_taste, get_format_from_extension, ) @@ -36,6 +37,7 @@ FORMAT_SC32, FORMAT_SC64, FORMAT_FREEZE, + FORMAT_SCRIPT, FORMAT_UNKNOWN, FORMAT_BINJA_DB, FORMAT_BINEXPORT2, @@ -139,6 +141,7 @@ def test_extensions_dot_prefix(): EXTENSIONS_SHELLCODE_64, EXTENSIONS_DYNAMIC, EXTENSIONS_BINEXPORT2, + EXTENSIONS_SUPPORTED_SCRIPTS, (EXTENSIONS_ELF,), (EXTENSIONS_FREEZE,), (EXTENSIONS_BINJA_DB,), @@ -160,6 +163,8 @@ def test_extensions_dot_prefix(): assert Path("sample.elf_").name.endswith(EXTENSIONS_ELF) assert Path("sample.frz").name.endswith(EXTENSIONS_FREEZE) assert Path("sample.bndb").name.endswith(EXTENSIONS_BINJA_DB) + assert Path("sample.py").name.endswith(EXTENSIONS_SUPPORTED_SCRIPTS) + assert Path("sample.py_").name.endswith(EXTENSIONS_SUPPORTED_SCRIPTS) def test_get_format_from_extension(): @@ -172,6 +177,8 @@ def test_get_format_from_extension(): assert get_format_from_extension(Path("sample.BinExport")) == FORMAT_BINEXPORT2 assert get_format_from_extension(Path("sample.BinExport2")) == FORMAT_BINEXPORT2 assert get_format_from_extension(Path("sample.bndb")) == FORMAT_BINJA_DB + assert get_format_from_extension(Path("sample.py")) == FORMAT_SCRIPT + assert get_format_from_extension(Path("sample.py_")) == FORMAT_SCRIPT assert get_format_from_extension(Path("sample.exe")) == FORMAT_UNKNOWN diff --git a/tests/test_ts.py b/tests/test_ts.py new file mode 100644 index 0000000000..3fbd3f2314 --- /dev/null +++ b/tests/test_ts.py @@ -0,0 +1,1241 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Tuple + +import pytest +from fixtures import parametrize, get_ts_extractor, resolve_scope_ts, resolve_sample_ts +from tree_sitter import Node, Tree + +from capa.features.insn import API, Number, Property +from capa.features.common import ( + OS, + OS_ANY, + ARCH_ANY, + FORMAT_SCRIPT, + Arch, + Class, + Format, + String, + Namespace, + Substring, + ScriptLanguage, +) +from capa.features.address import FileOffsetRangeAddress +from capa.features.extractors.script import LANG_CS, LANG_JS, LANG_PY, LANG_TEM, LANG_HTML, LANGUAGE_FEATURE_FORMAT +from capa.features.extractors.ts.query import QueryBinding, HTMLQueryBinding, TemplateQueryBinding +from capa.features.extractors.ts.tools import LANGUAGE_TOOLKITS +from capa.features.extractors.ts.engine import ( + TreeSitterBaseEngine, + TreeSitterHTMLEngine, + TreeSitterTemplateEngine, + TreeSitterExtractorEngine, +) + + +def do_test_ts_base_engine_init(engine: TreeSitterBaseEngine): + assert engine.language in [LANG_CS, LANG_TEM, LANG_HTML, LANG_JS] + assert isinstance(engine.query, QueryBinding) + assert isinstance(engine.buf, bytes) and len(engine.buf) > 0 + assert isinstance(engine.tree, Tree) + + +def do_test_ts_base_engine_get_str( + engine: TreeSitterBaseEngine, node: Node, expected_range: str, startswith: bool = False +): + assert engine.get_str(node).startswith(expected_range) if startswith else engine.get_str(node) == expected_range + + +def do_test_ts_base_engine_get_address(engine: TreeSitterBaseEngine, node: Node): + assert isinstance(engine.get_address(node), FileOffsetRangeAddress) + addr = engine.get_address(node) + assert addr.start_byte == node.start_byte and addr.end_byte == node.end_byte + + +def do_test_ts_base_engine_get_default_address(engine: TreeSitterBaseEngine): + assert isinstance(engine.get_default_address(), FileOffsetRangeAddress) + addr1 = engine.get_address(engine.tree.root_node) + addr2 = engine.get_default_address() + assert addr1.start_byte == addr2.start_byte and addr1.end_byte == addr2.end_byte + + +def do_test_ts_extractor_engine_init(engine: TreeSitterExtractorEngine, expected_language: str): + assert engine.language == expected_language + assert isinstance(engine.query, QueryBinding) + assert isinstance(engine.get_default_address(), FileOffsetRangeAddress) + assert isinstance(engine.buf_offset, int) and engine.buf_offset >= 0 + addr = engine.get_default_address() + assert ( + addr.start_byte == engine.tree.root_node.start_byte + engine.buf_offset + and addr.end_byte == engine.tree.root_node.end_byte + engine.buf_offset + ) + + +def do_test_ts_extractor_engine_get_address( + engine: TreeSitterExtractorEngine, node: Node, expected_range: str, startswith: bool = False +): + assert engine.get_str(node).startswith(expected_range) if startswith else engine.get_str(node) == expected_range + + +def do_test_ts_extractor_engine_get_new_objects( + engine: TreeSitterExtractorEngine, root_node: Node, expected: List[Tuple[str, str]] +): + assert len(list(engine.get_new_object_names(root_node))) == len(expected) + for node, (_, expected_name_range) in zip(engine.get_new_object_names(root_node), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_name_range) + do_test_ts_base_engine_get_address(engine, node) + + +def do_test_ts_extractor_engine_get_function_definitions( + engine: TreeSitterExtractorEngine, root_node: Node, expected: List[Tuple[str, str]] +): + assert list(engine.get_function_definitions(engine.tree.root_node)) == list(engine.get_function_definitions()) + assert len(list(engine.get_function_definitions(root_node))) == len(expected) + for node, (expected_range, expected_name_range) in zip(engine.get_function_definitions(root_node), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_range, startswith=True) + do_test_ts_base_engine_get_address(engine, node) + + name_node = engine.get_function_definition_name(node) + assert name_node is not None, "Expected a valid name node, but got None" + do_test_ts_base_engine_get_str(engine, name_node, expected_name_range) + + assert len(list(engine.get_function_definition_names(root_node))) == len(expected) + for node, (_, expected_name_range) in zip(engine.get_function_definition_names(root_node), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_name_range) + do_test_ts_base_engine_get_address(engine, node) + + +def do_test_ts_extractor_engine_get_function_calls( + engine: TreeSitterExtractorEngine, root_node: Node, expected: List[Tuple[str, str]] +): + assert len(list(engine.get_function_call_names(root_node))) == len(expected) + for node, (_, expected_id_range) in zip(engine.get_function_call_names(root_node), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_id_range) + do_test_ts_base_engine_get_address(engine, node) + + +def do_test_ts_extractor_engine_get_string_literals( + engine: TreeSitterExtractorEngine, root_node: Node, expected: List[str] +): + assert len(list(engine.get_string_literals(root_node))) == len(expected) + for node, expected_range in zip(engine.get_string_literals(root_node), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_range) + do_test_ts_base_engine_get_address(engine, node) + + +def do_test_ts_extractor_engine_get_integer_literals( + engine: TreeSitterExtractorEngine, root_node: Node, expected: List[str] +): + assert len(list(engine.get_integer_literals(root_node))) == len(expected) + for node, expected_range in zip(engine.get_integer_literals(root_node), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_range) + do_test_ts_base_engine_get_address(engine, node) + + +def do_test_ts_extractor_engine_get_namespaces(engine: TreeSitterExtractorEngine, expected: List[str]): + assert list(engine.get_namespaces(engine.tree.root_node)) == list(engine.get_namespaces()) + assert len(list(engine.get_namespaces())) == len(expected) + for (node, _), expected_range in zip(engine.get_namespaces(), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_range) + do_test_ts_base_engine_get_address(engine, node) + + +def do_test_ts_extractor_engine_get_global_statements(engine: TreeSitterExtractorEngine, expected: List[str]): + assert len(list(engine.get_global_statements())) == len(expected) + for node, expected_range in zip(engine.get_global_statements(), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_str(engine, node, expected_range, startswith=True) + do_test_ts_base_engine_get_address(engine, node) + + +def do_test_ts_extractor_engine_get_assigned_property_names( + engine: TreeSitterExtractorEngine, root_node: Node, expected: List[str] +): + assert len(list(engine.get_processed_property_names(root_node))) == len(expected) + for (node, _name), _expected_name in zip(engine.get_processed_property_names(root_node), expected): + assert isinstance(node, Node) + do_test_ts_base_engine_get_address(engine, node) + + +@parametrize( + "engine_str,expected", + [ + ( + "cs_138cdc_extractor_engine", + { + "language": LANG_CS, + "all objects": [ + ( + 'new Diagnostics.ProcessStartInfo("cmd", "/c " + Request.Form["c"])', + "Diagnostics.ProcessStartInfo", + ), + ("new System.Diagnostics.Process()", "System.Diagnostics.Process"), + ], + "all function definitions": [ + ("void die()", "die"), + ("void Page_Load(object sender, System.EventArgs e)", "Page_Load"), + ], + "all function calls": [ + ( + 'HttpContext.Current.Response.Write("

404 Not Found

")', + "HttpContext.Current.Response.Write", + ), + ( + "HttpContext.Current.Server.ClearError()", + "HttpContext.Current.Server.ClearError", + ), + ( + "HttpContext.Current.Response.End()", + "HttpContext.Current.Response.End", + ), + ( + "HttpContext.Current.Request.Headers[\"X-Forwarded-For\"].Split(new char[] { ',' })", + 'HttpContext.Current.Request.Headers["X-Forwarded-For"].Split', + ), + ( + "die()", + "die", + ), + ( + "p.Start()", + "p.Start", + ), + ( + "p.StandardOutput.ReadToEnd()", + "p.StandardOutput.ReadToEnd", + ), + ( + "p.StandardError.ReadToEnd()", + "p.StandardError.ReadToEnd", + ), + ], + "all string literals": [ + '""', + '""', + '"Not Found"', + '"

404 Not Found

"', + '"::1"', + '"192.168.0.1"', + '"127.0.0.1"', + '"X-Forwarded-For"', + '"X-Forwarded-For"', + '"c"', + '"cmd"', + '"/c "', + '"c"', + ], + "all integer literals": [ + "404", + "0", + ], + "namespaces": ["System"], + "global statements": [ + 'string stdout = "";', + 'string stderr = "";', + ], + "properties": [ + "Current.Response.StatusCode", + "Current.Response.StatusDescription", + "Current.Request.Headers", + "UserHostAddress", + "Current.Request.Headers", + "Form", + "Form", + "RedirectStandardOutput", + "RedirectStandardError", + "UseShellExecute", + "CreateNoWindow", + "StartInfo", + ], + }, + ), + ], +) +def test_ts_extractor_engine(request: pytest.FixtureRequest, engine_str: str, expected: dict): + engine: TreeSitterExtractorEngine = request.getfixturevalue(engine_str) + do_test_ts_extractor_engine_init(engine, expected["language"]) + do_test_ts_extractor_engine_get_new_objects(engine, engine.tree.root_node, expected["all objects"]) + do_test_ts_extractor_engine_get_function_definitions( + engine, engine.tree.root_node, expected["all function definitions"] + ) + do_test_ts_extractor_engine_get_function_calls(engine, engine.tree.root_node, expected["all function calls"]) + do_test_ts_extractor_engine_get_string_literals(engine, engine.tree.root_node, expected["all string literals"]) + do_test_ts_extractor_engine_get_integer_literals(engine, engine.tree.root_node, expected["all integer literals"]) + do_test_ts_extractor_engine_get_assigned_property_names(engine, engine.tree.root_node, expected["properties"]) + do_test_ts_extractor_engine_get_global_statements(engine, expected["global statements"]) + do_test_ts_extractor_engine_get_namespaces(engine, expected["namespaces"]) + do_test_ts_base_engine_get_default_address(engine) + + +def do_test_ts_template_engine_init(engine: TreeSitterTemplateEngine): + assert engine.language == LANG_TEM + assert isinstance(engine.query, TemplateQueryBinding) + assert isinstance(engine.buf, bytes) and len(engine.buf) > 0 + assert isinstance(engine.tree, Tree) + assert isinstance(engine.get_default_address(), FileOffsetRangeAddress) + addr = engine.get_default_address() + assert addr.start_byte == engine.tree.root_node.start_byte and addr.end_byte == engine.tree.root_node.end_byte + + +def do_test_ts_template_engine_get_template_namespaces( + engine: TreeSitterTemplateEngine, expected_language: str, expected: List[str] +): + default_namespaces = LANGUAGE_TOOLKITS[expected_language].get_default_namespaces(True) + template_namespaces = set(engine.get_namespaces()) + assert default_namespaces.issubset(template_namespaces) + assert len(list(engine.get_imported_namespaces())) == len(expected) + for namespace, expected_namespace in zip(list(engine.get_imported_namespaces()), expected): + assert isinstance(namespace.node, Node) + assert engine.is_aspx_import_directive(namespace.node) + aspx_namespace = engine.get_aspx_namespace(namespace.node) + assert aspx_namespace is not None and aspx_namespace.name == expected_namespace + assert namespace.name == expected_namespace + + +def do_test_ts_template_engine_get_code_sections(engine: TreeSitterTemplateEngine, expected: List[Tuple[int, int]]): + assert len(list(engine.get_code_sections())) == len(expected) + for node, (expected_start_byte, expected_end_byte) in zip(list(engine.get_code_sections()), expected): + assert isinstance(node, Node) + assert node.start_byte == expected_start_byte and node.end_byte == expected_end_byte + + +def do_test_ts_template_engine_get_content_sections(engine: TreeSitterTemplateEngine, expected: List[Tuple[int, int]]): + assert len(list(engine.get_content_sections())) == len(expected) + for node, (expected_start_byte, expected_end_byte) in zip(list(engine.get_content_sections()), expected): + assert isinstance(node, Node) + assert node.start_byte == expected_start_byte and node.end_byte == expected_end_byte + + +def do_test_ts_template_engine_get_parsed_code_sections( + engine: TreeSitterTemplateEngine, expected_language: str, expected: List[Tuple[int, int]] +): + assert len(list(engine.get_parsed_code_sections())) == len(expected) + for extractor_engine, (expected_start_byte, _) in zip(engine.get_parsed_code_sections(), expected): + do_test_ts_extractor_engine_init(extractor_engine, expected_language) + assert extractor_engine.buf_offset == expected_start_byte + root = extractor_engine.tree.root_node + addr = extractor_engine.get_default_address() + assert ( + addr.start_byte == root.start_byte + expected_start_byte + and addr.end_byte == root.end_byte + expected_start_byte + ) + addr = extractor_engine.get_address(extractor_engine.tree.root_node) + assert ( + addr.start_byte == root.start_byte + expected_start_byte + and addr.end_byte == root.end_byte + expected_start_byte + ) + + +@parametrize( + "engine_str,expected", + [ + ( + "aspx_1f8f40_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Reflection"], + "code sections": [(2, 23), (27, 64), (68, 469)], + "content sections": [], + }, + ), + ( + "aspx_2b71dd_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Diagnostics", "System.IO"], + "code sections": [(2, 50), (55, 95), (100, 131)], + "content sections": [(52, 53), (97, 98), (133, 1273)], + }, + ), + ( + "aspx_2e8c7e_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Diagnostics", "System.IO"], + "code sections": [(2, 23), (28, 67), (72, 103)], + "content sections": [(25, 26), (69, 70), (105, 2919)], + }, + ), + ( + "aspx_03bb5c_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Web.UI.WebControls", "System.Diagnostics", "System.IO"], + "code sections": [(2, 47), (53, 100), (106, 146), (152, 183), (1659, 7702)], + "content sections": [(49, 51), (102, 104), (148, 150), (185, 1657), (7704, 10790)], + }, + ), + ( + "aspx_4f6fa6_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Diagnostics", "System.IO", "System.IO.Compression"], + "code sections": [(2, 50), (55, 95), (100, 131), (136, 179), (186, 234)], + "content sections": [(52, 53), (97, 98), (133, 134), (181, 183), (237, 6039)], + }, + ), + ( + "aspx_a35878_template_engine", + { + "language": LANG_CS, + "aspx namespaces": [ + "System.IO", + "System.Diagnostics", + "System.Data", + "System.Management", + "System.Data.OleDb", + "Microsoft.Win32", + "System.Net.Sockets", + "System.Net", + "System.Web.UI", + "System.Runtime.InteropServices", + "System.DirectoryServices", + "System.ServiceProcess", + "System.Text.RegularExpressions", + "System.Threading", + "System.Data.SqlClient", + "Microsoft.VisualBasic", + ], + "code sections": [ + (2, 123), + (128, 158), + (163, 202), + (207, 239), + (244, 282), + (287, 325), + (330, 366), + (371, 411), + (416, 448), + (453, 487), + (492, 543), + (548, 593), + (598, 640), + (645, 696), + (701, 738), + (743, 785), + (790, 832), + (837, 943), + (948, 1047), + (1052, 1155), + (1160, 1266), + ], + "content sections": [ + (125, 126), + (160, 161), + (204, 205), + (241, 242), + (284, 285), + (327, 328), + (368, 369), + (413, 414), + (450, 451), + (489, 490), + (545, 546), + (595, 596), + (642, 643), + (698, 699), + (740, 741), + (787, 788), + (834, 835), + (945, 946), + (1049, 1050), + (1157, 1158), + (1268, 2680), + ], + }, + ), + ( + "aspx_10162f_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.IO"], + "code sections": [ + (2, 71), + (76, 106), + (162, 2122), + (25579, 25596), + (25625, 25642), + (25664, 25700), + (25738, 25747), + (25801, 25822), + (25960, 25973), + (26002, 26015), + (26092, 26115), + (26153, 26168), + (26278, 26295), + (26324, 26341), + (26402, 26455), + (26472, 26489), + (26550, 26555), + (26593, 26612), + (26752, 26765), + (26794, 26811), + (26863, 26880), + (26941, 26946), + (26995, 27020), + (27037, 27062), + (27123, 27128), + (27166, 27181), + (27291, 27308), + (27337, 27354), + (27456, 27475), + (27686, 27711), + (27740, 27761), + (27854, 27879), + (27896, 27926), + (27992, 28002), + (28040, 28055), + (28167, 28188), + (28271, 28312), + (28374, 28443), + (28511, 28548), + (28610, 28675), + (28699, 28728), + (28789, 28794), + (28813, 28826), + (28871, 28876), + (28921, 28932), + (29044, 29077), + (29141, 29158), + (29220, 29226), + (29264, 29275), + (29359, 29384), + (29446, 29452), + (29490, 29501), + (29585, 29602), + (29664, 29670), + (29708, 29719), + (30163, 30170), + ], + "content sections": [ + (73, 74), + (108, 160), + (2124, 25576), + (25598, 25622), + (25644, 25661), + (25702, 25735), + (25749, 25798), + (25824, 25957), + (25975, 25999), + (26017, 26089), + (26117, 26150), + (26170, 26275), + (26297, 26321), + (26343, 26399), + (26457, 26469), + (26491, 26547), + (26557, 26590), + (26614, 26749), + (26767, 26791), + (26813, 26860), + (26882, 26938), + (26948, 26992), + (27022, 27034), + (27064, 27120), + (27130, 27163), + (27183, 27288), + (27310, 27334), + (27356, 27453), + (27477, 27683), + (27713, 27737), + (27763, 27851), + (27881, 27893), + (27928, 27989), + (28004, 28037), + (28057, 28164), + (28190, 28268), + (28314, 28371), + (28445, 28508), + (28550, 28607), + (28677, 28696), + (28730, 28786), + (28796, 28810), + (28828, 28868), + (28878, 28918), + (28934, 29041), + (29079, 29138), + (29160, 29217), + (29228, 29261), + (29277, 29356), + (29386, 29443), + (29454, 29487), + (29503, 29582), + (29604, 29661), + (29672, 29705), + (29721, 30160), + (30172, 30635), + ], + }, + ), + ( + "aspx_606dbf_template_engine", + { + "language": LANG_CS, + "aspx namespaces": [ + "System", + "System.IO", + "System.Web", + "System.Web.SessionState", + "System.Web.UI", + "System.Web.Configuration", + "System.Threading", + "System.Net", + "System.Net.Sockets", + "System.Text", + ], + "code sections": [ + (2, 87), + (93, 121), + (127, 158), + (164, 196), + (202, 247), + (253, 288), + (294, 340), + (346, 384), + (390, 422), + (428, 468), + (474, 507), + ], + "content sections": [ + (89, 91), + (123, 125), + (160, 162), + (198, 200), + (249, 251), + (290, 292), + (342, 344), + (386, 388), + (424, 426), + (470, 472), + (509, 7078), + ], + }, + ), + ( + "aspx_ea2a01_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Diagnostics", "System.IO", "System.Security.Cryptography", "System"], + "code sections": [(2, 47), (53, 93), (99, 130), (136, 186), (192, 220), (228, 5811)], + "content sections": [(49, 51), (95, 97), (132, 134), (188, 190), (222, 226), (5813, 5818)], + }, + ), + ( + "aspx_a5c893_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Reflection"], + "code sections": [(2, 23), (27, 64), (68, 469)], + "content sections": [(471, 472)], + }, + ), + ( + "aspx_b75f16_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.IO"], + "code sections": [(2, 123), (127, 157), (303, 587)], + "content sections": [(159, 301), (589, 596)], + }, + ), + ( + "aspx_d460ca_template_engine", + { + "language": LANG_CS, + "aspx namespaces": [ + "System.Reflection", + "Microsoft.CSharp", + "System.CodeDom.Compiler", + "System.IO", + "System.Security.Cryptography", + ], + "code sections": [(2, 22), (27, 65), (70, 107), (112, 156), (161, 191), (196, 245)], + "content sections": [(24, 25), (67, 68), (109, 110), (158, 159), (193, 194), (247, 4866)], + }, + ), + ( + "aspx_b4bb14_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Diagnostics", "System.IO"], + "code sections": [(2, 50), (55, 95), (100, 131)], + "content sections": [(52, 53), (97, 98), (133, 1398)], + }, + ), + ( + "aspx_f2bf20_template_engine", + { + "language": LANG_CS, + "aspx namespaces": [ + "System.IO", + "System.IO.Compression", + "System.Diagnostics", + "System.Data", + "System.Data.OleDb", + "System.Data.Common", + "System.Data.SqlClient", + "System.Management", + "Microsoft.Win32", + "System.Net", + "System.Net.Sockets", + "System.Reflection", + "System.Runtime.InteropServices", + "System.DirectoryServices", + "System.ServiceProcess", + "System.Text.RegularExpressions", + "System.Security", + "System.Security.Permissions", + "System.Threading", + ], + "code sections": [ + (2, 125), + (133, 164), + (170, 213), + (219, 259), + (265, 298), + (304, 343), + (349, 389), + (395, 438), + (444, 483), + (489, 526), + (532, 564), + (570, 610), + (616, 655), + (661, 713), + (719, 765), + (771, 814), + (820, 872), + (878, 915), + (921, 970), + (976, 1014), + (1020, 1127), + (1133, 1233), + (1239, 1343), + (39508, 39563), + (45103, 45113), + (47599, 47609), + (48705, 48712), + ], + "content sections": [ + (127, 131), + (166, 168), + (215, 217), + (261, 263), + (300, 302), + (345, 347), + (391, 393), + (440, 442), + (485, 487), + (528, 530), + (566, 568), + (612, 614), + (657, 659), + (715, 717), + (767, 769), + (816, 818), + (874, 876), + (917, 919), + (972, 974), + (1016, 1018), + (1129, 1131), + (1235, 1237), + (1345, 39505), + (39565, 45100), + (45116, 47596), + (47612, 48702), + (48715, 55896), + ], + }, + ), + ( + "aspx_5f959f_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Diagnostics", "System.IO"], + "code sections": [(2, 50), (55, 95), (100, 131)], + "content sections": [(52, 53), (97, 98), (133, 1400)], + }, + ), + ( + "aspx_f39dc0_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Diagnostics", "System.IO", "System.Net"], + "code sections": [(2, 50), (56, 96), (102, 133), (139, 171), (678, 1421)], + "content sections": [(52, 54), (98, 100), (135, 137), (173, 676), (1423, 1441)], + }, + ), + ( + "aspx_54433d_template_engine", + { + "language": LANG_CS, + "aspx namespaces": [ + "System.Diagnostics", + "System.IO", + "System.IO.Compression", + "Microsoft.VisualBasic", + ], + "code sections": [(2, 50), (55, 95), (100, 131), (136, 179), (184, 227), (233, 280)], + "content sections": [(52, 53), (97, 98), (133, 134), (181, 182), (229, 230), (283, 10444)], + }, + ), + ( + "aspx_f397cb_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System"], + "code sections": [(2, 22), (28, 56), (3950, 3981), (4033, 4064)], + "content sections": [(24, 26), (58, 3948), (3983, 4031), (4066, 4388)], + }, + ), + ( + "aspx_15eed4_template_engine", + { + "language": LANG_CS, + "aspx namespaces": [ + "System.IO", + "System.Diagnostics", + "System.Data", + "System.Management", + "System.Data.OleDb", + "Microsoft.Win32", + "System.Net.Sockets", + "System.Net", + "System.Runtime.InteropServices", + "System.DirectoryServices", + "System.ServiceProcess", + "System.Text.RegularExpressions", + "System.Threading", + "System.Data.SqlClient", + "Microsoft.VisualBasic", + ], + "code sections": [ + (2, 123), + (128, 158), + (163, 202), + (207, 239), + (244, 282), + (287, 325), + (330, 366), + (371, 411), + (416, 448), + (453, 504), + (509, 554), + (559, 601), + (606, 657), + (662, 699), + (704, 746), + (751, 793), + (798, 904), + (909, 1008), + (1013, 1116), + (1121, 1227), + (54081, 54091), + (55610, 55620), + (56304, 56315), + (57500, 57508), + (57995, 58004), + (58531, 58541), + (58984, 58994), + (59512, 59521), + (60014, 60024), + (60284, 60291), + (61559, 61564), + (62217, 62227), + (62711, 62721), + (66897, 66906), + (67954, 67962), + ], + "content sections": [ + (125, 126), + (160, 161), + (204, 205), + (241, 242), + (284, 285), + (327, 328), + (368, 369), + (413, 414), + (450, 451), + (506, 507), + (556, 557), + (603, 604), + (659, 660), + (701, 702), + (748, 749), + (795, 796), + (906, 907), + (1010, 1011), + (1118, 1119), + (1229, 54078), + (54094, 55607), + (55623, 56301), + (56318, 57497), + (57511, 57992), + (58007, 58528), + (58544, 58981), + (58997, 59509), + (59524, 60011), + (60027, 60281), + (60294, 61556), + (61567, 62214), + (62230, 62708), + (62724, 66894), + (66909, 67951), + (67965, 70053), + ], + }, + ), + ( + "aspx_6f3261_template_engine", + { + "language": LANG_CS, + "aspx namespaces": ["System.Data", "System.Data.SqlClient"], + "code sections": [(2, 23), (28, 60), (65, 107)], + "content sections": [(25, 26), (62, 63), (109, 3303)], + }, + ), + ], +) +def test_ts_template_engine(request: pytest.FixtureRequest, engine_str: str, expected: dict): + engine: TreeSitterTemplateEngine = request.getfixturevalue(engine_str) + do_test_ts_template_engine_init(engine) + assert engine.identify_language() == expected["language"] + do_test_ts_template_engine_get_template_namespaces(engine, expected["language"], expected["aspx namespaces"]) + do_test_ts_template_engine_get_code_sections(engine, expected["code sections"]) + do_test_ts_template_engine_get_parsed_code_sections(engine, expected["language"], expected["code sections"]) + do_test_ts_template_engine_get_content_sections(engine, expected["content sections"]) + for expected_start_byte, expected_end_byte in expected["content sections"]: + html_engine = TreeSitterHTMLEngine( + engine.buf[expected_start_byte:expected_end_byte], set(engine.get_namespaces()) + ) + do_test_ts_html_engine_init(html_engine) + + +def do_test_ts_html_engine_init(engine: TreeSitterHTMLEngine): + assert engine.language == LANG_HTML + assert isinstance(engine.query, HTMLQueryBinding) + assert isinstance(engine.buf, bytes) and len(engine.buf) > 0 + assert isinstance(engine.tree, Tree) + assert isinstance(engine.get_default_address(), FileOffsetRangeAddress) + assert isinstance(engine.namespaces, set) + addr = engine.get_default_address() + assert addr.start_byte == engine.tree.root_node.start_byte and addr.end_byte == engine.tree.root_node.end_byte + + +FEATURE_PRESENCE_TESTS_SCRIPTS = sorted([ + ("cs_138cdc", "global", Arch(ARCH_ANY), True), + ("cs_138cdc", "global", OS(OS_ANY), True), + ("cs_138cdc", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("cs_138cdc", "file", Format(FORMAT_SCRIPT), True), + ("cs_138cdc", "file", Namespace("System"), True), + ("cs_138cdc", "function=PSEUDO MAIN", String(""), True), + ("cs_138cdc", "function=die", String("Not Found"), True), + ("cs_138cdc", "function=Page_Load", String("127.0.0.1"), True), + ("cs_138cdc", "function=Page_Load", Class("System.Diagnostics.ProcessStartInfo"), True), + ("cs_138cdc", "function=Page_Load", API("System.Diagnostics.ProcessStartInfo::ctor"), True), + ("cs_138cdc", "function=Page_Load", Class("System.Diagnostics.Process"), True), + ("cs_138cdc", "function=Page_Load", API("System.Diagnostics.Process::ctor"), True), + ( + "cs_138cdc", + "function=Page_Load", + Property("System.Diagnostics.ProcessStartInfo::RedirectStandardOutput"), + True, + ), + ("aspx_4f6fa6", "global", Arch(ARCH_ANY), True), + ("aspx_4f6fa6", "global", OS(OS_ANY), True), + ("aspx_4f6fa6", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_4f6fa6", "file", Format(FORMAT_SCRIPT), True), + ("aspx_4f6fa6", "file", Namespace("System.Diagnostics"), True), + ("aspx_4f6fa6", "file", Namespace("System.IO"), True), + ("aspx_4f6fa6", "file", Namespace("System.IO.Compression"), True), + ("aspx_4f6fa6", "function=do_ps", String("powershell.exe"), True), + ("aspx_4f6fa6", "function=do_ps", Substring("-executionpolicy bypass"), True), + ("aspx_4f6fa6", "function=do_ps", Class("System.Diagnostics.ProcessStartInfo"), True), + ("aspx_4f6fa6", "function=do_ps", API("System.Diagnostics.ProcessStartInfo::ctor"), True), + ("aspx_4f6fa6", "function=do_ps", API("System.Diagnostics.Process::Start"), True), + ("aspx_4f6fa6", "function=ps", String("\\nPS> "), True), + ("aspx_4f6fa6", "function=ps", Substring("PS>"), True), + ("aspx_4f6fa6", "function=downloadbutton_Click", Substring("filename"), True), + ("aspx_4f6fa6", "function=base64encode", API("System.Convert::ToBase64String"), True), + ("aspx_5f959f", "global", Arch(ARCH_ANY), True), + ("aspx_5f959f", "global", OS(OS_ANY), True), + ("aspx_5f959f", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_5f959f", "file", Format(FORMAT_SCRIPT), True), + ("aspx_5f959f", "file", Namespace("System.Diagnostics"), True), + ("aspx_5f959f", "file", Namespace("System.IO"), True), + ("aspx_5f959f", "file", Namespace("System.Web.SessionState"), True), + ("aspx_5f959f", "function=ExcuteCmd", Class("System.Diagnostics.ProcessStartInfo"), True), + ("aspx_5f959f", "function=ExcuteCmd", API("System.Diagnostics.ProcessStartInfo::ctor"), True), + ("aspx_5f959f", "function=ExcuteCmd", String("cmd.exe"), True), + ("aspx_5f959f", "function=ExcuteCmd", Substring("/c"), True), + ("aspx_5f959f", "function=ExcuteCmd", API("System.Diagnostics.Process::Start"), True), + ("aspx_5f959f", "function=ExcuteCmd", Property("System.Diagnostics.ProcessStartInfo::FileName"), True), + ("aspx_5f959f", "function=ExcuteCmd", Property("System.Diagnostics.ProcessStartInfo::Arguments"), True), + ("aspx_5f959f", "function=ExcuteCmd", Property("System.Diagnostics.ProcessStartInfo::UseShellExecute"), True), + ( + "aspx_5f959f", + "function=ExcuteCmd", + Property("System.Diagnostics.ProcessStartInfo::RedirectStandardOutput"), + True, + ), + ("aspx_5f959f", "function=cmdExe_Click", String("
"), True),
+    ("aspx_5f959f", "function=cmdExe_Click", String("
"), True), + ("aspx_10162f", "global", Arch(ARCH_ANY), True), + ("aspx_10162f", "global", OS(OS_ANY), True), + ("aspx_10162f", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_10162f", "file", Format(FORMAT_SCRIPT), True), + ("aspx_10162f", "file", Namespace("System.IO"), True), + ("aspx_10162f", "file", Namespace("System.Web.Security"), True), + ("aspx_10162f", "function=PSEUDO MAIN", String("data"), True), + ("aspx_10162f", "function=PSEUDO MAIN", String("gsize"), True), + ("aspx_10162f", "function=PSEUDO MAIN", String("cmd"), True), + ("aspx_10162f", "function=PSEUDO MAIN", String("ttar"), True), + ("aspx_10162f", "function=PSEUDO MAIN", String("sdfewq@#$51234234DF@#$!@#$ASDF"), True), + ("aspx_10162f", "function=rm", API("System.IO.File::Delete"), False), + ("aspx_10162f", "function=(0x564, 0x6af)", API("System.Convert::ToBase64String"), True), + ("aspx_10162f", "function=(0x564, 0x6af)", API("System.Convert::ToBase64String"), True), + ("aspx_10162f", "function=(0x564, 0x6af)", String("p"), True), + ( + "aspx_10162f", + "function=c", + API("System.Security.Cryptography.SHA256CryptoServiceProvider::ComputeHash"), + True, + ), + ("aspx_10162f", "function=z", API("System.IO.File::ReadAllBytes"), True), + ("aspx_10162f", "function=ti", API("System.IO.File::GetCreationTime"), True), + ("aspx_10162f", "function=ti", API("System.IO.File::GetLastAccessTime"), True), + ("aspx_10162f", "function=ti", API("System.IO.File::GetCreationTime"), True), + ("aspx_10162f", "function=g", API("System.IO.File::GetLastAccessTime"), True), + ("aspx_10162f", "function=g", API("System.IO.File::GetLastWriteTime"), True), + ("aspx_10162f", "function=g", API("System.IO.File::GetLastWriteTime"), True), + ("aspx_10162f", "function=g", API("System.IO.File::SetCreationTime"), True), + ("aspx_10162f", "function=g", API("System.IO.File::SetLastAccessTime"), True), + ("aspx_10162f", "function=g", API("System.IO.File::SetLastWriteTime"), True), + ("aspx_10162f", "function=h", API("System.IO.Path::GetTempPath"), True), + ("aspx_10162f", "function=h", API("System.IO.File::WriteAllBytes"), True), + ("aspx_10162f", "function=h", API("System.Convert::FromBase64String"), True), + ("aspx_10162f", "function=d", API("System.IO.File::Delete"), True), + ("aspx_10162f", "function=d", API("System.IO.File::Delete"), True), + ("aspx_10162f", "function=sq", Class("System.Data.SqlClient.SqlConnection"), True), + ("aspx_10162f", "function=sq", API("System.Data.SqlClient.SqlConnection::ctor"), True), + ("aspx_10162f", "function=sq", Class("System.Data.SqlClient.SqlCommand"), True), + ("aspx_10162f", "function=sq", API("System.Data.SqlClient.SqlCommand::ctor"), True), + ("aspx_10162f", "function=sq", Class("System.Data.SqlClient.SqlDataAdapter"), True), + ("aspx_10162f", "function=sq", API("System.Data.SqlClient.SqlDataAdapter::ctor"), True), + ("aspx_10162f", "function=sq", API("System.Data.SqlClient.SqlConnection::Open"), True), + ("aspx_10162f", "function=exec", Class("System.Diagnostics.Process"), True), + ("aspx_10162f", "function=exec", API("System.Diagnostics.Process::ctor"), True), + ("aspx_10162f", "function=exec", String("cmd.exe"), True), + ("aspx_10162f", "function=exec", Property("System.Diagnostics.Process.StartInfo::FileName"), True), + ("aspx_10162f", "function=exec", Property("System.Diagnostics.Process.StartInfo::UseShellExecute"), True), + ("aspx_10162f", "function=exec", Property("System.Diagnostics.Process.StartInfo::RedirectStandardInput"), True), + ( + "aspx_10162f", + "function=exec", + Property("System.Diagnostics.Process.StartInfo::RedirectStandardOutput"), + True, + ), + ("aspx_10162f", "function=exec", Property("System.Diagnostics.Process.StartInfo::CreateNoWindow"), True), + ("aspx_10162f", "function=gsize", Substring("error"), True), + ("aspx_10162f", "function=exp", Substring("root"), True), + ("aspx_10162f", "function=exp", Substring("net use"), True), + ("aspx_10162f", "function=exp", Number(2), True), + ("aspx_10162f", "function=exp", Class("System.IO.DirectoryInfo"), True), + ("aspx_10162f", "function=exp", API("System.IO.DirectoryInfo::ctor"), True), + ("aspx_10162f", "function=exp", API("System.IO.File::GetAttributes"), True), + ("aspx_10162f", "function=GetDirSize", Number(0), True), + ("aspx_10162f", "function=createJsonDirectory", String('\\"dir\\":['), True), + ("aspx_10162f", "function=createJsonDirectory", Number(0), True), + ("aspx_10162f", "function=createJsonFile", Substring("file"), True), + ("aspx_10162f", "function=sizeFix", Number(1024), True), + ("aspx_10162f", "function=sizeFix", Number(2), True), + ("aspx_10162f", "function=sizeFix", Substring("GB"), True), + ("aspx_2b71dd", "global", Arch(ARCH_ANY), True), + ("aspx_2b71dd", "global", OS(OS_ANY), True), + ("aspx_2b71dd", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_2b71dd", "file", Format(FORMAT_SCRIPT), True), + ("aspx_2b71dd", "file", Namespace("System.Diagnostics"), True), + ("aspx_2b71dd", "file", Namespace("System.IO"), True), + ("aspx_2b71dd", "function=ExcuteCmd", Class("System.Diagnostics.ProcessStartInfo"), True), + ("aspx_2b71dd", "function=ExcuteCmd", API("System.Diagnostics.ProcessStartInfo::ctor"), True), + ("aspx_2b71dd", "function=ExcuteCmd", String("cmd.exe"), True), + ("aspx_2b71dd", "function=ExcuteCmd", Substring("/c"), True), + ("aspx_2b71dd", "function=ExcuteCmd", API("System.Diagnostics.Process::Start"), True), + ("aspx_2b71dd", "function=ExcuteCmd", Property("System.Diagnostics.ProcessStartInfo::FileName"), True), + ("aspx_2b71dd", "function=ExcuteCmd", Property("System.Diagnostics.ProcessStartInfo::Arguments"), True), + ("aspx_2b71dd", "function=ExcuteCmd", Property("System.Diagnostics.ProcessStartInfo::UseShellExecute"), True), + ( + "aspx_2b71dd", + "function=ExcuteCmd", + Property("System.Diagnostics.ProcessStartInfo::RedirectStandardOutput"), + True, + ), + ("aspx_f2bf20", "global", Arch(ARCH_ANY), True), + ("aspx_f39dc0", "global", Arch(ARCH_ANY), True), + ("aspx_ea2a01", "global", Arch(ARCH_ANY), True), + ("aspx_6f3261", "global", Arch(ARCH_ANY), True), + ("aspx_6f3261", "global", OS(OS_ANY), True), + ("aspx_6f3261", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_6f3261", "file", Format(FORMAT_SCRIPT), True), + ("aspx_6f3261", "file", Namespace("System.Data"), True), + ("aspx_6f3261", "file", Namespace("System.Data.SqlClient"), True), + ("aspx_6f3261", "function=PSEUDO MAIN", String("woanware"), True), + ("aspx_6f3261", "function=btnExecute_Click", Class("System.Data.SqlClient.SqlConnection"), True), + ("aspx_6f3261", "function=btnExecute_Click", API("System.Data.SqlClient.SqlConnection::ctor"), True), + ("aspx_6f3261", "function=btnExecute_Click", API("System.Data.SqlClient.SqlConnection::Open"), True), + ("aspx_6f3261", "function=btnExecute_Click", Class("System.Data.SqlClient.SqlCommand"), True), + ("aspx_6f3261", "function=btnExecute_Click", API("System.Data.SqlClient.SqlCommand::ctor"), True), + ("aspx_6f3261", "function=btnExecute_Click", API("System.Data.SqlClient.SqlCommand::ExecuteReader"), True), + ("aspx_1f8f40", "global", Arch(ARCH_ANY), True), + ("aspx_1f8f40", "global", OS(OS_ANY), True), + ("aspx_1f8f40", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_1f8f40", "file", Format(FORMAT_SCRIPT), True), + ("aspx_1f8f40", "file", Namespace("System.Reflection"), True), + ("aspx_1f8f40", "function=PSEUDO MAIN", Class("System.Security.Cryptography.RijndaelManaged"), True), + ("aspx_1f8f40", "function=PSEUDO MAIN", API("System.Security.Cryptography.RijndaelManaged::ctor"), True), + ( + "aspx_1f8f40", + "function=PSEUDO MAIN", + API("System.Security.Cryptography.RijndaelManaged::CreateDecryptor"), + True, + ), + ("aspx_2e8c7e", "global", Arch(ARCH_ANY), True), + ("aspx_2e8c7e", "global", OS(OS_ANY), True), + ("aspx_2e8c7e", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_2e8c7e", "file", Format(FORMAT_SCRIPT), True), + ("aspx_2e8c7e", "file", Namespace("System.Diagnostics"), True), + ("aspx_2e8c7e", "file", Namespace("System.IO"), True), + ("aspx_2e8c7e", "function=ExecuteCommand", Class("System.Diagnostics.ProcessStartInfo"), True), + ("aspx_2e8c7e", "function=ExecuteCommand", API("System.Diagnostics.ProcessStartInfo::ctor"), True), + ("aspx_2e8c7e", "function=ExecuteCommand", String("cmd.exe"), True), + ("aspx_2e8c7e", "function=ExecuteCommand", Substring("/c"), True), + ("aspx_2e8c7e", "function=ExecuteCommand", API("System.Diagnostics.Process::Start"), True), + ("aspx_2e8c7e", "function=ExecuteCommand", Property("System.Diagnostics.ProcessStartInfo::FileName"), True), + ("aspx_2e8c7e", "function=ExecuteCommand", Property("System.Diagnostics.ProcessStartInfo::Arguments"), True), + ( + "aspx_2e8c7e", + "function=ExecuteCommand", + Property("System.Diagnostics.ProcessStartInfo::UseShellExecute"), + True, + ), + ( + "aspx_2e8c7e", + "function=ExecuteCommand", + Property("System.Diagnostics.ProcessStartInfo::RedirectStandardOutput"), + True, + ), + ("aspx_03bb5c", "global", Arch(ARCH_ANY), True), + ("aspx_03bb5c", "global", OS(OS_ANY), True), + ("aspx_03bb5c", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_CS]), True), + ("aspx_03bb5c", "file", Format(FORMAT_SCRIPT), True), + ("aspx_03bb5c", "file", Namespace("System.Diagnostics"), True), + ("aspx_03bb5c", "file", Namespace("System.IO"), True), + ("aspx_03bb5c", "function=PSEUDO MAIN", Class("System.Diagnostics.ProcessStartInfo"), True), + ("aspx_03bb5c", "function=PSEUDO MAIN", API("System.Diagnostics.ProcessStartInfo::ctor"), True), + ("aspx_03bb5c", "function=PSEUDO MAIN", API("System.Diagnostics.Process::Start"), True), + ("aspx_03bb5c", "function=PSEUDO MAIN", Property("System.Diagnostics.ProcessStartInfo::FileName"), True), + ("aspx_03bb5c", "function=PSEUDO MAIN", Property("System.Diagnostics.ProcessStartInfo::Arguments"), True), + ("aspx_03bb5c", "function=PSEUDO MAIN", Property("System.Diagnostics.ProcessStartInfo::UseShellExecute"), True), + ( + "aspx_03bb5c", + "function=PSEUDO MAIN", + Property("System.Diagnostics.ProcessStartInfo::RedirectStandardOutput"), + True, + ), + ("aspx_606dbf", "global", Arch(ARCH_ANY), True), + ("aspx_f397cb", "global", Arch(ARCH_ANY), True), + ("aspx_b4bb14", "global", Arch(ARCH_ANY), True), + ("aspx_54433d", "global", Arch(ARCH_ANY), True), + ("aspx_a35878", "global", Arch(ARCH_ANY), True), + ("aspx_a5c893", "global", Arch(ARCH_ANY), True), + ("aspx_15eed4", "global", Arch(ARCH_ANY), True), + ("aspx_b75f16", "global", Arch(ARCH_ANY), True), + ("aspx_d460ca", "global", Arch(ARCH_ANY), True), + ("py_24e48f", "global", Arch(ARCH_ANY), True), + ("py_24e48f", "global", OS(OS_ANY), True), + ("py_24e48f", "global", ScriptLanguage(LANGUAGE_FEATURE_FORMAT[LANG_PY]), True), + ("py_24e48f", "file", Format(FORMAT_SCRIPT), True), + ("py_24e48f", "file", Namespace("socket"), True), + ("py_24e48f", "file", Namespace("threading.Timer"), True), + ("py_24e48f", "file", Namespace("threading.Timer"), True), + ("py_24e48f", "function=icloud_phish", API("subprocess::Popen"), True), + ("py_24e48f", "function=icloud_phish", Class("urllib2.Request"), True), + ("py_24e48f", "function=icloud_phish", API("base64::encodestring"), True), + ("py_24e48f", "function=icloud_phish", API("urllib2::urlopen"), True), + ("py_24e48f", "function=get_itunes_backups", String("IMEI"), True), + ("py_24e48f", "function=PSEUDO MAIN", String("[I] "), True), + ("py_24e48f", "function=PSEUDO MAIN", Substring("[!]"), True), + ("py_24e48f", "function=get_itunes_backups", Number(0), True), + ("py_24e48f", "function=get_itunes_backups", Number(1), True), + ("py_a4d252", "file", Namespace("win32com.client"), True), + ("py_a4d252", "file", Namespace("shutil"), True), + ("py_a4d252", "function=PSEUDO MAIN", API("os::environ"), True), + ("py_a4d252", "function=yut", API("shutil::copytree"), True), + ("py_a4d252", "function=yut", API("os::getcwd"), True), + ("py_a4d252", "function=takk", API("win32com.client::Dispatch"), True), + ("py_a4d252", "function=takk", String("Schedule.Service"), True), + ("py_a4d252", "function=takk", Substring("Updatewmplayer.exe"), True), + ("py_a4d252", "function=llp", API("win32api::SetFileAttributes"), True), + ("py_a4d252", "function=llp", Substring("KMPlayer"), True), + ("py_a4d252", "function=fop", API("os::remove"), True), + ("py_a4d252", "function=fop", Substring("Projec.exe"), True), + ("py_a4d252", "function=htr", API("time::sleep"), True), + ("py_a4d252", "function=htr", Number(30), True), + ("py_a4d252", "function=htr", Number(25), True), + ("py_a4d252", "function=htr", Number(10), True), + ("py_a4d252", "function=vul", Number(5), True), + ("py_a4d252", "function=vul", Number(1), True), + ("py_a4d252", "function=vul", API("os::popen"), True), + ("py_a4d252", "function=vul", String("Updatewmplayer"), True), + ("py_a4d252", "function=vul", Substring("SCHTASKS"), True), + ("py_a4d252", "function=llp", API("win32con::FILE_ATTRIBUTE_HIDDEN"), True), +]) + + +def get_extractor_ts(sample: str): + path = resolve_sample_ts(sample) + return get_ts_extractor(path) + + +def do_test_feature_presence(get_extractor, sample, scope, feature, expected): + extractor = get_extractor(sample) + features = scope(extractor) + if expected: + msg = f"{str(feature)} should be found in {scope.__name__}" + else: + msg = f"{str(feature)} should not be found in {scope.__name__}" + assert feature.evaluate(features) == expected, msg + + +def do_test_feature_count(get_extractor, sample, scope, feature, expected): + extractor = get_extractor(sample) + features = scope(extractor) + msg = f"{str(feature)} should be found {expected} times in {scope.__name__}, found: {len(features[feature])}" + assert len(features[feature]) == expected, msg + + +@parametrize( + "sample,location,feature,expected", + FEATURE_PRESENCE_TESTS_SCRIPTS, +) +def test_feature_presence_scripts(sample, location, feature, expected): + scope = resolve_scope_ts(location) + do_test_feature_presence(get_extractor_ts, sample, scope, feature, expected) From 3a94b338aadeddeea5d79a20e0219f4ac90e769f Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Thu, 13 Aug 2026 09:50:55 -0600 Subject: [PATCH 3/4] fix(extractors/ts): deduplicate qualified names and resolve module namespaces (#3150) --- capa/features/extractors/ts/function.py | 20 ++++++++++++++------ tests/test_ts.py | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/capa/features/extractors/ts/function.py b/capa/features/extractors/ts/function.py index f9f004ab96..44d10ea1e1 100644 --- a/capa/features/extractors/ts/function.py +++ b/capa/features/extractors/ts/function.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Iterator +from typing import Tuple, Iterable, Iterator from dataclasses import dataclass from tree_sitter import Node @@ -55,10 +55,16 @@ def extract_integers(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterat continue -def get_possible_full_names(name: str, namespaces: set[BaseNamespace]) -> Iterator[str]: - yield name +def get_possible_full_names(name: str, namespaces: Iterable[BaseNamespace]) -> Iterator[str]: + seen = set() + if name: + seen.add(name) + yield name for namespace in namespaces: - yield namespace.join(name) + full_name = namespace.join(name) + if full_name and full_name not in seen: + seen.add(full_name) + yield full_name def get_default_constructor(fn_node: Node, engine: TreeSitterExtractorEngine) -> Iterator[str]: @@ -84,7 +90,8 @@ def _extract_default_constructor(fn_node: Node, engine: TreeSitterExtractorEngin for name_node in engine.get_new_object_names(fn_node): for full_name in get_possible_full_names(engine.get_str(name_node), engine.namespaces): if engine.language_toolkit.is_imported_class(full_name): - yield Namespace(full_name), engine.get_address(name_node) + ns = engine.language_toolkit.get_namespace_from_name(full_name) or full_name + yield Namespace(ns), engine.get_address(name_node) yield Class(engine.language_toolkit.format_imported_class(full_name)), engine.get_address(name_node) yield ( API(engine.language_toolkit.format_imported_default_constructor(full_name)), @@ -96,7 +103,8 @@ def _extract_custom_constructor(fn_node: Node, engine: TreeSitterExtractorEngine for name_node in engine.get_function_call_names(fn_node): for full_name in get_possible_full_names(engine.get_str(name_node), engine.namespaces): if engine.language_toolkit.is_imported_constructor(full_name): - yield Namespace(full_name), engine.get_address(name_node) + ns = engine.language_toolkit.get_namespace_from_name(full_name) or full_name + yield Namespace(ns), engine.get_address(name_node) yield Class(engine.language_toolkit.format_imported_class(full_name)), engine.get_address(name_node) yield ( API(engine.language_toolkit.format_imported_custom_constructor(full_name)), diff --git a/tests/test_ts.py b/tests/test_ts.py index 3fbd3f2314..cfbf29f9ea 100644 --- a/tests/test_ts.py +++ b/tests/test_ts.py @@ -1181,6 +1181,9 @@ def do_test_ts_html_engine_init(engine: TreeSitterHTMLEngine): ("py_24e48f", "function=icloud_phish", API("base64::encodestring"), True), ("py_24e48f", "function=icloud_phish", API("urllib2::urlopen"), True), ("py_24e48f", "function=get_itunes_backups", String("IMEI"), True), + ("py_24e48f", "function=get_itunes_backups", Namespace("socket"), True), + ("py_24e48f", "function=get_itunes_backups", Class("socket.socket"), True), + ("py_24e48f", "function=get_itunes_backups", API("socket.socket::ctor"), True), ("py_24e48f", "function=PSEUDO MAIN", String("[I] "), True), ("py_24e48f", "function=PSEUDO MAIN", Substring("[!]"), True), ("py_24e48f", "function=get_itunes_backups", Number(0), True), @@ -1239,3 +1242,19 @@ def do_test_feature_count(get_extractor, sample, scope, feature, expected): def test_feature_presence_scripts(sample, location, feature, expected): scope = resolve_scope_ts(location) do_test_feature_presence(get_extractor_ts, sample, scope, feature, expected) + + +FEATURE_COUNT_TESTS_SCRIPTS = ( + ("py_24e48f", "function=get_itunes_backups", Namespace("socket"), 2), + ("py_24e48f", "function=get_itunes_backups", Class("socket.socket"), 1), + ("py_24e48f", "function=get_itunes_backups", API("socket.socket::ctor"), 1), +) + + +@parametrize( + "sample,location,feature,expected", + FEATURE_COUNT_TESTS_SCRIPTS, +) +def test_feature_count_scripts(sample, location, feature, expected): + scope = resolve_scope_ts(location) + do_test_feature_count(get_extractor_ts, sample, scope, feature, expected) From 2d2d8ebca2b3dc1f8e98d4f0ea6699be8aea1eb0 Mon Sep 17 00:00:00 2001 From: saniyafatima07 Date: Sat, 8 Aug 2026 01:18:30 +0530 Subject: [PATCH 4/4] Add documentation for script analysis --- .github/CONTRIBUTING.md | 6 ++- README.md | 32 +++++++++++++ doc/script-analysis.md | 100 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 doc/script-analysis.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 056be94ae8..0b71b08eba 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -36,7 +36,7 @@ We host the capa project as three GitHub repositories: - [capa-rules](https://github.com/mandiant/capa-rules) - [capa-testfiles](https://github.com/mandiant/capa-testfiles) -The command line tools, logic engine, and other Python source code are found in the `capa` repository. +The command line tools, logic engine, scripting language analysis, and other Python source code are found in the `capa` repository. This is the repository to fork when you want to enhance the features, performance, or user interface of capa. Do *not* push rules directly to this repository, instead... @@ -46,11 +46,13 @@ We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and d If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`. Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way. -Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository. +Test fixtures, such as malware samples, script samples and analysis workspaces, are found in the `capa-testfiles` repository. These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites; however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate, a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth. +For more information on developing and extending Script Analysis, see the [Script Analysis documentation](../doc/script-analysis.md). + ### Design Decisions When we make a significant decision in how we maintain the project and what we can or cannot support, diff --git a/README.md b/README.md index b75190372c..d0e6683864 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,38 @@ $ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.json ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ ``` +# script analysis + +capa also supports static analysis of script-based files using [Tree-sitter](https://tree-sitter.github.io/tree-sitter/). Script analysis currently supports Python (`.py`), C# (`.cs`), HTML/ASPX (`.html`, `.aspx`) and Bash (`.sh`), allowing capa to identify capabilities in scripts. + +Here is an example for running a capa against a script: + +``` +$ capa 606dbfebdc7751ecb6cb9a845853ae1905afd4b8a2cb54e1e4a98c932e268712.aspx_ +┌──────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +| md5 | a73d772a6db180ea12c7f0c7dbb8eaf2 | +| sha1 | acf361167ea92fff4a3d509edab7025590faa337 | +| sha256 | 606dbfebdc7751ecb6cb9a845853ae1905afd4b8a2cb54e1e4a98c932e268712 | +| analysis | static | +| os | any | +| format | script | +| arch | any | +| path | path/606dbfebdc7751ecb6cb9a845853ae1905afd4b8a2cb54e1e4a98c932e268712.aspx_ | +└──────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ MBC Objective ┃ MBC Behavior ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ FILE SYSTEM │ Delete File [C0047] │ +└──────────────────────────────────────────────────┴───────────────────────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Capability ┃ Namespace ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ delete file │ host-interaction/file-system/delete │ +└─────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +For more details, see the [Script Analysis documentation](doc/script-analysis.md). + # capa rules capa uses a collection of rules to identify capabilities within a program. These rules are easy to write, even for those new to reverse engineering. diff --git a/doc/script-analysis.md b/doc/script-analysis.md new file mode 100644 index 0000000000..68d7e8c66a --- /dev/null +++ b/doc/script-analysis.md @@ -0,0 +1,100 @@ +# Script Analysis (Tree-Sitter) + +Script analysis extends capa to support the analysis of script-based files which includes Python(.py), C#(.cs), template languages such as ASPX and HTML (.aspx, .html) and Bash(.sh). It uses [Tree-sitter](https://tree-sitter.github.io/tree-sitter/), an incremental parsing library that generates a concrete syntax tree(CST). + +The Tree-sitter extractor walks the syntax tree, extracts semantic features such as imported modules, instantiated classes called APIs, string literals, properties and matches them against the existing capa rule set. Script analysis uses the same rule engine and output pipeline as binary analysis. + +## Supported languages + +| Language | File Extension(s) | Description | +|------------- |-------------------- |--------------------------------------------| +| Python | `.py` | Python scripts | +| C# | `.cs` | C# source files | +| HTML / ASPX | `.html`, `.aspx` | HTML templates with embedded server-side code | +| Bash | `.sh` | Bash shell scripts | + +## Workflow + +Script analysis introduces the `FORMAT_SCRIPT` file format alongside `FORMAT_PE`, `FORMAT_ELF` and `FORMAT_DOTNET`. Once a file is identified as a script, capa routes analysis through the Tree-sitter extractor. Rule matching and result rendering are identical to binary analysis. + +1. **Detect the source language** using either extension-based or content-based detection. +2. **Parse the source** into a Tree-sitter syntax tree. +3. **Extract semantic features** such as imports, classes, function calls, properties, and strings. +4. **Map extracted symbols** using language-specific signature files. +5. **Evaluate capa rules** against the extracted features. + +## Embedded Templates + +Some file formats embed one language inside another. For example, an ASPX page can contain HTML along with embedded C# or VB.NET code blocks (`<% ... %>`). + +Template files are first parsed using the outer language grammar to identify embedded code blocks. These blocks are then parsed using the appropriate grammar for the embedded language, allowing features to be extracted from both the outer and embedded layers. + +## Repository layout + +Tree-sitter extraction lives under: + +``` +capa/features/extractors/ts/ +├── autodetect.py # language detection +├── engine.py # Tree-sitter query engine +├── tools.py # shared parsing helpers +└── signatures/ + ├── cs.json + ├── py.json + ├── sh.json + └── ... +``` + +## Signature files + +Each supported language provides a signature file under +`capa/features/extractors/ts/signatures/`. + +Signature files map language-specific symbols to capa features. + +Example: + +```json +{ + "classes": [ + "socket.socket", + "subprocess.Popen" + ] +} +``` + +## Usage + +Script analysis is invoked the same way as binary analysis. + +```console +$ capa sample.py +$ capa script.sh +$ capa webshell.aspx +``` + +## Testing + +Tree-sitter tests live in `tests/test_ts.py`. + +Representative source samples are maintained in the `capa-testfiles` repository. + +## Installation + +Script analysis can currently be used by installing capa from the source code or by building a standalone binary using PyInstaller. See [Method 3: Inspecting the capa source code](installation.md#method-3-inspecting-the-capa-source-code) for instructions. + +It will also be available through the official standalone binaries and the Python package installed with `pip` in an upcoming capa release. + +## Contributing Guidelines + +When adding support for a new language or extending support for an existing language, consider the following: + +1. **Add dependencies and detection:** Add the required Tree-sitter runtime and language grammar packages to `requirements.txt` and `pyproject.toml`, and update language detection to recognize the new language and its file extensions. + +2. **Add extraction support:** Implement the required Tree-sitter parsing and feature extraction logic for the language. + +3. **Add signatures and rules:** Add a language-specific signature file under `capa/features/extractors/ts/signatures/` and add or update capa rules in the `capa-rules` repository to match the extracted features and capabilities. + +4. **Add test samples:** Add source samples to the [`capa-testfiles`](https://github.com/mandiant/capa-testfiles) repository and corresponding tests in the `capa` repository. + +5. **Run the tests:** Run the relevant Tree-sitter tests and verify that the expected features and capabilities are extracted from the samples.