From ae55cda103ac915c3638d766f8cd9ee76b582f2d Mon Sep 17 00:00:00 2001 From: mariusmue Date: Tue, 9 Mar 2021 00:09:29 +0100 Subject: [PATCH 1/7] Add binary, remove, and comment arguments to cli This allows to execute specific actions on a binary of choice in the database. This commit implements the addition of comments, and removal of the binary from the database. --- polypyus/cli.py | 25 ++++++++++++++++++++++++- polypyus/models.py | 2 ++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/polypyus/cli.py b/polypyus/cli.py index 2292448..efefad3 100644 --- a/polypyus/cli.py +++ b/polypyus/cli.py @@ -118,7 +118,7 @@ def format_binary_list( binary_list: Iterable[Binary], is_annotated: Optional[bool] = False ) -> Iterable[dict]: - keys = ["id", "name", "filepath"] + keys = ["id", "name", "filepath", "comment"] data = [] for binary in list(binaries): b_dict = binary.to_dict(only=keys) @@ -147,6 +147,16 @@ def format_binary_list( typer.echo(table) +@orm.db_session +@logger.catch +def _binary_ops(binary: str, comment: str, remove: bool): + b = Binary.get(name=binary) + if comment is not None: + b.comment = comment + if remove is True: + Binary.delete(b) + + @app.command() @show_time def analyze( @@ -162,6 +172,9 @@ def analyze( False, help="List annotated binaries registered in project" ), list_targets: bool = typer.Option(False, help="List targets registered in project"), + binary: str = typer.Option("", help="Perform action on specific binary"), + comment: str = typer.Option(None, help="Add comment to binary"), + remove: bool = typer.Option(False, help="remove binary from database"), ): """ Analyze targets with matchers generated from the given history (annotated binaries). @@ -179,6 +192,14 @@ def analyze( --min-size the minimum size in bytes a function needs to have to be considered for matcher creation. + --list-history/target lists the registered history or target binaries + + --binary specifies a specific binary to perform operations on + + --comment adds given comment to binary (requires --binary) + + --remove removes binary from database + """ if len(history) != len(annotation): @@ -193,6 +214,8 @@ def analyze( bind_db(project) if list_history is True or list_targets is True: _cli_list(list_history, list_targets) + elif binary != "": + _binary_ops(binary, comment, remove) else: _analyze(history, annotation, target, parallelize, min_size, max_rel_fuzz) diff --git a/polypyus/models.py b/polypyus/models.py index 7f4cd11..0eaf8d0 100644 --- a/polypyus/models.py +++ b/polypyus/models.py @@ -67,6 +67,7 @@ class Binary(DB.Entity, Serializable): raw (bytes): binary blob, obtained by reading the binary functions (Set): Set of functions found in the binary matched_by (Set): Set of matches that matched something in this binary + comment (str): A user defined comment for additional information """ @@ -78,6 +79,7 @@ class Binary(DB.Entity, Serializable): functions = Set("Function") is_target = Required(bool, default=False) partitions = Optional(Json) + comment = Optional(str) @property def path_obj(self): From f0beb3a34bb92e74eb79a729842646558c6e0efe Mon Sep 17 00:00:00 2001 From: mariusmue Date: Tue, 9 Mar 2021 15:06:41 +0100 Subject: [PATCH 2/7] add match-field to list-targets --- polypyus/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/polypyus/cli.py b/polypyus/cli.py index efefad3..70495bb 100644 --- a/polypyus/cli.py +++ b/polypyus/cli.py @@ -125,6 +125,8 @@ def format_binary_list( if is_annotated is True: b_dict["#annotations"] = binary.annotations.count() b_dict["#functions"] = binary.functions.count() + else: # this is a target binary + b_dict["#matches"] = binary.matches.count() data.append(b_dict) return data From 741596a06a3fe85918f38c07d31ba0417d4350dd Mon Sep 17 00:00:00 2001 From: mariusmue Date: Tue, 9 Mar 2021 15:37:27 +0100 Subject: [PATCH 3/7] bugfix: set is_target to True on target binaries --- polypyus/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polypyus/importer.py b/polypyus/importer.py index d65ed65..69c48f0 100644 --- a/polypyus/importer.py +++ b/polypyus/importer.py @@ -106,7 +106,7 @@ def get_or_create_binary(path: Path, make_target=False) -> Binary: if not path.is_file(): raise FileNotFoundError binary = Binary(filepath=str(path), name=path.name) - elif make_target: + if make_target: binary.is_target = True binary.partition() return binary From 24d3caf30e89f01ca365746261d0bd5d1efcdcd3 Mon Sep 17 00:00:00 2001 From: mariusmue Date: Tue, 9 Mar 2021 15:40:24 +0100 Subject: [PATCH 4/7] Allow csv-export from cli and for history binaries --- polypyus/cli.py | 12 ++++++++++-- polypyus/exporter.py | 28 ++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/polypyus/cli.py b/polypyus/cli.py index 70495bb..98a8e98 100644 --- a/polypyus/cli.py +++ b/polypyus/cli.py @@ -21,6 +21,7 @@ ) from polypyus.graph import Graph from polypyus.importer import get_or_create_annotation, get_or_create_binary +from polypyus.exporter import export_csv_combined from polypyus.models import DB, Binary, Function, Match, Matcher from polypyus.tools import format_addr, format_data, format_percentage, serialize @@ -151,10 +152,12 @@ def format_binary_list( @orm.db_session @logger.catch -def _binary_ops(binary: str, comment: str, remove: bool): +def _binary_ops(binary: str, comment: str, export_csv: str, remove: bool): b = Binary.get(name=binary) if comment is not None: b.comment = comment + if export_csv is not None: + export_csv_combined(b, export_csv) if remove is True: Binary.delete(b) @@ -176,6 +179,9 @@ def analyze( list_targets: bool = typer.Option(False, help="List targets registered in project"), binary: str = typer.Option("", help="Perform action on specific binary"), comment: str = typer.Option(None, help="Add comment to binary"), + export_csv: str = typer.Option( + None, help="Export annotations/matches to specified csv file" + ), remove: bool = typer.Option(False, help="remove binary from database"), ): """ @@ -200,6 +206,8 @@ def analyze( --comment adds given comment to binary (requires --binary) + --export-csv exports all matches/annotations for given binary to specified csv file + --remove removes binary from database """ @@ -217,7 +225,7 @@ def analyze( if list_history is True or list_targets is True: _cli_list(list_history, list_targets) elif binary != "": - _binary_ops(binary, comment, remove) + _binary_ops(binary, comment, export_csv, remove) else: _analyze(history, annotation, target, parallelize, min_size, max_rel_fuzz) diff --git a/polypyus/exporter.py b/polypyus/exporter.py index bdebf75..2b94583 100644 --- a/polypyus/exporter.py +++ b/polypyus/exporter.py @@ -6,25 +6,45 @@ from polypyus.models import Binary from polypyus.tools import serialize +from typing import Iterable + csv.register_dialect("space_delimiter", delimiter=" ", quoting=csv.QUOTE_NONE) +def export_csv_combined(binary: Binary, path: Path): + if binary.is_target is True: + export_matches_csv(binary, path) + else: + export_annotations_csv(binary, path) + + def export_matches_csv(binary: Binary, path: Path): logger.info(f"exporting matches for {binary.name} csv to {path}") stream = serialize(binary.matches, export=True) + _export_csv_internal(stream, path) + + +def export_annotations_csv(binary: Binary, path: Path): + logger.info(f"exporting annotations for {binary.name} csv to {path}") + stream = serialize(binary.functions, export=True) + _export_csv_internal(stream, path) + + +def _export_csv_internal(stream: Iterable[dict], path: Path): with open(path, "w") as csv_file: writer = csv.DictWriter( csv_file, fieldnames=CSV_KEYS_LONG, dialect="space_delimiter" ) writer.writeheader() - for match in stream: - match["addr"] = hex(match["addr"]) - match["name"] = match["name"].split(", ")[0] + for entry in stream: + entry["addr"] = hex(entry["addr"]) + entry["name"] = entry["name"].split(", ")[0] + print(entry) writer.writerow( { key: value - for key, value in {**match, "type": "FUNC"}.items() + for key, value in {**entry, "type": "FUNC"}.items() if key in CSV_KEYS_LONG } ) From 1035c3b4df28c70549195d7b295a41aaa51cc016 Mon Sep 17 00:00:00 2001 From: mariusmue Date: Fri, 12 Mar 2021 20:46:52 +0100 Subject: [PATCH 5/7] remove forgotten debug print --- polypyus/exporter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/polypyus/exporter.py b/polypyus/exporter.py index 2b94583..3b74f9b 100644 --- a/polypyus/exporter.py +++ b/polypyus/exporter.py @@ -40,7 +40,6 @@ def _export_csv_internal(stream: Iterable[dict], path: Path): for entry in stream: entry["addr"] = hex(entry["addr"]) entry["name"] = entry["name"].split(", ")[0] - print(entry) writer.writerow( { key: value From cb477845159679d80eca8a6fd068107d9dc590c0 Mon Sep 17 00:00:00 2001 From: mariusmue Date: Wed, 17 Mar 2021 14:05:03 +0100 Subject: [PATCH 6/7] remove erroneous spaces in comments --- polypyus/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/polypyus/models.py b/polypyus/models.py index 4ce575c..c222fe4 100644 --- a/polypyus/models.py +++ b/polypyus/models.py @@ -63,11 +63,11 @@ class Binary(DB.Entity, Serializable): """Binary model that stores raw bin blob as well as filepath. Attributes: - filepath (str): relative filepath to the binary - raw (bytes): binary blob, obtained by reading the binary - functions (Set): Set of functions found in the binary - matched_by (Set): Set of matches that matched something in this binary - comment (str): A user defined comment for additional information + filepath (str): relative filepath to the binary + raw (bytes): binary blob, obtained by reading the binary + functions (Set): Set of functions found in the binary + matched_by (Set): Set of matches that matched something in this binary + comment (str): A user defined comment for additional information """ name = Required(str) @@ -305,7 +305,7 @@ def start_blobs(cls, cut: int = 8) -> Iterable[Tuple[bytes, List["Function"]]]: # TODO: consider different exec modes for fnc in cls.select(lambda f: f.size >= cut): data = fnc.dump()[:cut] - if any(d != 0x0 for d in data) and any(d != 0xFF for d in data): + if any(d != 0x0 for d in data) and any(d != 0xff for d in data): starts[bytes(data)].append(fnc) return sorted(starts.items(), key=lambda x: len(x[1]), reverse=True) From 5333cbc9949b84747bbb3f2ef02a2c04593b9e0d Mon Sep 17 00:00:00 2001 From: mariusmue Date: Wed, 17 Mar 2021 14:16:49 +0100 Subject: [PATCH 7/7] minimal formatting changes due to different black versions --- polypyus/exporter.py | 1 - polypyus/models.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/polypyus/exporter.py b/polypyus/exporter.py index 5e31e26..e2e3e52 100644 --- a/polypyus/exporter.py +++ b/polypyus/exporter.py @@ -11,7 +11,6 @@ csv.register_dialect("space_delimiter", delimiter=" ", quoting=csv.QUOTE_MINIMAL) - def export_csv_combined(binary: Binary, path: Path): if binary.is_target is True: export_matches_csv(binary, path) diff --git a/polypyus/models.py b/polypyus/models.py index c222fe4..9efb83c 100644 --- a/polypyus/models.py +++ b/polypyus/models.py @@ -305,7 +305,7 @@ def start_blobs(cls, cut: int = 8) -> Iterable[Tuple[bytes, List["Function"]]]: # TODO: consider different exec modes for fnc in cls.select(lambda f: f.size >= cut): data = fnc.dump()[:cut] - if any(d != 0x0 for d in data) and any(d != 0xff for d in data): + if any(d != 0x0 for d in data) and any(d != 0xFF for d in data): starts[bytes(data)].append(fnc) return sorted(starts.items(), key=lambda x: len(x[1]), reverse=True)