From f4fab0bb06e0fc0fd0166614cc3503b6cf8f465d Mon Sep 17 00:00:00 2001 From: hapv Date: Tue, 25 Nov 2025 21:32:44 +0700 Subject: [PATCH 1/3] Optimize merge function to reduce memory usage with iterative merging (fixes #23) --- src/ucis/cmd/cmd_merge.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/ucis/cmd/cmd_merge.py b/src/ucis/cmd/cmd_merge.py index 5504d16..93490f4 100644 --- a/src/ucis/cmd/cmd_merge.py +++ b/src/ucis/cmd/cmd_merge.py @@ -25,29 +25,33 @@ def merge(args): input_desc : FormatDescDb = rgy.getDatabaseDesc(args.input_format) output_desc : FormatDescDb = rgy.getDatabaseDesc(args.output_format) - db_l : List[UCIS] = [] + out_if = output_desc.fmt_if() + out_db : UCIS = out_if.create() + db_if : FormatIfDb = input_desc.fmt_if() + merger = DbMerger() + for input in args.db: - db_if : FormatIfDb = input_desc.fmt_if() + print("read and merge: ", input) + out_db_ref : UCIS = out_if.create() + db_l : List[UCIS] = [] try: db = db_if.read(input) db_l.append(db) + db_l.append(out_db) except Exception as e: raise Exception("Failed to read input file %s: %s" % ( input, str(e) )) - out_if = output_desc.fmt_if() - out_db : UCIS = out_if.create() + try: + merger.merge(out_db_ref, db_l) + except Exception as e: + raise Exception("Merge operation failed: %s" % str(e)) - merger = DbMerger() - try: - merger.merge(out_db, db_l) - except Exception as e: - raise Exception("Merge operation failed: %s" % str(e)) + out_db = out_db_ref + db.close() out_db.write(args.out) - for db in db_l: - db.close() From e4409e47908eb9c999c1c64f60e7e1102da5c6f7 Mon Sep 17 00:00:00 2001 From: hapv Date: Tue, 25 Nov 2025 21:56:40 +0700 Subject: [PATCH 2/3] feat: Add --file-list option to merge command and comprehensive tests for it. --- src/ucis/__main__.py | 5 ++++- src/ucis/cmd/cmd_merge.py | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/ucis/__main__.py b/src/ucis/__main__.py index a87ca6a..5ceece1 100644 --- a/src/ucis/__main__.py +++ b/src/ucis/__main__.py @@ -42,9 +42,12 @@ def get_parser(): help="Specifies the format of the input databases. Defaults to 'xml'") merge.add_argument("--output-format", "-of", help="Specifies the format of the input databases. Defaults to 'xml'") + merge.add_argument("--file-list", "-f", + help="File containing list of databases to merge (one per line)") merge.add_argument("--libucis", "-l", help="Specifies the name/path of the UCIS shared library") - merge.add_argument("db", nargs="+") + merge.add_argument("db", nargs="*", + help="Database files to merge (can be combined with --file-list)") merge.set_defaults(func=cmd_merge.merge) list_db_formats = subparser.add_parser("list-db-formats", diff --git a/src/ucis/cmd/cmd_merge.py b/src/ucis/cmd/cmd_merge.py index 93490f4..2944484 100644 --- a/src/ucis/cmd/cmd_merge.py +++ b/src/ucis/cmd/cmd_merge.py @@ -22,6 +22,26 @@ def merge(args): if not rgy.hasDatabaseFormat(args.output_format): raise Exception("Output format %s not recognized" % args.output_format) + # Build list of databases to merge + db_files = list(args.db) if args.db else [] + + # Read from file list if provided + if args.file_list: + import os + if not os.path.exists(args.file_list): + raise Exception("File list not found: %s" % args.file_list) + + with open(args.file_list, 'r') as f: + for line in f: + line = line.strip() + # Skip comments and blank lines + if line and not line.startswith('#'): + db_files.append(line) + + # Validate we have at least one database + if len(db_files) == 0: + raise Exception("No input databases specified. Use --file-list or provide database arguments.") + input_desc : FormatDescDb = rgy.getDatabaseDesc(args.input_format) output_desc : FormatDescDb = rgy.getDatabaseDesc(args.output_format) @@ -30,7 +50,7 @@ def merge(args): db_if : FormatIfDb = input_desc.fmt_if() merger = DbMerger() - for input in args.db: + for input in db_files: print("read and merge: ", input) out_db_ref : UCIS = out_if.create() db_l : List[UCIS] = [] From 54b565e0c1af2af92e09d8767f012ddff384e5cf Mon Sep 17 00:00:00 2001 From: hapv Date: Sun, 8 Mar 2026 22:13:20 +0700 Subject: [PATCH 3/3] test & implement history --- merge.lists | 6 + test_combined.xml | 29 +++ test_coverage_example.waive.xml | 12 + test_coverage_example.xml | 169 ++++++++++++++ test_db1.yaml | 12 + test_db2.yaml | 12 + test_merged.waive.xml | 6 + test_merged.xml | 29 +++ ve/unit/test_file_list/README.md | 22 ++ ve/unit/test_file_list/__init__.py | 1 + .../test_file_list/test_file_list_feature.py | 210 ++++++++++++++++++ 11 files changed, 508 insertions(+) create mode 100644 merge.lists create mode 100644 test_combined.xml create mode 100644 test_coverage_example.waive.xml create mode 100644 test_coverage_example.xml create mode 100644 test_db1.yaml create mode 100644 test_db2.yaml create mode 100644 test_merged.waive.xml create mode 100644 test_merged.xml create mode 100644 ve/unit/test_file_list/README.md create mode 100644 ve/unit/test_file_list/__init__.py create mode 100644 ve/unit/test_file_list/test_file_list_feature.py diff --git a/merge.lists b/merge.lists new file mode 100644 index 0000000..b29c4d3 --- /dev/null +++ b/merge.lists @@ -0,0 +1,6 @@ +# Test file list for merge +# This is a comment and should be ignored +test_db1.yaml + +# Another comment +test_db2.yaml diff --git a/test_combined.xml b/test_combined.xml new file mode 100644 index 0000000..82a2a24 --- /dev/null +++ b/test_combined.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_coverage_example.waive.xml b/test_coverage_example.waive.xml new file mode 100644 index 0000000..e085dee --- /dev/null +++ b/test_coverage_example.waive.xml @@ -0,0 +1,12 @@ + + + + unused + + +abc + + +xyz + + \ No newline at end of file diff --git a/test_coverage_example.xml b/test_coverage_example.xml new file mode 100644 index 0000000..05d580d --- /dev/null +++ b/test_coverage_example.xml @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_db1.yaml b/test_db1.yaml new file mode 100644 index 0000000..bf2e7a2 --- /dev/null +++ b/test_db1.yaml @@ -0,0 +1,12 @@ +coverage: + covergroups: + - name: cvg1 + instances: + - name: inst1 + coverpoints: + - name: cp1 + bins: + - name: b0 + count: 1 + - name: b1 + count: 0 diff --git a/test_db2.yaml b/test_db2.yaml new file mode 100644 index 0000000..d7d46de --- /dev/null +++ b/test_db2.yaml @@ -0,0 +1,12 @@ +coverage: + covergroups: + - name: cvg1 + instances: + - name: inst1 + coverpoints: + - name: cp1 + bins: + - name: b0 + count: 0 + - name: b1 + count: 1 diff --git a/test_merged.waive.xml b/test_merged.waive.xml new file mode 100644 index 0000000..2d53db7 --- /dev/null +++ b/test_merged.waive.xml @@ -0,0 +1,6 @@ + + + + this item can waive + + \ No newline at end of file diff --git a/test_merged.xml b/test_merged.xml new file mode 100644 index 0000000..cdd95fb --- /dev/null +++ b/test_merged.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ve/unit/test_file_list/README.md b/ve/unit/test_file_list/README.md new file mode 100644 index 0000000..3057ba7 --- /dev/null +++ b/ve/unit/test_file_list/README.md @@ -0,0 +1,22 @@ +# Test directory for file list feature + +This directory contains tests for the `--file-list` feature of the merge command. + +## Tests + +- `test_file_list_feature.py` - Comprehensive tests including: + - Basic file list functionality + - Comments and blank lines handling + - Combined file list + direct arguments + - Large scale test with 1000 databases + - Error handling + +## Running Tests + +```bash +# Run all file list tests +python -m unittest ve.unit.test_file_list.test_file_list_feature -v + +# Run specific test +python -m unittest ve.unit.test_file_list.test_file_list_feature.TestFileList.test_file_list_large_scale -v +``` diff --git a/ve/unit/test_file_list/__init__.py b/ve/unit/test_file_list/__init__.py new file mode 100644 index 0000000..baeb63a --- /dev/null +++ b/ve/unit/test_file_list/__init__.py @@ -0,0 +1 @@ +# Empty __init__.py to make this a Python package diff --git a/ve/unit/test_file_list/test_file_list_feature.py b/ve/unit/test_file_list/test_file_list_feature.py new file mode 100644 index 0000000..8f1dec6 --- /dev/null +++ b/ve/unit/test_file_list/test_file_list_feature.py @@ -0,0 +1,210 @@ +''' +Test for file list feature with large number of databases + +This test creates thousands of small coverage databases and merges them +using the --file-list feature to verify it works at scale. +''' +import os +import sys +import tempfile +import shutil +from unittest.case import TestCase +from ucis.xml.xml_reader import XmlReader +from ucis.mem.mem_factory import MemFactory +from ucis.report.coverage_report_builder import CoverageReportBuilder +from ucis.__main__ import get_parser + + +class TestFileList(TestCase): + + def setUp(self): + """Create temporary directory for test databases""" + self.test_dir = tempfile.mkdtemp(prefix="pyucis_test_") + + def tearDown(self): + """Clean up temporary directory""" + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + def _create_test_db(self, name, bin_values): + """Create a simple test database with specified bin values using YAML""" + yaml_text = f""" + coverage: + covergroups: + - name: test_cg + instances: + - name: test_inst + coverpoints: + - name: test_cp + bins: +""" + for i, value in enumerate(bin_values): + yaml_text += f" - name: bin_{i}\n" + yaml_text += f" count: {value}\n" + + # Create database from YAML + from ucis.yaml.yaml_reader import YamlReader + from ucis.xml.xml_writer import XmlWriter + db = YamlReader().loads(yaml_text) + + # Write to XML file + filepath = os.path.join(self.test_dir, name) + writer = XmlWriter() + with open(filepath, 'w') as f: + writer.write(f, db) + return filepath + + def test_file_list_basic(self): + """Test basic file list functionality""" + # Create a few test databases + db1 = self._create_test_db("db1.xml", [1, 0]) + db2 = self._create_test_db("db2.xml", [0, 1]) + + # Create file list + file_list_path = os.path.join(self.test_dir, "merge.list") + with open(file_list_path, 'w') as f: + f.write("# Test file list\n") + f.write(f"{db1}\n") + f.write("\n") # blank line + f.write(f"# Comment\n") + f.write(f"{db2}\n") + + # Merge using file list + output_path = os.path.join(self.test_dir, "merged.xml") + parser = get_parser() + args = parser.parse_args([ + "merge", + "-f", file_list_path, + "-o", output_path + ]) + + args.func(args) + + # Verify output exists + self.assertTrue(os.path.exists(output_path)) + + # Verify merged content + merged_db = XmlReader().read(output_path) + rpt = CoverageReportBuilder.build(merged_db) + + self.assertEqual(len(rpt.covergroups), 1) + self.assertEqual(len(rpt.covergroups[0].coverpoints), 1) + self.assertEqual(len(rpt.covergroups[0].coverpoints[0].bins), 2) + # Both bins should have count=1 after merge + self.assertEqual(rpt.covergroups[0].coverpoints[0].coverage, 100.0) + + def test_file_list_with_direct_args(self): + """Test combining file list with direct arguments""" + db1 = self._create_test_db("db1.xml", [1, 0]) + db2 = self._create_test_db("db2.xml", [0, 1]) + db3 = self._create_test_db("db3.xml", [1, 1]) + + # Create file list with only db1 and db2 + file_list_path = os.path.join(self.test_dir, "merge.list") + with open(file_list_path, 'w') as f: + f.write(f"{db1}\n") + f.write(f"{db2}\n") + + # Merge using file list + direct argument (db3) + output_path = os.path.join(self.test_dir, "merged.xml") + parser = get_parser() + args = parser.parse_args([ + "merge", + "-f", file_list_path, + db3, + "-o", output_path + ]) + + args.func(args) + + # Verify output + self.assertTrue(os.path.exists(output_path)) + merged_db = XmlReader().read(output_path) + rpt = CoverageReportBuilder.build(merged_db) + + # Should have merged all 3 databases + self.assertEqual(len(rpt.covergroups[0].coverpoints[0].bins), 2) + + def test_file_list_large_scale(self): + """Test file list with thousands of databases""" + num_databases = 1000 # Create 1000 databases + print(f"\nCreating {num_databases} test databases...") + + # Create databases with different coverage patterns + db_paths = [] + for i in range(num_databases): + # Alternate bin coverage patterns + bin_values = [1, 0] if i % 2 == 0 else [0, 1] + db_path = self._create_test_db(f"db_{i:04d}.xml", bin_values) + db_paths.append(db_path) + + print(f"Created {len(db_paths)} databases") + + # Create file list + file_list_path = os.path.join(self.test_dir, "large_merge.list") + with open(file_list_path, 'w') as f: + f.write(f"# Large scale test with {num_databases} databases\n") + for db_path in db_paths: + f.write(f"{db_path}\n") + + print(f"Created file list with {num_databases} entries") + + # Merge using file list + output_path = os.path.join(self.test_dir, "merged_large.xml") + parser = get_parser() + args = parser.parse_args([ + "merge", + "-f", file_list_path, + "-o", output_path + ]) + + print("Starting merge...") + args.func(args) + print("Merge completed") + + # Verify output + self.assertTrue(os.path.exists(output_path)) + + # Verify merged content + merged_db = XmlReader().read(output_path) + rpt = CoverageReportBuilder.build(merged_db) + + self.assertEqual(len(rpt.covergroups), 1) + self.assertEqual(len(rpt.covergroups[0].coverpoints), 1) + self.assertEqual(len(rpt.covergroups[0].coverpoints[0].bins), 2) + + # With 1000 databases alternating between [1,0] and [0,1], + # both bins should be fully covered + self.assertEqual(rpt.covergroups[0].coverpoints[0].coverage, 100.0) + + print(f"Successfully merged {num_databases} databases") + print(f"Final coverage: {rpt.covergroups[0].coverpoints[0].coverage}%") + + def test_file_list_missing_file(self): + """Test error handling for missing file list""" + output_path = os.path.join(self.test_dir, "merged.xml") + parser = get_parser() + args = parser.parse_args([ + "merge", + "-f", "nonexistent.list", + "-o", output_path + ]) + + with self.assertRaises(Exception) as context: + args.func(args) + + self.assertIn("File list not found", str(context.exception)) + + def test_no_databases_specified(self): + """Test error when no databases are specified""" + output_path = os.path.join(self.test_dir, "merged.xml") + parser = get_parser() + args = parser.parse_args([ + "merge", + "-o", output_path + ]) + + with self.assertRaises(Exception) as context: + args.func(args) + + self.assertIn("No input databases specified", str(context.exception))