diff --git a/README.md b/README.md index d27a375..e0278f8 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,116 @@ if you use a Python interface to configure your data plane (as part of your tests). The `--interface` option (or `-i`) can be used to specify the interfaces on which to inject packets (along with the corresponding port number). +## Using PTF as a Python library + +The `ptf` binary is a command line parser around the library module +`ptf.runner`. The library supports scoped in-process execution. + +### Run tests in the current process + +Do these steps: + +1. Construct a `PtfConfig` object. Each option group of the binary corresponds + to one dataclass: `TestSelectionOptions`, `PlatformOptions`, + `LoggingOptions`, `TestBehaviorOptions`, and `SocketOptions`. +2. Give this object to `runner.run()`. +3. Read the return value of `run()`. The value `0` means that all selected + tests passed. The value `1` means that at least one test failed, errored, or + was skipped while `fail_skipped` is set. + +`run()` performs the same steps as the binary: it sets up logging, loads the +test modules and the platform, starts the data plane, executes the tests, and +releases PTF-owned resources. It restores PTF configuration, imported test and +platform modules, Python paths, random state, logging, profiling, and test +utility globals before returning. A +fatal configuration or environment problem raises `PtfError`. + +Only one `run()` call may be active in a process because existing PTF tests use +process-global configuration and dataplane objects. The call can run on a +non-main thread unless a selected test uses a signal-based timeout. Independent +PTF processes, including parallel invocations of the `ptf` binary, remain fully +supported. + +```python +from ptf import runner + +config = runner.PtfConfig( + pypath=["/path/to/my/library"], # same as --pypath + test_selection=runner.TestSelectionOptions( + test_dir="mytests/", + test_specs=["standard"], # same as the positional arguments + ), + platform=runner.PlatformOptions( + platform="eth", + interfaces=[ # same as --interface + runner.Interface(0, 0, "veth1"), + runner.Interface(0, 1, "veth3"), + ], + ), + logging=runner.LoggingOptions(log_file="ptf.log"), + test_behavior=runner.TestBehaviorOptions( + # Same as --test-params: a dictionary, or a "key=value;key=value" + # string. The binary and run() evaluate each string value as a Python + # expression; a dictionary entry is passed to the tests as given. + test_params={"key1": 17, "key2": True}, + ), +) +exit_code = runner.run(config) +``` + +### Integrate output and logging + +`RunOutput` configures runtime-only integration. PTF does not close or replace +handlers owned by the caller. Framework output and unittest progress can be +routed independently, and PTF log records can be forwarded to a caller-owned +logger: + +```python +output = runner.RunOutput( + stdout=my_output_stream, + stderr=my_status_stream, + logger=logging.getLogger("my_application.ptf"), +) +exit_code = runner.run(config, output=output) +``` + +Set `LoggingOptions(log_file=None)` when the application should receive PTF +records without creating PTF-owned log and pcap files. + +Test code which logs through `logging.*` continues to use the application's +root logging policy. PTF modules log below the `ptf` logger. The command line +and serialized-configuration entry points additionally capture root logging to +preserve the traditional PTF log-file behavior. + +### Configuration formats + +* `PtfConfig.to_dict()` converts the configuration to the flat dictionary + format of the global `ptf.config`. `PtfConfig.from_dict()` converts such a + dictionary back to a `PtfConfig`. `run()` also accepts the dictionary + directly. +* `PtfConfig.to_json()` converts the configuration to JSON text. + `PtfConfig.from_json()` converts the JSON text back to a `PtfConfig`. + Values in `extra_config` must themselves be JSON-serializable. + +### Run a serialized configuration + +To run a serialized configuration in a process that a caller has already +isolated, for example in a network namespace, write the configuration with +`to_json()` and run: + + python -m ptf.runner + +Use `-` instead of `` to read the JSON configuration from stdin. + +### Global state + +Test modules retain compatibility with the global `ptf.config` dictionary, +`ptf.dataplane_instance`, and globals in `ptf.testutils` and `ptf.ptfutils`. +`run()` scopes and restores that state, but the packet manipulation module and +its feature flags are fixed when `ptf.packet` is first imported. A later +in-process run requesting a different packet configuration raises `PtfError`; +start a new PTF process for that case. + ## Install PTF PTF can be installed with `uv`: diff --git a/ptf b/ptf index 124520b..e0325ce 100755 --- a/ptf +++ b/ptf @@ -10,28 +10,17 @@ """ @package ptf -Packet Test Framework (ptf) top level script +Packet Test Framework (ptf) top level script. -To add a new command line option, edit both the config_default dictionary and -the config_setup function. The option's result will end up in the global -oftest.config dictionary. +This script is a thin wrapper around the library. It calls ptf.cli.main(). +That function parses the command line, builds a ptf.runner.PtfConfig, and +starts the run through the library (ptf.runner.run). The test-run logic is +in the ptf package. Other programs can call the library directly, without +this script. See the module docstring of ptf.runner. """ -import sys -import argparse -from subprocess import Popen, PIPE -import logging -import unittest -import time import os -import importlib -import random -import signal -import fnmatch -import copy -import shutil -import types -from collections import OrderedDict +import sys root_dir = os.path.dirname(os.path.realpath(__file__)) @@ -40,950 +29,7 @@ if os.path.exists(os.path.join(pydir, "ptf")): # Running from source tree sys.path.insert(0, pydir) -import ptf -from ptf import config, __version__ -import ptf.ptfutils - -##@var DEBUG_LEVELS -# Map from strings to debugging levels -DEBUG_LEVELS = { - "debug": logging.DEBUG, - "verbose": logging.DEBUG, - "info": logging.INFO, - "warning": logging.WARNING, - "warn": logging.WARNING, - "error": logging.ERROR, - "critical": logging.CRITICAL, -} - -##@var config_default -# The default configuration dictionary for PTF -config_default = { - # Miscellaneous options - "list": False, - "list_test_names": False, - "allow_user": False, - # Test selection options - "test_spec": "", - "test_file": None, - "test_dir": None, - "test_order": "default", - "test_order_seed": 0xABA, - "num_shards": 1, - "shard_id": 0, - # Switch connection options - "platform": "eth", - "platform_args": None, - "platform_dir": None, - "interfaces": [], - "port_info": {}, - "device_sockets": [], # when using nanomsg - # Logging options - "log_file": "ptf.log", - "log_dir": None, - "debug": "verbose", - "profile": False, - "profile_file": "profile.out", - "xunit": False, - "xunit_dir": "xunit", - # Test behavior options - "relax": False, - "test_params": None, - "failfast": False, - "fail_skipped": False, - "default_timeout": 2.0, - "default_negative_timeout": 0.1, - "minsize": 0, - "random_seed": None, - "disable_ipv6": False, - "disable_vxlan": False, - "disable_erspan": False, - "disable_geneve": False, - "disable_mpls": False, - "disable_nvgre": False, - "disable_igmp": False, - "disable_rocev2": False, - "qlen": 100, - "test_case_timeout": None, - # Socket options - "socket_recv_size": 4096, - # Other configuration - "port_map": None, -} - - -def import_module(root_path, module_name): - """Try to import a module and class directly instead of the typical - Python method. Allows for dynamic imports.""" - finder = importlib.machinery.PathFinder() - module_specs = finder.find_spec(module_name, [root_path]) - return module_specs.loader.load_module() - - -def config_setup(): - """ - Set up the configuration including parsing the arguments - - @return A pair (config, args) where config is an config - object and args is any additional arguments from the command line - """ - - usage = "usage: ptf [options] --test-dir TEST_DIR [tests]" - - description = """PTF (Packet Test Framework) is a framework and set of tests -to test a software switch. It is strongly inspired by the OFTest framework, but -it is not tied to OpenFlow. It does not provide any control plane features, but -it is targetted at helping you test a dataplane. - -The default configuration assumes that interfaces veth1, veth3, veth5, and veth7 -should be connected to the switch's dataplane. - -If no positional arguments are given then OFTest will run all tests found in the ---test-dir directory. Otherwise each positional argument is interpreted as -either a test name or a test group name. The union of these will be executed. To -see what groups each test belongs to use the --list option. Tests and groups can -be subtracted from the result by prefixing them with the '^' character. """ - - class ActionInterface(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - # Parse --interface - def check_interface(value): - port_cfg = {} - sp = ";" - try: - if sp in value: - value, p_info = value.split(sp, 1) - params = p_info.split(sp) - for elem in params: - key, val = elem.split("=") - port_cfg[key.lower()] = val - dev_and_port, interface = value.split("@", 1) - dev_and_port = dev_and_port.split("-") - if len(dev_and_port) == 1: - dev, port = 0, int(dev_and_port[0]) - elif len(dev_and_port) == 2: - dev, port = int(dev_and_port[0]), int(dev_and_port[1]) - else: - raise ValueError("") - if port_cfg: - getattr(namespace, "port_info")[port] = port_cfg - except ValueError: - parser.error( - "incorrect interface syntax (got %s, expected 'port@interface' or 'device-port@interface' \ - or providing port configuration using 'device-port@interface;arg=val;arg2=val...' )" - % repr(value) - ) - return (dev, port, interface) - - assert type(values) is str - getattr(namespace, self.dest).append(check_interface(values)) - - class ActionDeviceSocket(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - # Parse --device-socket - def check_device_socket(value): - def parse_ports(ports): - port_set = set() - try: - ports = ports.strip("{}") - ports = ports.split(",") - except: - raise ValueError("") - for port in ports: - try: - p = int(port) - port_set.add(p) - continue - except: - pass - try: - p1, p2 = port.split("-", 1) - p1, p2 = int(p1), int(p2) - for p in range(p1, p2 + 1): # p2 included - port_set.add(p) - except: - raise ValueError("") - return port_set - - try: - dev_and_port, addr = value.split("@", 1) - if dev_and_port[0] == "{": - dev, ports = (0, parse_ports(dev_and_port)) - else: - dev_and_port = dev_and_port.split("-", 1) - if len(dev_and_port) != 2: - raise ValueError("") - dev, ports = ( - int(dev_and_port[0]), - parse_ports(dev_and_port[1]), - ) - except ValueError: - parser.error( - "incorrect device-socket syntax (got %s, expected something of the form 0-{1,2,5-8}@)" - % repr(value) - ) - return (dev, ports, addr) - - assert type(values) is str - getattr(namespace, self.dest).append(check_device_socket(values)) - - class ActionTestDir(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - assert type(values) is str - if not os.path.isdir(values): - parser.error( - "invalid value for --test-dir: directory %s does not exist" % values - ) - setattr(namespace, self.dest, values) - - parser = argparse.ArgumentParser(usage=usage, description=description) - - # Set up default values - parser.set_defaults(**config_default) - - parser.add_argument("--version", action="version", version=__version__) - - parser.add_argument("test_specs", nargs="*", help="Tests / Groups to run") - - parser.add_argument("--list", action="store_true", help="List all tests and exit") - parser.add_argument( - "--list-test-names", - action="store_true", - help="List test names matching the test spec and exit", - ) - parser.add_argument( - "--allow-user", - action="store_true", - help="Proceed even if ptf is not run as root", - ) - - parser.add_argument("--pypath", dest="pypath", action="append") - - parser.add_argument( - "-pmm", - "--packet-manipulation-module", - type=str, - help="Provide packet manipulation module which should be used " - "as a 'packet' one for other PTF modules", - ) - - group = parser.add_argument_group("Test selection options") - group.add_argument("-f", "--test-file", help="File of tests to run, one per line") - group.add_argument( - "--test-dir", - type=str, - action=ActionTestDir, - required=True, - help="Directory containing tests", - ) - test_order_help = """Choose the order in which the tests will be run: - default (tests are run in the order in which they appear on command line), - lexico (use default string ordering on test names), - rand (random order, use --test-order-seed to specify a seed) - """ - group.add_argument( - "--test-order", choices=["default", "lexico", "rand"], help=test_order_help - ) - group.add_argument( - "--test-order-seed", type=int, help="Specify seed to randomize test order" - ) - group.add_argument( - "--num-shards", - type=int, - help="Number of shards that can be used to parallelize test execution", - ) - group.add_argument( - "--shard-id", type=int, help="Index of shard (>= 0 and < number of shards)" - ) - - group = parser.add_argument_group("Switch connection options") - group.add_argument("-P", "--platform", help="Platform module name") - group.add_argument( - "-a", "--platform-args", help="Custom arguments for the platform" - ) - group.add_argument( - "--platform-dir", type=str, help="Directory containing platform modules" - ) - group.add_argument( - "--interface", - "-i", - type=str, - dest="interfaces", - metavar="INTERFACE", - action=ActionInterface, - help="Specify a port number and the dataplane interface to use. May be given multiple times. Example: 1@eth1 or 0-1@eth2 (use eth2 as port 1 of device 0)", - ) - group.add_argument( - "--device-socket", - type=str, - dest="device_sockets", - metavar="DEVICE-SOCKET", - action=ActionDeviceSocket, - help="Specify the nanomsg socket to use to send / receive packets for a given device, as well as the ports to enable on the device. May be given multiple times. Example: 0-{1,2,5-8}@", - ) - - group = parser.add_argument_group("Logging options") - group.add_argument("--log-file", help="Name of log file") - group.add_argument("--log-dir", help="Name of log directory") - dbg_lvl_names = sorted(list(DEBUG_LEVELS.keys()), key=lambda x: DEBUG_LEVELS[x]) - group.add_argument( - "--debug", - choices=dbg_lvl_names, - help="Debug lvl: debug, info, warning, error, critical", - ) - group.add_argument( - "--verbose", - action="store_const", - dest="debug", - const="verbose", - help="Shortcut for --debug=verbose", - ) - group.add_argument( - "-q", - "--quiet", - action="store_const", - dest="debug", - const="warning", - help="Shortcut for --debug=warning", - ) - group.add_argument("--profile", action="store_true", help="Enable Python profiling") - group.add_argument("--profile-file", help="Output file for Python profiler") - group.add_argument( - "--xunit", action="store_true", help="Enable xUnit-formatted results" - ) - group.add_argument( - "--xunit-dir", help="Output directory for xUnit-formatted results" - ) - - group = parser.add_argument_group("Test behavior options") - group.add_argument( - "--relax", - action="store_true", - help="Relax packet match checks allowing other packets", - ) - group.add_argument( - "--failfast", - action="store_true", - help="Stop running tests as soon as one fails", - ) - test_params_help = """Set test parameters: [key=val]*;key=val - """ - group.add_argument("-t", "--test-params", help=test_params_help) - group.add_argument( - "--fail-skipped", - action="store_true", - help="Return failure if any test was skipped", - ) - group.add_argument( - "--default-timeout", type=float, help="Timeout in seconds for most operations" - ) - group.add_argument( - "--default-negative-timeout", - type=float, - help="Timeout in seconds for negative checks", - ) - group.add_argument( - "--minsize", type=int, help="Minimum allowable packet size on the dataplane." - ) - group.add_argument("--random-seed", type=int, help="Random number generator seed") - group.add_argument("--disable-ipv6", action="store_true", help="Disable IPv6 tests") - group.add_argument("--qlen", type=int, help="Default queue length ") - group.add_argument( - "--test-case-timeout", - type=int, - help="Timeout for each test case, 0 means no timeout", - ) - - group.add_argument( - "--disable-vxlan", - action="store_true", - help="Disable VXLAN (do not import from scapy even if supported)", - ) - group.add_argument( - "--disable-geneve", - action="store_true", - help="Disable GENEVE (do not import from scapy even if supported)", - ) - group.add_argument( - "--disable-erspan", - action="store_true", - help="Disable ERSPAN (do not import from scapy even if supported)", - ) - group.add_argument( - "--disable-mpls", - action="store_true", - help="Disable MPLS (do not import from scapy even if supported)", - ) - group.add_argument( - "--disable-nvgre", - action="store_true", - help="Disable NVGRE (do not import from scapy even if supported)", - ) - group.add_argument( - "--disable-igmp", - action="store_true", - help="Disable IGMP (do not import from scapy even if supported)", - ) - - group = parser.add_argument_group("Socket options") - group.add_argument( - "--socket-recv-size", - type=int, - help="When using raw sockets, specify the size of the buffer used to receive packets with socket.recv.", - ) - - # Might need this if other parsers want command line - # parser.allow_interspersed_args = False - args = parser.parse_args() - if args.pypath: - for p in args.pypath: - sys.path.append(p) - - # Convert args from a Namespace to a plain dictionary - config = config_default.copy() - for key in config.keys(): - config[key] = getattr(args, key) - # For selecting the packet manipulation module when running the - # `ptf` command, the order of precedence is: - # (1) If the `--packet-manipulation-module` command line option is - # present, use its value. - # (2) Otherwise, if the environment variable - # PTF_PACKET_MANIPULATION_MODULE is defined, use its value. - # (3) Otherwise, use "ptf.packet_scapy" - pmm_key = "packet_manipulation_module" - if getattr(args, pmm_key): - # Then use the value from the command line option. - config[pmm_key] = getattr(args, pmm_key) - elif os.getenv("PTF_PACKET_MANIPULATION_MODULE"): - config[pmm_key] = os.getenv("PTF_PACKET_MANIPULATION_MODULE") - else: - config[pmm_key] = "ptf.packet_scapy" - - return (config, args) - - -def logging_setup(config): - """ - Set up logging based on config - """ - - logging.getLogger().setLevel(DEBUG_LEVELS[config["debug"]]) - - if config["log_dir"] != None: - if os.path.exists(config["log_dir"]): - shutil.rmtree(config["log_dir"]) - os.makedirs(config["log_dir"]) - ptf.ptfutils.chown_to_invoking_user(config["log_dir"]) - else: - if os.path.exists(config["log_file"]): - os.remove(config["log_file"]) - - ptf.open_logfile("main") - - -def xunit_setup(config): - """ - Set up xUnit output based on config - """ - - if not config["xunit"]: - return - - if os.path.exists(config["xunit_dir"]): - shutil.rmtree(config["xunit_dir"]) - os.makedirs(config["xunit_dir"]) - ptf.ptfutils.chown_to_invoking_user(config["xunit_dir"]) - - -def pcap_setup(config): - """ - Set up dataplane packet capturing based on config - """ - - if config["log_dir"] == None: - filename = os.path.splitext(config["log_file"])[0] + ".pcap" - ptf.dataplane_instance.start_pcap(filename) - else: - # start_pcap is called per-test in base_tests - pass - - -def profiler_setup(config): - """ - Set up profiler based on config - """ - - if not config["profile"]: - return - - import cProfile - - profiler = cProfile.Profile() - profiler.enable() - - return profiler - - -def profiler_teardown(profiler): - """ - Tear down profiler based on config - """ - - if not config["profile"]: - return - - profiler.disable() - profiler.dump_stats(config["profile_file"]) - ptf.ptfutils.chown_to_invoking_user(config["profile_file"]) - - -def load_test_modules(config): - """ - Load tests from the test_dir directory. - - Test cases are subclasses of unittest.TestCase - - Also updates the _groups member to include "standard" and - module test groups if appropriate. - - @param config The ptf configuration dictionary - @returns A dictionary from test module names to tuples of - (module, dictionary from test names to test classes). - """ - - result = OrderedDict() - - for root, dirs, filenames in os.walk(config["test_dir"]): - pyfiles = fnmatch.filter(filenames, "[!.]*.py") - - # guarantee that files will be visited in the same order every time tests are loaded - pyfiles.sort() - dirs.sort() - - if len(pyfiles) == 0: - continue - - # Allow tests to import each other - sys.path.append(root) - - # Iterate over each python file - for filename in pyfiles: - modname = os.path.splitext(os.path.basename(filename))[0] - - try: - if modname in sys.modules: - mod = sys.modules[modname] - else: - mod = import_module(root, modname) - except: - logging.warning("Could not import file " + filename) - raise - - # Find all testcases defined in the module - tests = dict( - (k, v) - for (k, v) in mod.__dict__.items() - if type(v) == type - and issubclass(v, unittest.TestCase) - and hasattr(v, "runTest") - ) - if tests: - for testname, test in tests.items(): - # Set default annotation values - if not hasattr(test, "_groups"): - test._groups = [] - if not hasattr(test, "_nonstandard"): - test._nonstandard = False - if not hasattr(test, "_disabled"): - test._disabled = False - if not hasattr(test, "_testtimeout"): - test._testtimeout = None - - # Put test in its module's test group - if not test._disabled: - test._groups.append(modname) - else: - # If the test is disabled, create a group named - # disabled and add it too. This is so that - # users can conveniently exclude disabled tests - # too when including only groups. Eg. - # -s "group1 ^disabled" - test._groups.append("disabled") - - # Put test in the standard test group - if not test._disabled and not test._nonstandard: - test._groups.append("standard") - test._groups.append("all") # backwards compatibility - - result[modname] = (mod, tests) - - return result - - -def prune_tests(test_specs, test_modules): - """ - Return tests matching the given test-specs. - @param test_specs A list of group names or test names. - @param test_modules Same format as the output of load_test_modules. - @returns Same format as the output of load_test_modules. - """ - result = OrderedDict() - for e in test_specs: - matched = False - - if e.startswith("^"): - negated = True - e = e[1:] - else: - negated = False - - for modname, (mod, tests) in test_modules.items(): - for testname, test in tests.items(): - if e in test._groups or e == "%s.%s" % (modname, testname): - result.setdefault(modname, (mod, OrderedDict())) - if not negated: - # if not hasattr(test, "_versions") or version in test._versions: - result[modname][1][testname] = test - else: - if modname in result and testname in result[modname][1]: - del result[modname][1][testname] - if not result[modname][1]: - del result[modname] - matched = True - - if not matched and not negated: - die("test-spec element %s did not match any tests" % e) - - return result - - -def apply_test_timeout(test, default_test_case_timeout=None): - original_run = test.run - - def run_with_timeout(self, result=None): - from ptf.ptfutils import Timeout - - test_case_timeout = getattr(self, "_testtimeout", None) - if test_case_timeout is None: - test_case_timeout = default_test_case_timeout - - if test_case_timeout: - with Timeout(test_case_timeout): - return original_run(result) - return original_run(result) - - test.run = types.MethodType(run_with_timeout, test) - return test - - -def die(msg, exit_val=1): - logging.critical(msg) - sys.exit(exit_val) - - -def _space_to(n, str): - """ - Generate a string of spaces to achieve width n given string str - If length of str >= n, return one space - """ - spaces = n - len(str) - if spaces > 0: - return " " * spaces - return " " - - -def test_params_parse(config): - test_params = config["test_params"] - if test_params is None: - logging.debug("No test params were provided with '--test-params' / '-t'") - return None - params_str = "class _TestParams:\n " + test_params - namespace = {} - try: - exec(params_str, namespace) - except: - logging.error( - "Error when parsing test params " - "(provided with '--test-params' / '-t'). " - "Make sure you used the correct syntax: " - '--test-params="[k=v;]*k=v"' - ) - return None - params = {} - logging.debug("Parsed test parameters:") - for k, v in list(vars(namespace["_TestParams"]).items()): - if k[:2] != "__": - params[k] = v - logging.debug("\t*{}={}".format(k, v)) - logging.debug( - "If something is missing, make sure you used the correct syntax: " - '--test-params="[k=v;]*k=v"' - ) - return params - - -# -# Main script -# - -# Setup global configuration -new_config, args = config_setup() -ptf.config.update(new_config) - -logging_setup(config) -xunit_setup(config) -logging.info("++++++++ " + time.asctime() + " ++++++++") - -# import after logging is configured so that scapy error logs (from importing -# packet.py) are silenced and our own warnings are logged properly. -import ptf.testutils -import ptf.ptfutils - -# Try parsing test params and log them -# We do this before importing the test modules in case test parameters are being -# accessed at test import time. -ptf.testutils.TEST_PARAMS = test_params_parse(config) -# Initiallize port information -ptf.testutils.PORT_INFO = config["port_info"] - -test_specs = args.test_specs -if config["test_file"] != None: - with open(config["test_file"], "r") as f: - for line in f: - line, _, _ = line.partition("#") # remove comments - line = line.strip() - if line: - test_specs.append(line) -if test_specs == []: - test_specs = ["standard"] - -test_modules = load_test_modules(config) - -# Check if test list is requested; display and exit if so -if config["list"]: - mod_count = 0 - test_count = 0 - all_groups = set() - print("""\ -Tests are shown grouped by module. If a test is in any groups beyond "standard" -and its module's group then they are shown in parentheses.""") - print() - print("""\ -Tests marked with '!' are disabled because they are experimental, special-purpose, -or are too long to be run normally. These are not part of the "standard" test -group or their module's test group.""") - print() - print("Test List:") - for modname, (mod, tests) in test_modules.items(): - mod_count += 1 - desc = (mod.__doc__ or "No description").strip().split("\n")[0] - start_str = " Module " + mod.__name__ + ": " - print(start_str + _space_to(22, start_str) + desc) - for testname, test in list(tests.items()): - try: - desc = (test.__doc__ or "").strip() - desc = desc.split("\n")[0] - except: - desc = "No description" - groups = set(test._groups) - set(["all", "standard", modname]) - all_groups.update(test._groups) - if groups: - desc = "(%s) %s" % (",".join(groups), desc) - if hasattr(test, "_versions"): - desc = "(%s) %s" % (",".join(sorted(test._versions)), desc) - start_str = " %s%s %s:" % ( - test._nonstandard and "*" or " ", - test._disabled and "!" or " ", - testname, - ) - if len(start_str) > 22: - desc = "\n" + _space_to(22, "") + desc - print(start_str + _space_to(22, start_str) + desc) - test_count += 1 - print() - print("%d modules shown with a total of %d tests" % (mod_count, test_count)) - print() - print("Test groups: %s" % (", ".join(sorted(all_groups)))) - - sys.exit(0) - -test_modules = prune_tests(test_specs, test_modules) - -# Check if test list is requested; display and exit if so -if config["list_test_names"]: - for modname, (mod, tests) in test_modules.items(): - for testname, test in tests.items(): - print("%s.%s" % (modname, testname)) - sys.exit(0) - -# Generate the test suite -test_suite = [] -for modname, (mod, tests) in test_modules.items(): - for testname, test in tests.items(): - test_suite.append(test()) - -if config["shard_id"] < 0 or config["shard_id"] >= config["num_shards"]: - die("shard id should be equal or greater than 0 and lower than number of shards") -test_suite = test_suite[config["shard_id"] :: config["num_shards"]] - -if config["test_order"] == "lexico": - test_suite.sort() -elif config["test_order"] == "rand": - seed = config["test_order_seed"] - random.seed(seed) - random.shuffle(test_suite) - -test_suite = [ - apply_test_timeout(test, config["test_case_timeout"]) for test in test_suite -] -test_suite = unittest.TestSuite(test_suite) - - -if config["platform_dir"] is None: - from ptf import platforms - - config["platform_dir"] = os.path.dirname(os.path.abspath(platforms.__file__)) - -# Allow platforms to import each other -sys.path.append(config["platform_dir"]) - -# Load the platform module -platform_name = config["platform"] -logging.info("Importing platform: " + platform_name) - -# TODO(antonin): put this check in platforms/nn.py ? -if platform_name == "nn": - try: - import pynng - except: - die("Cannot use 'nn' platform if pynng package is not installed") - -platform_mod = None -try: - platform_mod = import_module(config["platform_dir"], platform_name) -except: - logging.warn("Failed to import " + platform_name + " platform module") - raise - -try: - platform_mod.platform_config_update(config) -except: - logging.warn("Could not run platform host configuration") - raise - -if config["port_map"] is None: - die("Interface port map was not defined by the platform. Exiting.") - -logging.debug("Configuration: " + str(config)) -logging.info("port map: " + str(config["port_map"])) - -ptf.ptfutils.default_timeout = config["default_timeout"] -ptf.ptfutils.default_negative_timeout = config["default_negative_timeout"] -ptf.testutils.MINSIZE = config["minsize"] - -if os.getuid() != 0 and not config["allow_user"] and platform_name != "nn": - die("Super-user privileges required. Please re-run with sudo or as root.") - -if config["random_seed"] is not None: - logging.info("Random seed: %d" % config["random_seed"]) - random.seed(config["random_seed"]) -else: - # Generate random seed and report to log file - seed = random.randrange(100000000) - logging.info("Autogen random seed: %d" % seed) - random.seed(seed) - -# Remove python's signal handler which raises KeyboardError. Exiting from an -# exception waits for all threads to terminate which might not happen. -signal.signal(signal.SIGINT, signal.SIG_DFL) +from ptf.cli import main if __name__ == "__main__": - profiler = profiler_setup(config) - - if config["port_map"]: - import ptf.dataplane - - # Set up the dataplane only when the selected platform exposes ports. - ptf.dataplane_instance = ptf.dataplane.DataPlane(config) - pcap_setup(config) - for port_id, ifname in config["port_map"].items(): - device, port = port_id - ptf.dataplane_instance.port_add(ifname, device, port) - else: - ptf.dataplane_instance = None - - logging.info("*** TEST RUN START: " + time.asctime()) - if config["xunit"]: - try: - import xmlrunner # fail-fast if module missing - except ImportError as ex: - ptf.dataplane_instance.kill() - profiler_teardown(profiler) - raise ex - runner = xmlrunner.XMLTestRunner( - output=config["xunit_dir"], outsuffix="", verbosity=2 - ) - else: - runner = unittest.TextTestRunner(verbosity=2) - result = runner.run(test_suite) - if config["xunit"]: - # The XML result files are only written once the run completes. - ptf.ptfutils.chown_to_invoking_user(config["xunit_dir"], recursive=True) - run_failures = result.failures - run_errors = result.errors - run_timeouts = [] - for case in result.errors: - traceback_str = case[1] - # TODO: hacky? could not think of a better way - if "raise Timeout.TimeoutError()" in traceback_str: - logging.info("Test case failed because of timeout") - run_timeouts.append(case) - - ptf.open_logfile("main") - if ptf.testutils.skipped_test_count > 0: - ts = " tests" - if ptf.testutils.skipped_test_count == 1: - ts = " test" - logging.info("Skipped " + str(ptf.testutils.skipped_test_count) + ts) - print("Skipped " + str(ptf.testutils.skipped_test_count) + ts) - logging.info("*** TEST RUN END : " + time.asctime()) - - # Shutdown the dataplane - if ptf.dataplane_instance is not None: - ptf.dataplane_instance.stop_pcap() # no-op is pcap not started - ptf.dataplane_instance.kill() - ptf.dataplane_instance = None - - profiler_teardown(profiler) - - if run_failures or run_errors: - print() - print("******************************************") - print("ATTENTION: SOME TESTS DID NOT PASS!!!") - if (not config["xunit"]) and run_failures: - print() - print("The following tests failed:") - print(", ".join([f[0].__class__.__name__ for f in run_failures])) - if (not config["xunit"]) and run_errors: - print() - print("The following tests errored:") - print(", ".join([f[0].__class__.__name__ for f in run_errors])) - if (not config["xunit"]) and run_timeouts: - print() - print("The following tests errored because of a timeout:") - print(", ".join([f[0].__class__.__name__ for f in run_timeouts])) - print() - print("******************************************") - # exit(1) hangs sometimes - sys.stdout.flush() - sys.stderr.flush() - os._exit(1) - if ptf.testutils.skipped_test_count > 0 and config["fail_skipped"]: - print() - print("******************************************") - print("ATTENTION: %d TESTS WERE SKIPPED!!!", ptf.testutils.skipped_test_count) - print("******************************************") - print() - sys.stdout.flush() - sys.stderr.flush() - os._exit(1) + sys.exit(main()) diff --git a/src/bf_pktpy/ptf/packet_pktpy.py b/src/bf_pktpy/ptf/packet_pktpy.py index b4db78f..933e0aa 100644 --- a/src/bf_pktpy/ptf/packet_pktpy.py +++ b/src/bf_pktpy/ptf/packet_pktpy.py @@ -10,11 +10,29 @@ see PTF documentation (section "Pluggable packet manipulation module"). """ -import bf_pktpy.packets +import logging + import bf_pktpy.commands +import bf_pktpy.packets from bf_pktpy.all import hexdump as bf_pktpy_hexdump, ls as bf_pktpy_ls from ptf import config +logger = logging.getLogger(__name__) + +_ptf_packet_config = { + name: config.get(name, False) + for name in ( + "disable_ipv6", + "disable_vxlan", + "disable_erspan", + "disable_geneve", + "disable_mpls", + "disable_nvgre", + "disable_igmp", + "disable_rocev2", + ) +} + # Headers set to None are not yet implemented (or conditionally being set) Packet = bf_pktpy.packets.Packet Ether = bf_pktpy.packets.Ether @@ -53,7 +71,7 @@ ERSPAN_III = bf_pktpy.packets.ERSPAN_III PlatformSpecific = bf_pktpy.packets.ERSPAN_PlatformSpecific except ImportError as e: - print("ERSPAN support not found in bf_pktpy. Details:\n%s" % e) + logger.warning("ERSPAN support not found in bf_pktpy. Details:\n%s", e) GENEVE = None @@ -68,7 +86,7 @@ try: IGMP = bf_pktpy.packets.IGMP except ImportError as e: - print("IGMP support not found in bf_pktpy. Details:\n%s" % e) + logger.warning("IGMP support not found in bf_pktpy. Details:\n%s", e) ############################################################################## @@ -112,6 +130,18 @@ def get_erspan_alternative(): hexdump = bf_pktpy_hexdump ls = bf_pktpy_ls + +def format_hexdump(value): + result = bf_pktpy_hexdump(value, dump=True) + if isinstance(result, (list, tuple)): + return "\n".join(result) + return str(result) + + +def format_packet(value): + return str(value) + + # The names below are assigned here so that, like the other names # above, they can be used by importers of the ptf.packet module as if # they were defined inside of ptf.packet, and they are commonly diff --git a/src/ptf/__init__.py b/src/ptf/__init__.py index bae606e..347c061 100644 --- a/src/ptf/__init__.py +++ b/src/ptf/__init__.py @@ -30,6 +30,20 @@ # Populated by oft. dataplane_instance = None +# A runner installs a scoped logfile opener while it is active. Keeping this +# hook here preserves the long-standing ptf.open_logfile() API used by tests +# without making those tests aware of runner internals. +_logfile_opener = None +_logging_disable_stack = [] + + +def _close_owned_handlers(logger): + """Remove and close handlers created by PTF, leaving caller handlers alone.""" + for handler in list(logger.handlers): + if getattr(handler, "_ptf_owned", False): + logger.removeHandler(handler) + handler.close() + def open_logfile(name): """ @@ -39,6 +53,9 @@ def open_logfile(name): code is used to implement a single logfile in the absence of --log-dir. """ + if _logfile_opener is not None: + return _logfile_opener(name) + _format = "%(asctime)s.%(msecs)03d %(name)-10s: %(levelname)-8s: %(message)s" _datefmt = "%H:%M:%S" @@ -49,15 +66,13 @@ def open_logfile(name): logger = logging.getLogger() - # Remove any existing handlers - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() + _close_owned_handlers(logger) formatter = logging.Formatter(_format, _datefmt) # Add a new handler handler = logging.FileHandler(filename, mode="a") + handler._ptf_owned = True handler.setFormatter(formatter) logger.addHandler(handler) ptfutils.chown_to_invoking_user(filename) @@ -65,6 +80,7 @@ def open_logfile(name): # We log all ERROR and CRITICAL messages to stdout as well as to the # logfile. stream_handler = logging.StreamHandler() + stream_handler._ptf_owned = True stream_handler.setLevel(logging.ERROR) stream_handler.setFormatter(formatter) logger.addHandler(stream_handler) @@ -75,6 +91,7 @@ def disable_logging(): Temporarily disable all logging by setting the global log level to CRITICAL, which is the highest log level in use. """ + _logging_disable_stack.append(logging.root.manager.disable) logging.disable(logging.CRITICAL) @@ -82,4 +99,5 @@ def enable_logging(): """ Turn logging back on after a call to disable_logging(). """ - logging.disable(logging.NOTSET) + if _logging_disable_stack: + logging.disable(_logging_disable_stack.pop()) diff --git a/src/ptf/__main__.py b/src/ptf/__main__.py new file mode 100644 index 0000000..384cab0 --- /dev/null +++ b/src/ptf/__main__.py @@ -0,0 +1,19 @@ +# Copyright 2010 The Board of Trustees of The Leland Stanford Junior University +# SPDX-License-Identifier: Apache-2.0 + +# This file was derived from code in the Floodlight OFTest repository +# https://github.com/floodlight/oftest released under the OpenFlow +# Software License: +# https://github.com/floodlight/oftest/blob/master/LICENSE +# See file README-oftest.md in the ptf repository for more details. +""" +Entry point for ``python -m ptf``. This module behaves like the ``ptf`` +binary. +""" + +import sys + +from ptf.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/ptf/base_tests.py b/src/ptf/base_tests.py index 080def8..83a7f05 100644 --- a/src/ptf/base_tests.py +++ b/src/ptf/base_tests.py @@ -21,6 +21,8 @@ import ptf from ptf import config +logger = logging.getLogger(__name__) + class BaseTest(unittest.TestCase): def __str__(self): @@ -28,13 +30,13 @@ def __str__(self): def setUp(self): ptf.open_logfile(str(self)) - logging.info("** START TEST CASE " + str(self)) + logger.info("** START TEST CASE " + str(self)) def run(self, result=None): unittest.TestCase.run(self, result) def tearDown(self): - logging.info("** END TEST CASE " + str(self)) + logger.info("** END TEST CASE " + str(self)) def before_send(self, pkt, device_number=0, port_number=-1): """ diff --git a/src/ptf/cli.py b/src/ptf/cli.py new file mode 100644 index 0000000..e386472 --- /dev/null +++ b/src/ptf/cli.py @@ -0,0 +1,440 @@ +# Copyright 2010 The Board of Trustees of The Leland Stanford Junior University +# SPDX-License-Identifier: Apache-2.0 + +# This file was derived from code in the Floodlight OFTest repository +# https://github.com/floodlight/oftest released under the OpenFlow +# Software License: +# https://github.com/floodlight/oftest/blob/master/LICENSE +# See file README-oftest.md in the ptf repository for more details. + +""" +PTF command line interface. + +This module contains the command line parser of the ``ptf`` binary. It +converts the parsed arguments to a :class:`ptf.runner.PtfConfig` object. +The test-run logic is in :mod:`ptf.runner`. The ``ptf`` binary and +``python -m ptf`` call :func:`main` in this module. +""" + +import argparse +import logging +import os +import sys + +import ptf +from ptf import __version__, runner + + +def build_parser(): + # type: () -> argparse.ArgumentParser + """Build the command line parser of the ptf binary. + + To add a new command line option, add an argument in this function. + Then add the matching field to ptf.runner.PtfConfig, or to one of its + option groups. The value of the option ends up in the global + ptf.config dictionary.""" + + usage = "usage: ptf [options] --test-dir TEST_DIR [tests]" + + description = """PTF (Packet Test Framework) is a framework and set of tests +to test a software switch. It is strongly inspired by the OFTest framework, but +it is not tied to OpenFlow. It does not provide any control plane features, but +it is targetted at helping you test a dataplane. + +The default configuration assumes that interfaces veth1, veth3, veth5, and veth7 +should be connected to the switch's dataplane. + +If no positional arguments are given then OFTest will run all tests found in the +--test-dir directory. Otherwise each positional argument is interpreted as +either a test name or a test group name. The union of these will be executed. To +see what groups each test belongs to use the --list option. Tests and groups can +be subtracted from the result by prefixing them with the '^' character. """ + + class ActionInterface(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + # Parse --interface + def check_interface(value): + port_cfg = {} + sp = ";" + try: + if sp in value: + value, p_info = value.split(sp, 1) + params = p_info.split(sp) + for elem in params: + key, val = elem.split("=") + port_cfg[key.lower()] = val + dev_and_port, interface = value.split("@", 1) + dev_and_port = dev_and_port.split("-") + if len(dev_and_port) == 1: + dev, port = 0, int(dev_and_port[0]) + elif len(dev_and_port) == 2: + dev, port = int(dev_and_port[0]), int(dev_and_port[1]) + else: + raise ValueError("") + if port_cfg: + getattr(namespace, "port_info")[port] = port_cfg + except ValueError: + parser.error( + "incorrect interface syntax (got %s, expected 'port@interface' or 'device-port@interface' \ + or providing port configuration using 'device-port@interface;arg=val;arg2=val...' )" + % repr(value) + ) + return (dev, port, interface) + + assert type(values) is str + getattr(namespace, self.dest).append(check_interface(values)) + + class ActionDeviceSocket(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + # Parse --device-socket + def check_device_socket(value): + def parse_ports(ports): + port_set = set() + try: + ports = ports.strip("{}") + ports = ports.split(",") + except: + raise ValueError("") + for port in ports: + try: + p = int(port) + port_set.add(p) + continue + except: + pass + try: + p1, p2 = port.split("-", 1) + p1, p2 = int(p1), int(p2) + for p in range(p1, p2 + 1): # p2 included + port_set.add(p) + except: + raise ValueError("") + return port_set + + try: + dev_and_port, addr = value.split("@", 1) + if dev_and_port[0] == "{": + dev, ports = (0, parse_ports(dev_and_port)) + else: + dev_and_port = dev_and_port.split("-", 1) + if len(dev_and_port) != 2: + raise ValueError("") + dev, ports = ( + int(dev_and_port[0]), + parse_ports(dev_and_port[1]), + ) + except ValueError: + parser.error( + "incorrect device-socket syntax (got %s, expected something of the form 0-{1,2,5-8}@)" + % repr(value) + ) + return (dev, ports, addr) + + assert type(values) is str + getattr(namespace, self.dest).append(check_device_socket(values)) + + class ActionTestDir(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + assert type(values) is str + if not os.path.isdir(values): + parser.error( + "invalid value for --test-dir: directory %s does not exist" % values + ) + setattr(namespace, self.dest, values) + + parser = argparse.ArgumentParser(usage=usage, description=description) + + # The default values come from PtfConfig. PtfConfig is the single + # source of truth for the defaults; this file duplicated the defaults + # before. to_dict() always creates new containers. Parsers therefore + # do not share state. + defaults = runner.PtfConfig().to_dict() + defaults.pop("test_spec") # legacy key, not a command line option + defaults.pop("port_map") # the platform fills this key at run time + parser.set_defaults(**defaults) + + parser.add_argument("--version", action="version", version=__version__) + + parser.add_argument("test_specs", nargs="*", help="Tests / Groups to run") + + parser.add_argument("--list", action="store_true", help="List all tests and exit") + parser.add_argument( + "--list-test-names", + action="store_true", + help="List test names matching the test spec and exit", + ) + parser.add_argument( + "--allow-user", + action="store_true", + help="Proceed even if ptf is not run as root", + ) + + parser.add_argument("--pypath", dest="pypath", action="append") + + parser.add_argument( + "-pmm", + "--packet-manipulation-module", + type=str, + help="Provide packet manipulation module which should be used " + "as a 'packet' one for other PTF modules", + ) + + group = parser.add_argument_group("Test selection options") + group.add_argument("-f", "--test-file", help="File of tests to run, one per line") + group.add_argument( + "--test-dir", + type=str, + action=ActionTestDir, + required=True, + help="Directory containing tests", + ) + test_order_help = """Choose the order in which the tests will be run: + default (tests are run in the order in which they appear on command line), + lexico (use default string ordering on test names), + rand (random order, use --test-order-seed to specify a seed) + """ + group.add_argument( + "--test-order", choices=list(runner.TEST_ORDERS), help=test_order_help + ) + group.add_argument( + "--test-order-seed", type=int, help="Specify seed to randomize test order" + ) + group.add_argument( + "--num-shards", + type=int, + help="Number of shards that can be used to parallelize test execution", + ) + group.add_argument( + "--shard-id", type=int, help="Index of shard (>= 0 and < number of shards)" + ) + + group = parser.add_argument_group("Switch connection options") + group.add_argument("-P", "--platform", help="Platform module name") + group.add_argument( + "-a", "--platform-args", help="Custom arguments for the platform" + ) + group.add_argument( + "--platform-dir", type=str, help="Directory containing platform modules" + ) + group.add_argument( + "--interface", + "-i", + type=str, + dest="interfaces", + metavar="INTERFACE", + action=ActionInterface, + help="Specify a port number and the dataplane interface to use. May be given multiple times. Example: 1@eth1 or 0-1@eth2 (use eth2 as port 1 of device 0)", + ) + group.add_argument( + "--device-socket", + type=str, + dest="device_sockets", + metavar="DEVICE-SOCKET", + action=ActionDeviceSocket, + help="Specify the nanomsg socket to use to send / receive packets for a given device, as well as the ports to enable on the device. May be given multiple times. Example: 0-{1,2,5-8}@", + ) + + group = parser.add_argument_group("Logging options") + group.add_argument("--log-file", help="Name of log file") + group.add_argument("--log-dir", help="Name of log directory") + dbg_lvl_names = sorted( + list(runner.DEBUG_LEVELS.keys()), key=lambda x: runner.DEBUG_LEVELS[x] + ) + group.add_argument( + "--debug", + choices=dbg_lvl_names, + help="Debug lvl: debug, info, warning, error, critical", + ) + group.add_argument( + "--verbose", + action="store_const", + dest="debug", + const="verbose", + help="Shortcut for --debug=verbose", + ) + group.add_argument( + "-q", + "--quiet", + action="store_const", + dest="debug", + const="warning", + help="Shortcut for --debug=warning", + ) + group.add_argument("--profile", action="store_true", help="Enable Python profiling") + group.add_argument("--profile-file", help="Output file for Python profiler") + group.add_argument( + "--xunit", action="store_true", help="Enable xUnit-formatted results" + ) + group.add_argument( + "--xunit-dir", help="Output directory for xUnit-formatted results" + ) + + group = parser.add_argument_group("Test behavior options") + group.add_argument( + "--relax", + action="store_true", + help="Relax packet match checks allowing other packets", + ) + group.add_argument( + "--failfast", + action="store_true", + help="Stop running tests as soon as one fails", + ) + test_params_help = """Set test parameters: [key=val]*;key=val + """ + group.add_argument("-t", "--test-params", help=test_params_help) + group.add_argument( + "--fail-skipped", + action="store_true", + help="Return failure if any test was skipped", + ) + group.add_argument( + "--default-timeout", type=float, help="Timeout in seconds for most operations" + ) + group.add_argument( + "--default-negative-timeout", + type=float, + help="Timeout in seconds for negative checks", + ) + group.add_argument( + "--minsize", type=int, help="Minimum allowable packet size on the dataplane." + ) + group.add_argument("--random-seed", type=int, help="Random number generator seed") + group.add_argument("--disable-ipv6", action="store_true", help="Disable IPv6 tests") + group.add_argument("--qlen", type=int, help="Default queue length ") + group.add_argument( + "--test-case-timeout", + type=int, + help="Timeout for each test case, 0 means no timeout", + ) + + group.add_argument( + "--disable-vxlan", + action="store_true", + help="Disable VXLAN (do not import from scapy even if supported)", + ) + group.add_argument( + "--disable-geneve", + action="store_true", + help="Disable GENEVE (do not import from scapy even if supported)", + ) + group.add_argument( + "--disable-erspan", + action="store_true", + help="Disable ERSPAN (do not import from scapy even if supported)", + ) + group.add_argument( + "--disable-mpls", + action="store_true", + help="Disable MPLS (do not import from scapy even if supported)", + ) + group.add_argument( + "--disable-nvgre", + action="store_true", + help="Disable NVGRE (do not import from scapy even if supported)", + ) + group.add_argument( + "--disable-igmp", + action="store_true", + help="Disable IGMP (do not import from scapy even if supported)", + ) + + group = parser.add_argument_group("Socket options") + group.add_argument( + "--socket-recv-size", + type=int, + help="When using raw sockets, specify the size of the buffer used to receive packets with socket.recv.", + ) + + return parser + + +def config_from_args(args): + # type: (argparse.Namespace) -> runner.PtfConfig + """Convert parsed command line arguments to a PtfConfig.""" + return runner.PtfConfig( + list_tests=args.list, + list_test_names=args.list_test_names, + allow_user=args.allow_user, + packet_manipulation_module=args.packet_manipulation_module, + pypath=list(args.pypath or []), + test_selection=runner.TestSelectionOptions( + test_dir=args.test_dir, + test_specs=list(args.test_specs), + test_file=args.test_file, + test_order=args.test_order, + test_order_seed=args.test_order_seed, + num_shards=args.num_shards, + shard_id=args.shard_id, + ), + platform=runner.PlatformOptions( + platform=args.platform, + platform_args=args.platform_args, + platform_dir=args.platform_dir, + interfaces=[runner.Interface(*i) for i in args.interfaces], + device_sockets=[ + runner.DeviceSocket(device=s[0], ports=set(s[1]), address=s[2]) + for s in args.device_sockets + ], + port_info={int(port): dict(info) for port, info in args.port_info.items()}, + ), + logging=runner.LoggingOptions( + log_file=args.log_file, + log_dir=args.log_dir, + debug=args.debug, + profile=args.profile, + profile_file=args.profile_file, + xunit=args.xunit, + xunit_dir=args.xunit_dir, + ), + test_behavior=runner.TestBehaviorOptions( + relax=args.relax, + failfast=args.failfast, + fail_skipped=args.fail_skipped, + test_params=args.test_params, + default_timeout=args.default_timeout, + default_negative_timeout=args.default_negative_timeout, + minsize=args.minsize, + random_seed=args.random_seed, + test_case_timeout=args.test_case_timeout, + qlen=args.qlen, + disable_ipv6=args.disable_ipv6, + disable_vxlan=args.disable_vxlan, + disable_erspan=args.disable_erspan, + disable_geneve=args.disable_geneve, + disable_mpls=args.disable_mpls, + disable_nvgre=args.disable_nvgre, + disable_igmp=args.disable_igmp, + disable_rocev2=args.disable_rocev2, + ), + socket=runner.SocketOptions(socket_recv_size=args.socket_recv_size), + ) + + +def main(argv=None): + # type: (list) -> int + """Parse the command line, build a PtfConfig, and run the tests. + + When the run fails, this function does not return: it calls + os._exit(rc), as the ptf script did before. A normal exit can hang + when non-daemon threads are active. Call this function only from a + process entry point: the ptf binary or python -m ptf.""" + parser = build_parser() + args = parser.parse_args(argv) + config = config_from_args(args) + try: + rc = runner.run( + config, + output=runner.RunOutput(capture_root_logging=True), + _manage_signals=True, + ) + except runner.PtfError as e: + if not getattr(e, "_ptf_logged", False): + logging.critical(str(e)) + rc = 1 + if rc != 0: + # A normal exit can hang when non-daemon threads are active. + sys.stdout.flush() + sys.stderr.flush() + os._exit(rc) + return rc diff --git a/src/ptf/dataplane.py b/src/ptf/dataplane.py index 71741e0..44674f1 100644 --- a/src/ptf/dataplane.py +++ b/src/ptf/dataplane.py @@ -23,8 +23,8 @@ for filters should include a callback or a counter """ -import sys import os +import sys import socket import time import select @@ -41,7 +41,6 @@ from . import mask from . import packet from .pcap_writer import PcapWriter -from io import StringIO try: import pynng @@ -142,23 +141,32 @@ class DataPlanePortLinux(DataPlanePortIface, DataPlanePacketSourceIface): ETH_P_ALL = 0x03 RCV_TIMEOUT = 10000 - def __init__(self, interface_name, device_number, port_number, config={}): + def __init__(self, interface_name, device_number, port_number, config=None): """ @param interface_name The name of the physical interface like eth1 """ self.interface_name = interface_name self.device_number = device_number self.port_number = port_number - self.socket = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, 0) - afpacket.enable_auxdata(self.socket) - self.socket.bind((interface_name, self.ETH_P_ALL)) - netutils.set_promisc(self.socket, interface_name) - self.socket.settimeout(self.RCV_TIMEOUT) - self.recv_size = config.get("socket_recv_size", self.RCV_SIZE_DEFAULT) + self.socket = None + try: + self.socket = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, 0) + afpacket.enable_auxdata(self.socket) + self.socket.bind((interface_name, self.ETH_P_ALL)) + netutils.set_promisc(self.socket, interface_name) + self.socket.settimeout(self.RCV_TIMEOUT) + except Exception: + self.close() + raise + self.recv_size = (config or {}).get("socket_recv_size", self.RCV_SIZE_DEFAULT) def __del__(self): - if self.socket: + self.close() + + def close(self): + if self.socket is not None: self.socket.close() + self.socket = None def fileno(self): """ @@ -260,6 +268,9 @@ def fileno(self): return self.socket.recv_fd def __send_port_msg(self, msg_type, port_number, more): + if self.socket is None: + # The socket is closed; do nothing. + return hdr = struct.pack(" or tcp://:) """ self.interface_name = interface_name self.device_number = device_number - if (device_number, interface_name) not in self.packet_injecters: - self.packet_injecters[(self.device_number, self.interface_name)] = ( - DataPlanePacketSourceNN( - device_number, interface_name, self.RCV_TIMEOUT, self.SND_TIMEOUT - ) - ) - self.packet_inject = self.packet_injecters[ - (self.device_number, self.interface_name) - ] self.port_number = port_number - self.packet_inject.port_add(port_number) + self.packet_inject = None + key = (device_number, interface_name) + with self.packet_injecters_lock: + if key not in self.packet_injecters: + self.packet_injecters[key] = DataPlanePacketSourceNN( + device_number, + interface_name, + self.RCV_TIMEOUT, + self.SND_TIMEOUT, + ) + self.packet_inject = self.packet_injecters[key] + self.packet_inject.port_add(port_number) def __del__(self): - if self.packet_inject: - self.packet_inject.port_remove(self.port_number) + self.close() + + def close(self): + """ + Close the port. + + The packet source and its nanomsg socket are shared by all ports of + the same device. The final port closes and unregisters that source. + """ + if self.packet_inject is None: + return + key = (self.device_number, self.interface_name) + error = None + with self.packet_injecters_lock: + injecter = self.packet_inject + try: + injecter.port_remove(self.port_number) + except Exception as exception: + error = exception + injecter.ports.discard(self.port_number) + if not injecter.ports: + try: + injecter.close() + except Exception as exception: + if error is None: + error = exception + finally: + if self.packet_injecters.get(key) is injecter: + del self.packet_injecters[key] + self.packet_inject = None + if error is not None: + raise error def get_packet_source(self): """ @retval An object implementing DataPlanePacketSourceIface """ - return self.packet_injecters[(self.device_number, self.interface_name)] + return self.packet_inject def send(self, packet): """ @@ -405,41 +450,31 @@ def send(self, packet): @param packet The packet data to send to the port @retval The number of bytes sent """ - return self.packet_injecters[(self.device_number, self.interface_name)].send( - self.port_number, packet - ) + return self.packet_inject.send(self.port_number, packet) def down(self): """ Bring the physical link down. """ - self.packet_injecters[ - (self.device_number, self.interface_name) - ].port_bring_down(self.port_number) + self.packet_inject.port_bring_down(self.port_number) def up(self): """ Bring the physical link up. """ - self.packet_injecters[(self.device_number, self.interface_name)].port_bring_up( - self.port_number - ) + self.packet_inject.port_bring_up(self.port_number) def mac(self): """ Return mac address """ - return self.packet_injecters[(self.device_number, self.interface_name)].get_mac( - self.port_number - ) + return self.packet_inject.get_mac(self.port_number) def nn_counters(self): """ Return counters """ - return self.packet_injecters[ - (self.device_number, self.interface_name) - ].get_nn_counters(self.port_number) + return self.packet_inject.get_nn_counters(self.port_number) class DataPlanePort(DataPlanePortIface, DataPlanePacketSourceIface): @@ -451,24 +486,33 @@ class DataPlanePort(DataPlanePortIface, DataPlanePacketSourceIface): ETH_P_ALL = 0x03 RCV_TIMEOUT = 10000 - def __init__(self, interface_name, device_number, port_number, config={}): + def __init__(self, interface_name, device_number, port_number, config=None): """ @param interface_name The name of the physical interface like eth1 """ self.interface_name = interface_name self.device_number = device_number self.port_number = port_number - self.socket = socket.socket( - socket.AF_PACKET, socket.SOCK_RAW, socket.htons(self.ETH_P_ALL) - ) - self.socket.bind((interface_name, 0)) - netutils.set_promisc(self.socket, interface_name) - self.socket.settimeout(self.RCV_TIMEOUT) - self.recv_size = config.get("socket_recv_size", self.RCV_SIZE_DEFAULT) + self.socket = None + try: + self.socket = socket.socket( + socket.AF_PACKET, socket.SOCK_RAW, socket.htons(self.ETH_P_ALL) + ) + self.socket.bind((interface_name, 0)) + netutils.set_promisc(self.socket, interface_name) + self.socket.settimeout(self.RCV_TIMEOUT) + except Exception: + self.close() + raise + self.recv_size = (config or {}).get("socket_recv_size", self.RCV_SIZE_DEFAULT) def __del__(self): - if self.socket: + self.close() + + def close(self): + if self.socket is not None: self.socket.close() + self.socket = None def fileno(self): """ @@ -525,7 +569,7 @@ class DataPlanePortPcap: socket. libpcap understands how to read the VLAN tag from the kernel. """ - def __init__(self, interface_name, device_number, port_number, config={}): + def __init__(self, interface_name, device_number, port_number, config=None): self.device_number = device_number self.port_number = port_number self.pcap = pcap.pcap(interface_name) @@ -544,6 +588,14 @@ def get_packet_source(self): def send(self, packet): return self.pcap.inject(packet, len(packet)) + def close(self): + close = getattr(self.pcap, "close", None) + try: + if callable(close): + close() + finally: + self.pcap = None + def down(self): pass @@ -568,6 +620,7 @@ class DataPlane(Thread): def __init__(self, config=None): Thread.__init__(self) + self.daemon = True # dict from device number, port number to port object self.ports = {} @@ -589,7 +642,7 @@ def __init__(self, config=None): self.waker = ptfutils.EventDescriptor() self.killed = False - self.logger = logging.getLogger("dataplane") + self.logger = logging.getLogger(__name__) self.pcap_writer = None if config is None: @@ -634,15 +687,23 @@ def run(self): """ Activity function for class """ + select_errors = 0 while not self.killed: sockets = set([p.get_packet_source() for p in list(self.ports.values())]) sockets.add(self.waker) try: sel_in, sel_out, sel_err = select.select(sockets, [], [], 1) - except: - print(sys.exc_info()) - self.logger.error("Select error, exiting") - break + except Exception: + if self.killed: + break + self.logger.exception("Select error, retrying") + select_errors += 1 + if select_errors >= 3: + self.logger.error("Too many consecutive select errors, exiting") + break + time.sleep(0.01) + continue + select_errors = 0 with self.cvar: for sel in sel_in: @@ -673,7 +734,9 @@ def run(self): self.pcap_writer.write( pkt, timestamp, device_number, port_number ) - queue = self.packet_queues[(device_number, port_number)] + queue = self.packet_queues.get((device_number, port_number)) + if queue is None: + continue if len(queue) >= self.qlen: # Queue full, throw away oldest queue.pop(0) @@ -697,11 +760,19 @@ def port_add(self, interface_name, device_number, port_number): """ port_id = (device_number, port_number) with self.cvar: - self.ports[port_id] = self.dppclass( - interface_name, device_number, port_number, self.config - ) - self.ports[port_id]._port_number = port_number - self.ports[port_id]._device_number = device_number + old_port = self.ports.pop(port_id, None) + self.packet_queues.pop(port_id, None) + if old_port is not None: + close = getattr(old_port, "close", None) + if callable(close): + close() + new_port = self.dppclass( + interface_name, device_number, port_number, self.config + ) + with self.cvar: + self.ports[port_id] = new_port + new_port._port_number = port_number + new_port._device_number = device_number self.packet_queues[port_id] = [] # Need to wake up event loop to change the sockets being selected # on. @@ -718,8 +789,11 @@ def port_remove(self, device_number, port_number): ) ) return False - del self.ports[port_id] + port = self.ports.pop(port_id) del self.packet_queues[port_id] + close = getattr(port, "close", None) + if callable(close): + close() self.waker.notify() return True @@ -823,25 +897,19 @@ def format(self): this packet. If the expected packet is a scapy packet, it's used to include detailed information about the fields in the packet. """ - try: - stdout_save = sys.stdout - # The scapy packet dissection methods print directly to stdout, - # so we have to redirect stdout to a string. - sys.stdout = StringIO() - - print("========== RECEIVED ==========") - if isinstance(self.expected_packet, packet.Packet): - # Dissect this packet as if it were an instance of - # the expected packet's class. - packet.ls(self.expected_packet.__class__(self.packet)) - print("--") - packet.hexdump(self.packet) - print("==============================") - - return sys.stdout.getvalue() - finally: - sys.stdout.close() - sys.stdout = stdout_save # Restore the original stdout. + lines = ["========== RECEIVED =========="] + if isinstance(self.expected_packet, packet.Packet): + # Dissect this packet as if it were an instance of the + # expected packet's class. + lines.append( + packet.format_packet( + self.expected_packet.__class__(self.packet) + ).rstrip() + ) + lines.append("--") + lines.append(packet.format_hexdump(self.packet).rstrip()) + lines.append("==============================") + return "\n".join(lines) + "\n" class PollFailure(PollResult): """ @@ -870,49 +938,46 @@ def format(self): in the output. If the expected packet is a scapy packet object, the output will include information about the fields in the packet. """ - try: - stdout_save = sys.stdout - # The scapy packet dissection methods print directly to stdout, - # so we have to redirect stdout to a string. - sys.stdout = StringIO() - - if self.expected_packet is not None: - print("========== EXPECTED ==========") - if isinstance(self.expected_packet, packet.Packet): - packet.ls(self.expected_packet) - print("--") - packet.hexdump(self.expected_packet) - elif isinstance(self.expected_packet, mask.Mask): - print("Mask:") - print(self.expected_packet) - else: - packet.hexdump(self.expected_packet) - - print("========== RECEIVED ==========") - if self.recent_packets: - print( - "%d total packets. Displaying most recent %d packets:" - % (self.packet_count, len(self.recent_packets)) - ) - for recent_packet in self.recent_packets: - print("------------------------------") - if isinstance(self.expected_packet, packet.Packet): - # Dissect this packet as if it were an instance of - # the expected packet's class. - packet.ls(self.expected_packet.__class__(recent_packet)) - print("--") - packet.hexdump(recent_packet) + lines = [] + if self.expected_packet is not None: + lines.append("========== EXPECTED ==========") + if isinstance(self.expected_packet, packet.Packet): + lines.append(packet.format_packet(self.expected_packet).rstrip()) + lines.append("--") + lines.append(packet.format_hexdump(self.expected_packet).rstrip()) + elif isinstance(self.expected_packet, mask.Mask): + lines.extend(("Mask:", str(self.expected_packet).rstrip())) else: - print("%d total packets." % self.packet_count) - print("==============================") + lines.append(packet.format_hexdump(self.expected_packet).rstrip()) - return sys.stdout.getvalue() - finally: - sys.stdout.close() - sys.stdout = stdout_save # Restore the original stdout. + lines.append("========== RECEIVED ==========") + if self.recent_packets: + lines.append( + "%d total packets. Displaying most recent %d packets:" + % (self.packet_count, len(self.recent_packets)) + ) + for recent_packet in self.recent_packets: + lines.append("------------------------------") + if isinstance(self.expected_packet, packet.Packet): + lines.append( + packet.format_packet( + self.expected_packet.__class__(recent_packet) + ).rstrip() + ) + lines.append("--") + lines.append(packet.format_hexdump(recent_packet).rstrip()) + else: + lines.append("%d total packets." % self.packet_count) + lines.append("==============================") + return "\n".join(lines) + "\n" def poll( - self, device_number=0, port_number=None, timeout=None, exp_pkt=None, filters=[] + self, + device_number=0, + port_number=None, + timeout=None, + exp_pkt=None, + filters=None, ): """ Poll one or all dataplane ports for a packet @@ -937,7 +1002,7 @@ def poll( """ def filter_check(pkt): - for f in filters: + for f in filters or (): if not f(pkt): return False return True @@ -997,13 +1062,52 @@ def kill(self): """ Stop the dataplane thread. """ - self.killed = True - self.waker.notify() - self.join() - # Explicitly release ports to ensure we don't run out of sockets - # even if someone keeps holding a reference to the dataplane. - del self.ports - self.waker.close() + errors = [] + try: + self.stop_pcap() + except Exception as error: + errors.append(error) + self.logger.exception("Failed to stop dataplane packet capture") + if not self.killed: + self.killed = True + self.waker.notify() + if self.is_alive(): + self.join(timeout=5) + if self.is_alive(): + self.logger.warning( + "Dataplane thread did not stop promptly; closing its ports" + ) + # Close each port explicitly. A caller can keep a reference to the + # dataplane after this call. Explicit closing makes sure that the + # sockets do not stay open. This matters when ptf runs inside + # another process: an open nanomsg socket must not live longer + # than the run. + for port in list(self.ports.values()): + close = getattr(port, "close", None) + if callable(close): + try: + close() + except Exception as error: + errors.append(error) + self.logger.exception("Failed to close a dataplane port") + if self.is_alive(): + self.waker.notify() + self.join(timeout=1) + if self.is_alive(): + self.logger.error("Dataplane thread did not stop after port cleanup") + errors.append(RuntimeError("dataplane thread did not stop")) + self.ports.clear() + self.packet_queues.clear() + try: + self.waker.close() + except Exception as error: + errors.append(error) + self.logger.exception("Failed to close the dataplane wake descriptor") + if errors: + raise RuntimeError( + "dataplane cleanup failed: %s" + % "; ".join(str(error) for error in errors) + ) def port_down(self, device_number, port_number): """Brings the specified port down""" @@ -1044,6 +1148,9 @@ def start_pcap(self, filename): def stop_pcap(self): if self.pcap_writer: with self.cvar: - self.pcap_writer.close() + writer = self.pcap_writer self.pcap_writer = None - self.cvar.notify_all() + try: + writer.close() + finally: + self.cvar.notify_all() diff --git a/src/ptf/mask.py b/src/ptf/mask.py index d928379..87797e4 100644 --- a/src/ptf/mask.py +++ b/src/ptf/mask.py @@ -3,8 +3,6 @@ import warnings -from io import StringIO -import sys from . import packet @@ -123,13 +121,9 @@ def _calculate_fields_offset_and_bitwidth(self, hdr_type, field_name): return hdr_offset * 8 + offset, bitwidth def __str__(self): - old_stdout = sys.stdout - sys.stdout = buffer = StringIO() - print("\npacket status: %s" % "OK" if self.valid else "INVALID") - print("packet:") - packet.hexdump(self.exp_pkt) # noqa - print("\npacket's mask:") - packet.hexdump(self.mask) # noqa - - sys.stdout = old_stdout - return buffer.getvalue() + status = "OK" if self.valid else "INVALID" + return "\npacket status: %s\npacket:\n%s\n\npacket's mask:\n%s\n" % ( + status, + packet.format_hexdump(self.exp_pkt).rstrip(), + packet.format_hexdump(self.mask).rstrip(), + ) diff --git a/src/ptf/netutils.py b/src/ptf/netutils.py index a161c66..e93f9e2 100644 --- a/src/ptf/netutils.py +++ b/src/ptf/netutils.py @@ -27,10 +27,17 @@ import ctypes import fcntl import socket +import struct # Constant from Linux /usr/include/linux/if.h or net/if.h IFF_PROMISC = 0x100 +# Constants from Linux /usr/include/linux/if_packet.h. +SOL_PACKET = 263 +PACKET_ADD_MEMBERSHIP = 1 +PACKET_DROP_MEMBERSHIP = 2 +PACKET_MR_PROMISC = 1 + # Constants from Linux bits/ioctls.h or linux/sockios.h SIOCGIFHWADDR = 0x8927 # Get hardware address SIOCGIFFLAGS = 0x8913 @@ -42,10 +49,8 @@ class ifreq(ctypes.Structure): def get_if(iff: str, cmd: int) -> bytes: - s = socket.socket() - ifreq = fcntl.ioctl(s, cmd, struct.pack("16s16x", iff.encode("utf-8"))) - s.close() - return ifreq + with socket.socket() as sock: + return fcntl.ioctl(sock, cmd, struct.pack("16s16x", iff.encode("utf-8"))) # Given iff, the name of a network interface (e.g. 'veth0') as a @@ -62,6 +67,23 @@ def get_mac(iff: str) -> str: # interface in promiscuous mode if parameter val != 0, or into # non-promiscuous mode if val == 0. def set_promisc(s, iff, val=1): + """Change promiscuous membership for one packet socket. + + Linux drops this membership automatically when the socket closes, unlike + changing the interface-wide IFF_PROMISC flag. + """ + if hasattr(socket, "if_nametoindex"): + membership = struct.pack( + "IHH8s", socket.if_nametoindex(iff), PACKET_MR_PROMISC, 0, b"" + ) + option = PACKET_ADD_MEMBERSHIP if val else PACKET_DROP_MEMBERSHIP + try: + s.setsockopt(SOL_PACKET, option, membership) + return + except OSError: + # Retain the ioctl fallback for platforms without packet-socket + # membership support. + pass ifr = ifreq() ifr.ifr_ifrn = bytes(iff, "utf-8") # Get current interface flags diff --git a/src/ptf/packet.py b/src/ptf/packet.py index a37bb71..425bd5b 100644 --- a/src/ptf/packet.py +++ b/src/ptf/packet.py @@ -15,7 +15,9 @@ then, create an implementation of packet module for it (for Scapy it is packet_scapy.py) """ +import logging as _logging import os as _os + from ptf import config # When module ptf.packet is imported, this is the order of precedence for @@ -43,15 +45,46 @@ else: _packet_manipulation_module = "ptf.packet_scapy" +_packet_config = { + name: config.get(name, False) + for name in ( + "disable_ipv6", + "disable_vxlan", + "disable_erspan", + "disable_geneve", + "disable_mpls", + "disable_nvgre", + "disable_igmp", + "disable_rocev2", + ) +} + __module = __import__(_packet_manipulation_module, fromlist=["*"]) __keys = [] # import logic - everything from __all__ if provided, otherwise # everything not starting with underscore. -print("Using packet manipulation module: %s" % __module.__name__) +_logging.getLogger(__name__).info( + "Using packet manipulation module: %s", __module.__name__ +) if "__all__" in __module.__dict__: __keys = __module.__dict__["__all__"] else: __keys = [k for k in __module.__dict__ if not k.startswith("_")] locals().update({k: getattr(__module, k) for k in __keys}) + + +if "format_hexdump" not in locals(): + + def format_hexdump(value): + try: + return bytes(value).hex(" ") + except (TypeError, ValueError): + return repr(value) + + +if "format_packet" not in locals(): + + def format_packet(value): + return repr(value) diff --git a/src/ptf/packet_scapy.py b/src/ptf/packet_scapy.py index dd4c32b..e144ad2 100644 --- a/src/ptf/packet_scapy.py +++ b/src/ptf/packet_scapy.py @@ -13,10 +13,26 @@ Scapy implementation of packet manipulation module """ +import logging + import ptf from ptf import config -import sys -import logging + +logger = logging.getLogger(__name__) + +_ptf_packet_config = { + name: config.get(name, False) + for name in ( + "disable_ipv6", + "disable_vxlan", + "disable_erspan", + "disable_geneve", + "disable_mpls", + "disable_nvgre", + "disable_igmp", + "disable_rocev2", + ) +} try: import scapy.config @@ -34,8 +50,8 @@ if not config.get("disable_ipv6", False): import scapy.route6 import scapy.layers.inet6 -except ImportError: - sys.exit("Need to install scapy for packet parsing") +except ImportError as error: + raise ImportError("Need to install scapy for packet parsing") from error Packet = scapy.packet.Packet Ether = scapy.layers.l2.Ether @@ -65,10 +81,10 @@ scapy.main.load_contrib("roce") BTH = scapy.contrib.roce.BTH ptf.enable_logging() - logging.info("ROCEv2 support found in Scapy") + logger.info("ROCEv2 support found in Scapy") except: ptf.enable_logging() - logging.warn("ROCEv2 support not found in Scapy") + logger.warning("ROCEv2 support not found in Scapy") pass if not config.get("disable_ipv6", False): @@ -89,10 +105,10 @@ ERSPAN_III = scapy.contrib.erspan.ERSPAN_III PlatformSpecific = scapy.contrib.erspan.ERSPAN_PlatformSpecific ptf.enable_logging() - logging.info("ERSPAN support found in Scapy") + logger.info("ERSPAN support found in Scapy") except: ptf.enable_logging() - logging.warn("ERSPAN support not found in Scapy") + logger.warning("ERSPAN support not found in Scapy") pass GENEVE = None @@ -102,10 +118,10 @@ scapy.main.load_contrib("geneve") GENEVE = scapy.contrib.geneve.GENEVE ptf.enable_logging() - logging.info("GENEVE support found in Scapy") + logger.info("GENEVE support found in Scapy") except: ptf.enable_logging() - logging.warn("GENEVE support not found in Scapy") + logger.warning("GENEVE support not found in Scapy") pass MPLS = None @@ -115,10 +131,10 @@ scapy.main.load_contrib("mpls") MPLS = scapy.contrib.mpls.MPLS ptf.enable_logging() - logging.info("MPLS support found in Scapy") + logger.info("MPLS support found in Scapy") except: ptf.enable_logging() - logging.warn("MPLS support not found in Scapy") + logger.warning("MPLS support not found in Scapy") pass NVGRE = None @@ -151,10 +167,10 @@ def mysummary(self): scapy.main.load_contrib("igmp") IGMP = scapy.contrib.igmp.IGMP ptf.enable_logging() - logging.info("IGMP support found in Scapy") + logger.info("IGMP support found in Scapy") except: ptf.enable_logging() - logging.warn("IGMP support not found in Scapy") + logger.warning("IGMP support not found in Scapy") pass @@ -162,6 +178,15 @@ def mysummary(self): hexdump = scapy.utils.hexdump ls = scapy.packet.ls + +def format_hexdump(value): + return scapy.utils.hexdump(value, dump=True) + + +def format_packet(value): + return value.show2(dump=True) + + # The names below are assigned here so that, like the other names # above, they can be used by importers of the ptf.packet module as if # they were defined inside of ptf.packet, and they are commonly diff --git a/src/ptf/pcap_writer.py b/src/ptf/pcap_writer.py index 18e4936..21654bc 100644 --- a/src/ptf/pcap_writer.py +++ b/src/ptf/pcap_writer.py @@ -125,7 +125,11 @@ def flush(self): self.stream.flush() def close(self): - self.stream.close() + if self.stream is not None: + try: + self.stream.close() + finally: + self.stream = None def rdpcap_one_packet(f, path: Union[str, os.PathLike], return_packet_metadata: bool): diff --git a/src/ptf/ptfutils.py b/src/ptf/ptfutils.py index 4f90d77..8f86bfd 100644 --- a/src/ptf/ptfutils.py +++ b/src/ptf/ptfutils.py @@ -18,6 +18,8 @@ import logging import signal +logger = logging.getLogger(__name__) + default_timeout = None # set by ptf default_negative_timeout = None # set by ptf @@ -49,7 +51,7 @@ def chown_to_invoking_user(path, recursive=False): try: uid, gid = int(sudo_uid), int(sudo_gid) except ValueError: - logging.warning( + logger.warning( "Ignoring malformed SUDO_UID/SUDO_GID: %s/%s", sudo_uid, sudo_gid ) return @@ -67,7 +69,7 @@ def chown_to_invoking_user(path, recursive=False): # system chowned to themselves. os.lchown(target, uid, gid) except OSError as e: - logging.warning("Could not change ownership of %s: %s", target, e) + logger.warning("Could not change ownership of %s: %s", target, e) """ @@ -106,19 +108,29 @@ def __init__(self): fcntl.fcntl(self.pipe_wr, fcntl.F_SETFL, os.O_NONBLOCK) def close(self): - os.close(self.pipe_rd) - os.close(self.pipe_wr) + if self.pipe_rd is not None: + os.close(self.pipe_rd) + self.pipe_rd = None + if self.pipe_wr is not None: + os.close(self.pipe_wr) + self.pipe_wr = None def notify(self): + if self.pipe_wr is None: + return try: os.write(self.pipe_wr, "x".encode("utf-8")) except OSError as e: - logging.warn("Failed to notify EventDescriptor: %s", e) + logger.warning("Failed to notify EventDescriptor: %s", e) def wait(self): + if self.pipe_rd is None: + return os.read(self.pipe_rd, 1) def fileno(self): + if self.pipe_rd is None: + return -1 return self.pipe_rd @@ -130,34 +142,42 @@ class TimeoutError(Exception): pass def __init__(self, sec): - try: - from signal import alarm - - self.supported = True - except ImportError: - logging.warn( + self.supported = all( + hasattr(signal, name) + for name in ("SIGALRM", "ITIMER_REAL", "getitimer", "setitimer") + ) + if not self.supported: + logger.warning( "Your platform does not support alarm signals, " "the Timeout feature is therefore not supported" ) - self.supported = False return self.sec = sec if sec > 0: self.valid = True else: self.valid = False - logging.warn("Invalid timeout requested") + logger.warning("Invalid timeout requested") def __enter__(self): if not self.supported or not self.valid: - return + return self + self.previous_handler = signal.getsignal(signal.SIGALRM) + self.previous_timer = signal.getitimer(signal.ITIMER_REAL) + self.started_at = time.monotonic() signal.signal(signal.SIGALRM, self.raise_timeout) - signal.alarm(self.sec) + signal.setitimer(signal.ITIMER_REAL, self.sec) + return self def __exit__(self, *args): if not self.supported or not self.valid: return - signal.alarm(0) # disable alarm + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, self.previous_handler) + delay, interval = self.previous_timer + if delay > 0: + delay = max(delay - (time.monotonic() - self.started_at), 1e-6) + signal.setitimer(signal.ITIMER_REAL, delay, interval) def raise_timeout(self, *args): raise Timeout.TimeoutError() diff --git a/src/ptf/runner.py b/src/ptf/runner.py new file mode 100644 index 0000000..eaddb54 --- /dev/null +++ b/src/ptf/runner.py @@ -0,0 +1,1637 @@ +# Copyright 2010 The Board of Trustees of The Leland Stanford Junior University +# SPDX-License-Identifier: Apache-2.0 + +# This file was derived from code in the Floodlight OFTest repository +# https://github.com/floodlight/oftest released under the OpenFlow +# Software License: +# https://github.com/floodlight/oftest/blob/master/LICENSE +# See file README-oftest.md in the ptf repository for more details. + +""" +PTF runner library. + +This module contains the test-run logic of the Packet Test Framework (PTF). +The logic was part of the top-level ``ptf`` script before. Use this module +to run PTF tests inside any Python program, without a separate ``ptf`` +process: + + from ptf import runner + + config = runner.PtfConfig( + test_selection=runner.TestSelectionOptions(test_dir="tests"), + platform=runner.PlatformOptions( + platform="nn", + device_sockets=[ + runner.DeviceSocket(0, {0, 1}, "ipc:///tmp/ptf_packets.ipc") + ], + ), + test_behavior=runner.TestBehaviorOptions( + test_params={"key1": 17, "key2": True} + ), + ) + exit_code = runner.run(config) + +``run()`` performs the same steps as the ``ptf`` binary: logging setup, test +discovery and selection, sharding, platform loading, dataplane setup, test +execution, and teardown. It returns the exit code that the binary produces +for the same configuration. ``run()`` also accepts a dictionary in the flat +``ptf.config`` format; see :meth:`PtfConfig.from_dict`. + +The tests read their settings from global state: the ``ptf.config`` +dictionary, ``ptf.dataplane_instance``, and the module globals of +``ptf.testutils`` and ``ptf.ptfutils``. ``run()`` fills this state from the +given :class:`PtfConfig`. Set a non-default packet manipulation module in +the configuration before ``ptf.packet`` or ``ptf.testutils`` is imported for +the first time. ``run()`` keeps this order when it imports the test modules. +""" + +import dataclasses +import fnmatch +import importlib +import importlib.machinery +import importlib.util +import json +import logging +import os +import random +import shutil +import signal +import sys +import threading +import time +import types +import unittest +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Set, TextIO, Union + +import ptf +from . import ptfutils + +LOGGER = logging.getLogger(__name__) + +##@var DEBUG_LEVELS +# Map from strings to debugging levels +DEBUG_LEVELS = { + "debug": logging.DEBUG, + "verbose": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "warn": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +TEST_ORDERS = ("default", "lexico", "rand") + +# Default packet manipulation module, used when neither the configuration +# nor the PTF_PACKET_MANIPULATION_MODULE environment variable specify one. +DEFAULT_PACKET_MANIPULATION_MODULE = "ptf.packet_scapy" + + +class PtfError(Exception): + """A fatal error that stops a test run before it starts. + + ``run()`` raises this error when the configuration or the environment + is invalid. The ``ptf`` binary reports the error as a critical log + message and exits with code 1. + """ + + +@dataclass +class Interface: + """The mapping between a (device, port) pair and a dataplane interface. + + This class represents one entry in ``port@interface`` or + ``device-port@interface`` syntax of the ``--interface`` command line + option. + """ + + device: int = 0 + port: int = 0 + interface: str = "" + + +@dataclass +class DeviceSocket: + """The nanomsg socket for packet input and output on a set of ports. + + This class represents one entry in the + ``device-{port,port,...}@socketaddr`` syntax of the ``--device-socket`` + command line option. + """ + + device: int = 0 + ports: Set[int] = field(default_factory=set) + address: str = "" + + +@dataclass +class TestSelectionOptions: + """The tests to run, and their order. + + ``test_specs`` holds test names (``module.Test``), group names, or names + negated with ``^``. An empty list selects the ``standard`` group. This is + the same as the positional arguments of the binary. + """ + + test_dir: Optional[str] = None + test_specs: List[str] = field(default_factory=list) + test_file: Optional[str] = None + test_order: str = "default" + test_order_seed: int = 0xABA + num_shards: int = 1 + shard_id: int = 0 + + +@dataclass +class PlatformOptions: + """The connection to the device under test (the PTF platform). + + ``port_info`` holds optional parameters for each port. The + ``device-port@interface;key=value;...`` syntax of ``--interface`` + carries these parameters on the command line. + """ + + platform: str = "eth" + platform_args: Optional[str] = None + platform_dir: Optional[str] = None + interfaces: List[Interface] = field(default_factory=list) + device_sockets: List[DeviceSocket] = field(default_factory=list) + port_info: Dict[int, Dict[str, str]] = field(default_factory=dict) + + +@dataclass +class LoggingOptions: + """Log file, xUnit and profiling outputs.""" + + log_file: Optional[str] = "ptf.log" + log_dir: Optional[str] = None + debug: str = "verbose" + profile: bool = False + profile_file: str = "profile.out" + xunit: bool = False + xunit_dir: str = "xunit" + + +@dataclass +class TestBehaviorOptions: + """Options that control the behavior of the selected tests. + + ``test_params`` accepts two forms. The first form is a string in the + ``key=value;key=value`` syntax of ``--test-params``. The binary and + ``run()`` evaluate each value as a Python expression. The second form is + a dictionary; the tests receive each entry as given. + """ + + relax: bool = False + failfast: bool = False + fail_skipped: bool = False + test_params: Optional[Union[str, Dict[str, Any]]] = None + default_timeout: float = 2.0 + default_negative_timeout: float = 0.1 + minsize: int = 0 + random_seed: Optional[int] = None + test_case_timeout: Optional[int] = None + qlen: int = 100 + disable_ipv6: bool = False + disable_vxlan: bool = False + disable_erspan: bool = False + disable_geneve: bool = False + disable_mpls: bool = False + disable_nvgre: bool = False + disable_igmp: bool = False + disable_rocev2: bool = False + + +@dataclass +class SocketOptions: + """Options for the sockets used by the dataplane.""" + + socket_recv_size: int = 4096 + + +@dataclass(frozen=True) +class RunOutput: + """Runtime-only output and logging integration for :func:`run`. + + These values are deliberately separate from :class:`PtfConfig` so that + the configuration remains serializable. + """ + + stdout: Optional[TextIO] = None + stderr: Optional[TextIO] = None + logger: Optional[logging.Logger] = None + capture_root_logging: bool = False + + +@dataclass +class PtfConfig: + """The complete configuration of one PTF test run. + + The nested option groups match the option groups of the ``ptf`` + binary. Use :meth:`to_dict` and :meth:`from_dict` to convert between + this class and the flat ``ptf.config`` dictionary format. Use + :meth:`to_json` and :meth:`from_json` to serialize the configuration, + for example for a worker process. + """ + + # Print the list of available tests instead of running the tests. + list_tests: bool = False + # Print the names of the tests that match the test spec, instead of + # running the tests. + list_test_names: bool = False + # Proceed when ptf does not run as root. + allow_user: bool = False + # The packet manipulation module. None selects the + # PTF_PACKET_MANIPULATION_MODULE environment variable, or the default + # module. + packet_manipulation_module: Optional[str] = None + # Additional directories appended to sys.path before the tests and the + # platforms load (same as --pypath). + pypath: List[str] = field(default_factory=list) + test_selection: TestSelectionOptions = field(default_factory=TestSelectionOptions) + platform: PlatformOptions = field(default_factory=PlatformOptions) + logging: LoggingOptions = field(default_factory=LoggingOptions) + test_behavior: TestBehaviorOptions = field(default_factory=TestBehaviorOptions) + socket: SocketOptions = field(default_factory=SocketOptions) + # Preserve platform-specific entries accepted by the legacy flat + # ptf.config dictionary. + extra_config: Dict[str, Any] = field(default_factory=dict) + + ######################################################################## + # (De)serialization helpers + ######################################################################## + + @staticmethod + def _test_params_to_str(test_params): + # Render dictionary test params in the string syntax of the binary. + # The flat dictionary then stays compatible with the value that the + # binary writes. + if test_params is None or isinstance(test_params, str): + return test_params + return ";".join("{}={!r}".format(k, v) for k, v in test_params.items()) + + def to_dict(self): + # type: () -> Dict[str, Any] + """Return the equivalent flat dictionary in the ``ptf.config`` + format. The PTF library and the platforms consume this format. + ``port_map`` stays None; the platform fills this key in when + ``run()`` executes.""" + tb = self.test_behavior + result = dict(self.extra_config) + result.update( + { + # Miscellaneous options + "list": self.list_tests, + "list_test_names": self.list_test_names, + "allow_user": self.allow_user, + "pypath": list(self.pypath), + # Test selection options + "test_spec": "", # legacy key, unused + "test_specs": list(self.test_selection.test_specs), + "test_file": self.test_selection.test_file, + "test_dir": self.test_selection.test_dir, + "test_order": self.test_selection.test_order, + "test_order_seed": self.test_selection.test_order_seed, + "num_shards": self.test_selection.num_shards, + "shard_id": self.test_selection.shard_id, + # Switch connection options + "platform": self.platform.platform, + "platform_args": self.platform.platform_args, + "platform_dir": self.platform.platform_dir, + "interfaces": [ + (i.device, i.port, i.interface) for i in self.platform.interfaces + ], + "port_info": { + port: dict(info) for port, info in self.platform.port_info.items() + }, + "device_sockets": [ + (s.device, set(s.ports), s.address) + for s in self.platform.device_sockets + ], + # Logging options + "log_file": self.logging.log_file, + "log_dir": self.logging.log_dir, + "debug": self.logging.debug, + "profile": self.logging.profile, + "profile_file": self.logging.profile_file, + "xunit": self.logging.xunit, + "xunit_dir": self.logging.xunit_dir, + # Test behavior options + "relax": tb.relax, + "test_params": self._test_params_to_str(tb.test_params), + "failfast": tb.failfast, + "fail_skipped": tb.fail_skipped, + "default_timeout": tb.default_timeout, + "default_negative_timeout": tb.default_negative_timeout, + "minsize": tb.minsize, + "random_seed": tb.random_seed, + "disable_ipv6": tb.disable_ipv6, + "disable_vxlan": tb.disable_vxlan, + "disable_erspan": tb.disable_erspan, + "disable_geneve": tb.disable_geneve, + "disable_mpls": tb.disable_mpls, + "disable_nvgre": tb.disable_nvgre, + "disable_igmp": tb.disable_igmp, + "disable_rocev2": tb.disable_rocev2, + "qlen": tb.qlen, + "test_case_timeout": tb.test_case_timeout, + # Socket options + "socket_recv_size": self.socket.socket_recv_size, + # Other configuration; "port_map" is set by the platform. + "port_map": None, + # Left as None here on purpose: run() applies the + # CLI > environment variable > default precedence. + "packet_manipulation_module": self.packet_manipulation_module, + } + ) + return result + + @classmethod + def from_dict(cls, d): + # type: (Dict[str, Any]) -> PtfConfig + """Build a PtfConfig from a dictionary in the flat ``ptf.config`` + format. The dictionary may be partial; missing keys get their + default values. Each entry of ``interfaces`` and + ``device_sockets`` may be a legacy tuple or a structured dataclass + instance.""" + + def interfaces(value): + return [ + i if isinstance(i, Interface) else Interface(*i) for i in value or [] + ] + + def device_sockets(value): + return [ + ( + s + if isinstance(s, DeviceSocket) + else DeviceSocket(device=s[0], ports=set(s[1]), address=s[2]) + ) + for s in value or [] + ] + + defaults = cls() + known_keys = set(defaults.to_dict()) + return cls( + list_tests=d.get("list", defaults.list_tests), + list_test_names=d.get("list_test_names", defaults.list_test_names), + allow_user=d.get("allow_user", defaults.allow_user), + packet_manipulation_module=d.get( + "packet_manipulation_module", defaults.packet_manipulation_module + ), + pypath=list(d.get("pypath", [])), + test_selection=TestSelectionOptions( + test_dir=d.get("test_dir"), + test_specs=list(d.get("test_specs", [])), + test_file=d.get("test_file"), + test_order=d.get("test_order", "default"), + test_order_seed=d.get("test_order_seed", 0xABA), + num_shards=d.get("num_shards", 1), + shard_id=d.get("shard_id", 0), + ), + platform=PlatformOptions( + platform=d.get("platform", "eth"), + platform_args=d.get("platform_args"), + platform_dir=d.get("platform_dir"), + interfaces=interfaces(d.get("interfaces")), + device_sockets=device_sockets(d.get("device_sockets")), + port_info={ + int(port): dict(info) + for port, info in (d.get("port_info") or {}).items() + }, + ), + logging=LoggingOptions( + log_file=d.get("log_file", "ptf.log"), + log_dir=d.get("log_dir"), + debug=d.get("debug", "verbose"), + profile=d.get("profile", False), + profile_file=d.get("profile_file", "profile.out"), + xunit=d.get("xunit", False), + xunit_dir=d.get("xunit_dir", "xunit"), + ), + test_behavior=TestBehaviorOptions( + relax=d.get("relax", False), + failfast=d.get("failfast", False), + fail_skipped=d.get("fail_skipped", False), + test_params=d.get("test_params"), + default_timeout=d.get("default_timeout", 2.0), + default_negative_timeout=d.get("default_negative_timeout", 0.1), + minsize=d.get("minsize", 0), + random_seed=d.get("random_seed"), + test_case_timeout=d.get("test_case_timeout"), + qlen=d.get("qlen", 100), + disable_ipv6=d.get("disable_ipv6", False), + disable_vxlan=d.get("disable_vxlan", False), + disable_erspan=d.get("disable_erspan", False), + disable_geneve=d.get("disable_geneve", False), + disable_mpls=d.get("disable_mpls", False), + disable_nvgre=d.get("disable_nvgre", False), + disable_igmp=d.get("disable_igmp", False), + disable_rocev2=d.get("disable_rocev2", False), + ), + socket=SocketOptions( + socket_recv_size=d.get("socket_recv_size", 4096), + ), + extra_config={k: v for k, v in d.items() if k not in known_keys}, + ) + + def to_json(self): + # type: () -> str + """Serialize the configuration to JSON text. Sets become sorted + lists. Use this method to pass a configuration to a worker + process; see :meth:`from_json`.""" + + data = dataclasses.asdict(self) + for device_socket in data["platform"]["device_sockets"]: + device_socket["ports"] = sorted(device_socket["ports"]) + return json.dumps(data, indent=2) + + @classmethod + def from_json(cls, text): + # type: (str) -> PtfConfig + """Build a PtfConfig from the JSON text that :meth:`to_json` + writes.""" + data = json.loads(text) + + def group(name): + return data.get(name) or {} + + sel = group("test_selection") + plat = group("platform") + log = group("logging") + beh = group("test_behavior") + sock = group("socket") + return cls( + list_tests=data.get("list_tests", False), + list_test_names=data.get("list_test_names", False), + allow_user=data.get("allow_user", False), + packet_manipulation_module=data.get("packet_manipulation_module"), + pypath=data.get("pypath", []), + test_selection=TestSelectionOptions( + test_dir=sel.get("test_dir"), + test_specs=sel.get("test_specs", []), + test_file=sel.get("test_file"), + test_order=sel.get("test_order", "default"), + test_order_seed=sel.get("test_order_seed", 0xABA), + num_shards=sel.get("num_shards", 1), + shard_id=sel.get("shard_id", 0), + ), + platform=PlatformOptions( + platform=plat.get("platform", "eth"), + platform_args=plat.get("platform_args"), + platform_dir=plat.get("platform_dir"), + interfaces=[Interface(**i) for i in plat.get("interfaces", [])], + device_sockets=[ + DeviceSocket( + device=s["device"], + ports=set(s["ports"]), + address=s["address"], + ) + for s in plat.get("device_sockets", []) + ], + port_info={ + int(port): dict(info) + for port, info in plat.get("port_info", {}).items() + }, + ), + logging=LoggingOptions( + log_file=log.get("log_file", "ptf.log"), + log_dir=log.get("log_dir"), + debug=log.get("debug", "verbose"), + profile=log.get("profile", False), + profile_file=log.get("profile_file", "profile.out"), + xunit=log.get("xunit", False), + xunit_dir=log.get("xunit_dir", "xunit"), + ), + test_behavior=TestBehaviorOptions( + relax=beh.get("relax", False), + failfast=beh.get("failfast", False), + fail_skipped=beh.get("fail_skipped", False), + test_params=beh.get("test_params"), + default_timeout=beh.get("default_timeout", 2.0), + default_negative_timeout=beh.get("default_negative_timeout", 0.1), + minsize=beh.get("minsize", 0), + random_seed=beh.get("random_seed"), + test_case_timeout=beh.get("test_case_timeout"), + qlen=beh.get("qlen", 100), + disable_ipv6=beh.get("disable_ipv6", False), + disable_vxlan=beh.get("disable_vxlan", False), + disable_erspan=beh.get("disable_erspan", False), + disable_geneve=beh.get("disable_geneve", False), + disable_mpls=beh.get("disable_mpls", False), + disable_nvgre=beh.get("disable_nvgre", False), + disable_igmp=beh.get("disable_igmp", False), + disable_rocev2=beh.get("disable_rocev2", False), + ), + socket=SocketOptions( + socket_recv_size=sock.get("socket_recv_size", 4096), + ), + extra_config=data.get("extra_config", {}), + ) + + +######################################################################## +# Helpers (ported from the former top-level ptf script) +######################################################################## + + +_RUN_LOCK = threading.Lock() +_active_run_state = None +_MISSING = object() + + +class _ForwardingHandler(logging.Handler): + """Forward PTF records to a caller-owned logger without owning it.""" + + def __init__(self, target): + super().__init__() + self.target = target + + def emit(self, record): + if ( + not self.target.disabled + and record.levelno >= self.target.getEffectiveLevel() + ): + self.target.handle(record) + + +def _logger_propagates_to(logger, ancestor): + while logger is not None: + if logger is ancestor: + return True + if not logger.propagate: + return False + logger = logger.parent + return False + + +def _caller_uses_log_path(path, directory=False): + expected = os.path.realpath(os.path.abspath(path)) + loggers = [logging.getLogger()] + loggers.extend( + logger + for logger in logging.root.manager.loggerDict.values() + if isinstance(logger, logging.Logger) + ) + for logger in loggers: + for handler in logger.handlers: + if not isinstance(handler, logging.FileHandler) or getattr( + handler, "_ptf_owned", False + ): + continue + actual = os.path.realpath(handler.baseFilename) + if directory: + try: + if os.path.commonpath((expected, actual)) == expected: + return True + except ValueError: + continue + elif actual == expected: + return True + return False + + +class _LoggingSession: + """Own the handlers installed for one run and restore logger state.""" + + def __init__(self, config, output): + self.config = config + self.output = output + self.logger = ( + logging.getLogger() + if output.capture_root_logging + else logging.getLogger("ptf") + ) + self.saved_level = self.logger.level + self.saved_disabled = self.logger.disabled + self.saved_propagate = self.logger.propagate + self.saved_opener = ptf._logfile_opener + self.handlers = [] + + def start(self): + if ( + self.output.logger is not None + and self.output.logger is not self.logger + and _logger_propagates_to(self.output.logger, self.logger) + ): + raise ValueError("output logger must not propagate back to the PTF logger") + self.logger.setLevel(DEBUG_LEVELS[self.config.logging.debug]) + self.logger.disabled = False + if not self.output.capture_root_logging: + self.logger.propagate = False + ptf._logfile_opener = self.open_logfile + self.open_logfile("main") + + def open_logfile(self, name): + for handler in self.handlers: + self.logger.removeHandler(handler) + handler.close() + self.handlers = [] + + if self.config.logging.log_dir is not None: + filename = os.path.join(self.config.logging.log_dir, name) + ".log" + else: + filename = self.config.logging.log_file + + formatter = logging.Formatter( + "%(asctime)s.%(msecs)03d %(name)-10s: %(levelname)-8s: %(message)s", + "%H:%M:%S", + ) + if filename is not None: + file_handler = logging.FileHandler(filename, mode="a") + file_handler._ptf_owned = True + file_handler.setFormatter(formatter) + self._add_handler(file_handler) + ptfutils.chown_to_invoking_user(filename) + + error_handler = logging.StreamHandler(self.output.stderr) + error_handler._ptf_owned = True + error_handler.setLevel(logging.ERROR) + error_handler.setFormatter(formatter) + self._add_handler(error_handler) + + if self.output.logger is not None and self.output.logger is not self.logger: + observer = _ForwardingHandler(self.output.logger) + observer._ptf_owned = True + self._add_handler(observer) + + def _add_handler(self, handler): + self.logger.addHandler(handler) + self.handlers.append(handler) + + def close(self): + ptf._logfile_opener = self.saved_opener + for handler in self.handlers: + self.logger.removeHandler(handler) + handler.close() + self.handlers = [] + self.logger.setLevel(self.saved_level) + self.logger.disabled = self.saved_disabled + self.logger.propagate = self.saved_propagate + + +class _RunState: + """Snapshot and restore process state owned by an in-process PTF run.""" + + def __init__(self, config): + self.config = config + self.config_object = ptf.config + self.config_contents = dict(self.config_object) + self.dataplane = None + self.saved_dataplane = ptf.dataplane_instance + self.added_paths = [] + self.modules = dict(sys.modules) + self.touched_modules = set() + self.module_roots = set() + self.random_state = random.getstate() + self.profile = sys.getprofile() + self.logging_disable = logging.root.manager.disable + self.logging_disable_stack = list(ptf._logging_disable_stack) + self.ptfutils_timeouts = ( + ptfutils.default_timeout, + ptfutils.default_negative_timeout, + ) + self.testutils_state = None + self.platform_module = None + self.track_root(config.test_selection.test_dir) + for path in config.pypath: + self.track_root(path) + + def activate(self, config_dict): + ptf.config = self.config_object + self.config_object.clear() + self.config_object.update(config_dict) + for path in self.config.pypath: + self.add_path(path) + + def add_path(self, path): + self.added_paths.append((len(sys.path), path)) + sys.path.append(path) + + def track_root(self, path): + if path: + self.module_roots.add(os.path.realpath(os.path.abspath(path))) + + def mark_module(self, name): + self.touched_modules.add(name) + + def module_was_loaded(self, name, source_path): + module = sys.modules.get(name) + if module is None: + return False + module_path = getattr(module, "__file__", None) + if module_path is None: + return False + return module is not self.modules.get(name) and os.path.realpath( + module_path + ) == os.path.realpath(source_path) + + def capture_testutils(self, testutils): + filters = testutils.FILTERS + self.testutils_state = ( + testutils, + testutils.TEST_PARAMS, + testutils.PORT_INFO, + testutils.MINSIZE, + testutils.skipped_test_count, + filters, + list(filters), + ) + + def close_resources(self): + errors = [] + if self.dataplane is not None: + try: + self.dataplane.stop_pcap() + except Exception as error: + LOGGER.exception("Failed to stop PTF packet capture") + errors.append(error) + try: + self.dataplane.kill() + except Exception as error: + LOGGER.exception("Failed to shut down the PTF dataplane") + errors.append(error) + self.dataplane = None + if self.platform_module is not None: + teardown = getattr(self.platform_module, "platform_config_teardown", None) + if callable(teardown): + try: + teardown(ptf.config) + except Exception as error: + LOGGER.exception("Failed to tear down the PTF platform") + errors.append(error) + return errors + + def _track_modules_from_roots(self): + for name, module in list(sys.modules.items()): + if module is self.modules.get(name): + continue + module_path = getattr(module, "__file__", None) + if module_path is None: + continue + path = os.path.realpath(module_path) + if any( + path == root or path.startswith(root + os.sep) + for root in self.module_roots + ): + self.touched_modules.add(name) + + def restore(self): + self._track_modules_from_roots() + for name in self.touched_modules: + previous = self.modules.get(name, _MISSING) + if previous is _MISSING: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous + + if self.testutils_state is not None: + ( + testutils, + testutils.TEST_PARAMS, + testutils.PORT_INFO, + testutils.MINSIZE, + testutils.skipped_test_count, + filters, + filter_contents, + ) = self.testutils_state + testutils.FILTERS = filters + filters[:] = filter_contents + + ptfutils.default_timeout, ptfutils.default_negative_timeout = ( + self.ptfutils_timeouts + ) + ptf.dataplane_instance = self.saved_dataplane + ptf.config = self.config_object + self.config_object.clear() + self.config_object.update(self.config_contents) + for index, path in reversed(self.added_paths): + if index < len(sys.path) and sys.path[index] == path: + sys.path.pop(index) + else: + for current in range(len(sys.path) - 1, index - 1, -1): + if sys.path[current] == path: + sys.path.pop(current) + break + importlib.invalidate_caches() + random.setstate(self.random_state) + sys.setprofile(self.profile) + logging.disable(self.logging_disable) + ptf._logging_disable_stack[:] = self.logging_disable_stack + + +def import_module(root_path, module_name): + """Import a module from the given directory, instead of the standard + Python search path. The function registers the module in sys.modules + under its name. This registration lets test modules import each + other.""" + finder = importlib.machinery.PathFinder() + module_spec = finder.find_spec(module_name, [root_path]) + if module_spec is None or module_spec.loader is None: + raise ImportError("No module named %r in %r" % (module_name, root_path)) + module = importlib.util.module_from_spec(module_spec) + if _active_run_state is not None: + _active_run_state.mark_module(module_name) + # Register the module so that subsequent imports of the same name + # resolve to it (this also lets test modules import each other). + sys.modules[module_name] = module + module_spec.loader.exec_module(module) + return module + + +def logging_setup(session): + """ + Set up logging based on the global ptf.config + """ + + if ptf.config["log_dir"] != None: + if os.path.exists(ptf.config["log_dir"]): + if _caller_uses_log_path(ptf.config["log_dir"], directory=True): + raise PtfError("PTF log directory contains a caller-owned log file") + shutil.rmtree(ptf.config["log_dir"]) + os.makedirs(ptf.config["log_dir"]) + ptfutils.chown_to_invoking_user(ptf.config["log_dir"]) + else: + if ( + ptf.config["log_file"] is not None + and os.path.exists(ptf.config["log_file"]) + and not _caller_uses_log_path(ptf.config["log_file"]) + ): + os.remove(ptf.config["log_file"]) + + session.start() + + +def xunit_setup(): + """ + Set up xUnit output based on the global ptf.config + """ + + if not ptf.config["xunit"]: + return + + if os.path.exists(ptf.config["xunit_dir"]): + shutil.rmtree(ptf.config["xunit_dir"]) + os.makedirs(ptf.config["xunit_dir"]) + ptfutils.chown_to_invoking_user(ptf.config["xunit_dir"]) + + +def pcap_setup(): + """ + Set up dataplane packet capturing based on the global ptf.config + """ + + if ptf.config["log_dir"] is None and ptf.config["log_file"] is not None: + filename = os.path.splitext(ptf.config["log_file"])[0] + ".pcap" + ptf.dataplane_instance.start_pcap(filename) + + +def profiler_setup(): + """ + Set up profiler based on the global ptf.config; returns the profiler + object (or None when profiling is disabled). + """ + + if not ptf.config["profile"]: + return None + + import cProfile + + profiler = cProfile.Profile() + profiler.enable() + + return profiler + + +def profiler_teardown(profiler): + """ + Tear down profiler based on the global ptf.config + """ + + if profiler is None: + return + + profiler.disable() + profiler.dump_stats(ptf.config["profile_file"]) + ptfutils.chown_to_invoking_user(ptf.config["profile_file"]) + + +def load_test_modules(): + """ + Load tests from the test_dir directory. + + Test cases are subclasses of unittest.TestCase + + Also updates the _groups member to include "standard" and + module test groups if appropriate. + + @returns A dictionary from test module names to tuples of + (module, dictionary from test names to test classes). + """ + + result = OrderedDict() + loaded_paths = {} + + for root, dirs, filenames in os.walk(ptf.config["test_dir"]): + pyfiles = fnmatch.filter(filenames, "[!.]*.py") + + # guarantee that files will be visited in the same order every time tests are loaded + pyfiles.sort() + dirs.sort() + + if len(pyfiles) == 0: + continue + + # Allow tests to import each other + if _active_run_state is not None: + _active_run_state.add_path(root) + else: + sys.path.append(root) + + # Iterate over each python file + for filename in pyfiles: + modname = os.path.splitext(os.path.basename(filename))[0] + source_path = os.path.realpath(os.path.join(root, filename)) + if modname == "ptf": + raise PtfError("a test module cannot be named 'ptf': %r" % source_path) + + try: + previous_path = loaded_paths.get(modname) + if previous_path is not None and previous_path != source_path: + raise PtfError( + "duplicate test module name %r in %r and %r" + % (modname, previous_path, source_path) + ) + if previous_path is not None or ( + _active_run_state is not None + and _active_run_state.module_was_loaded(modname, source_path) + ): + mod = sys.modules[modname] + else: + mod = import_module(root, modname) + loaded_paths[modname] = source_path + except: + LOGGER.warning("Could not import file " + filename) + raise + + # Find all testcases defined in the module + tests = dict( + (k, v) + for (k, v) in mod.__dict__.items() + if type(v) == type + and issubclass(v, unittest.TestCase) + and hasattr(v, "runTest") + ) + if tests: + for testname, test in tests.items(): + # Set default annotation values + if "_groups" not in test.__dict__: + test._groups = list(getattr(test, "_groups", ())) + if not hasattr(test, "_nonstandard"): + test._nonstandard = False + if not hasattr(test, "_disabled"): + test._disabled = False + if not hasattr(test, "_testtimeout"): + test._testtimeout = None + + # Put test in its module's test group + if not test._disabled: + if modname not in test._groups: + test._groups.append(modname) + else: + # If the test is disabled, create a group named + # disabled and add it too. This is so that + # users can conveniently exclude disabled tests + # too when including only groups. Eg. + # -s "group1 ^disabled" + if "disabled" not in test._groups: + test._groups.append("disabled") + + # Put test in the standard test group + if not test._disabled and not test._nonstandard: + if "standard" not in test._groups: + test._groups.append("standard") + if "all" not in test._groups: + test._groups.append("all") # backwards compatibility + + result[modname] = (mod, tests) + + return result + + +def prune_tests(test_specs, test_modules): + """ + Return tests matching the given test-specs. + @param test_specs A list of group names or test names. + @param test_modules Same format as the output of load_test_modules. + @returns Same format as the output of load_test_modules. + """ + result = OrderedDict() + for e in test_specs: + matched = False + + if e.startswith("^"): + negated = True + e = e[1:] + else: + negated = False + + for modname, (mod, tests) in test_modules.items(): + for testname, test in tests.items(): + if e in test._groups or e == "%s.%s" % (modname, testname): + result.setdefault(modname, (mod, OrderedDict())) + if not negated: + # if not hasattr(test, "_versions") or version in test._versions: + result[modname][1][testname] = test + else: + if modname in result and testname in result[modname][1]: + del result[modname][1][testname] + if not result[modname][1]: + del result[modname] + matched = True + + if not matched and not negated: + raise PtfError("test-spec element %s did not match any tests" % e) + + return result + + +def apply_test_timeout(test, default_test_case_timeout=None): + original_run = test.run + + def run_with_timeout(self, result=None): + test_case_timeout = getattr(self, "_testtimeout", None) + if test_case_timeout is None: + test_case_timeout = default_test_case_timeout + + if test_case_timeout: + with ptfutils.Timeout(test_case_timeout): + return original_run(result) + return original_run(result) + + test.run = types.MethodType(run_with_timeout, test) + return test + + +def parse_test_params(test_params): + """ + Parse the test parameters. The input accepts three forms: None (no + parameters), a dictionary (used as given), or a string in the + 'key=value;key=value' syntax of the --test-params command line option. + The binary and this function evaluate each string value as a Python + expression. + @returns A dictionary of parameters, or None. + """ + if test_params is None: + LOGGER.debug("No test params were provided with '--test-params' / '-t'") + return None + if isinstance(test_params, dict): + params = dict(test_params) + LOGGER.debug("Parsed test parameters:") + for k, v in params.items(): + LOGGER.debug("\t*{}={}".format(k, v)) + return params + params_str = "class _TestParams:\n " + test_params + namespace = {} + try: + exec(params_str, namespace) + except: + LOGGER.error( + "Error when parsing test params " + "(provided with '--test-params' / '-t'). " + "Make sure you used the correct syntax: " + '--test-params="[k=v;]*k=v"' + ) + return None + params = {} + LOGGER.debug("Parsed test parameters:") + for k, v in list(vars(namespace["_TestParams"]).items()): + if k[:2] != "__": + params[k] = v + LOGGER.debug("\t*{}={}".format(k, v)) + LOGGER.debug( + "If something is missing, make sure you used the correct syntax: " + '--test-params="[k=v;]*k=v"' + ) + return params + + +def _space_to(n, str): + """ + Generate a string of spaces to achieve width n given string str + If length of str >= n, return one space + """ + spaces = n - len(str) + if spaces > 0: + return " " * spaces + return " " + + +def _print_test_list(test_modules, stream): + print( + """\ +Tests are shown grouped by module. If a test is in any groups beyond "standard" +and its module's group then they are shown in parentheses.""", + file=stream, + ) + print(file=stream) + print( + """\ +Tests marked with '!' are disabled because they are experimental, special-purpose, +or are too long to be run normally. These are not part of the "standard" test +group or their module's test group.""", + file=stream, + ) + print(file=stream) + print("Test List:", file=stream) + mod_count = 0 + test_count = 0 + all_groups = set() + for modname, (mod, tests) in test_modules.items(): + mod_count += 1 + desc = (mod.__doc__ or "No description").strip().split("\n")[0] + start_str = " Module " + mod.__name__ + ": " + print(start_str + _space_to(22, start_str) + desc, file=stream) + for testname, test in list(tests.items()): + try: + desc = (test.__doc__ or "").strip() + desc = desc.split("\n")[0] + except: + desc = "No description" + groups = set(test._groups) - set(["all", "standard", modname]) + all_groups.update(test._groups) + if groups: + desc = "(%s) %s" % (",".join(groups), desc) + if hasattr(test, "_versions"): + desc = "(%s) %s" % (",".join(sorted(test._versions)), desc) + start_str = " %s%s %s:" % ( + test._nonstandard and "*" or " ", + test._disabled and "!" or " ", + testname, + ) + if len(start_str) > 22: + desc = "\n" + _space_to(22, "") + desc + print(start_str + _space_to(22, start_str) + desc, file=stream) + test_count += 1 + print(file=stream) + print( + "%d modules shown with a total of %d tests" % (mod_count, test_count), + file=stream, + ) + print(file=stream) + print("Test groups: %s" % (", ".join(sorted(all_groups))), file=stream) + + +######################################################################## +# Test run entry point +######################################################################## + + +def _validate_config(config): + ts = config.test_selection + if ts.test_dir is None or not os.path.isdir(ts.test_dir): + raise PtfError("invalid test directory: %r" % (ts.test_dir,)) + if ts.test_order not in TEST_ORDERS: + raise PtfError( + "invalid test order %r, expected one of %s" % (ts.test_order, TEST_ORDERS) + ) + if config.logging.debug not in DEBUG_LEVELS: + raise PtfError( + "invalid debug level %r, expected one of %s" + % (config.logging.debug, sorted(DEBUG_LEVELS, key=DEBUG_LEVELS.get)) + ) + + +def _coerce_config(config): + if config is None: + return PtfConfig() + if isinstance(config, dict): + return PtfConfig.from_dict(config) + if not isinstance(config, PtfConfig): + raise TypeError("expected a PtfConfig or dict, got %r" % (config,)) + return config + + +def run(config=None, *, output=None, _manage_signals=False): + # type: (Union[PtfConfig, Dict[str, Any], None], Optional[RunOutput], bool) -> int + """Run PTF tests as described by the given configuration. + + @param config A PtfConfig instance, or a dictionary in the flat + 'ptf.config' format. run() converts a dictionary with + PtfConfig.from_dict. None is the same as a default PtfConfig. + @return The exit code that the ptf binary produces for the same + configuration: 0 on success, 0 for the list modes, 1 when a test + failed, errored, or was skipped while fail_skipped is set. + + A fatal configuration or environment problem raises PtfError. Exactly + one in-process run may be active. Independent PTF processes can run in + parallel. + + ``run()`` restores PTF configuration, imports, random state, logging, + profiling, and test utility globals. + It does not change SIGINT. Signal-based per-test timeouts require the main + thread. ``output`` controls framework streams and optional logging + forwarding without becoming part of the serializable configuration. + """ + config = _coerce_config(config) + _validate_config(config) + output = output or RunOutput() + if not isinstance(output, RunOutput): + raise TypeError("output must be a RunOutput, got %r" % (output,)) + output = RunOutput( + stdout=sys.stdout if output.stdout is None else output.stdout, + stderr=sys.stderr if output.stderr is None else output.stderr, + logger=output.logger, + capture_root_logging=output.capture_root_logging, + ) + if _manage_signals and threading.current_thread() is not threading.main_thread(): + raise PtfError("process signal management requires the main Python thread") + if not _RUN_LOCK.acquire(blocking=False): + raise PtfError( + "another in-process PTF run is active; parallel PTF runs require " + "separate processes" + ) + + global _active_run_state + state = None + session = None + saved_sigint_handler = _MISSING + try: + state = _RunState(config) + _active_run_state = state + config_dict = config.to_dict() + if config_dict["packet_manipulation_module"] is None: + config_dict["packet_manipulation_module"] = ( + os.environ.get("PTF_PACKET_MANIPULATION_MODULE") + or DEFAULT_PACKET_MANIPULATION_MODULE + ) + state.activate(config_dict) + if _manage_signals: + saved_sigint_handler = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, signal.SIG_DFL) + + session = _LoggingSession(config, output) + logging_setup(session) + xunit_setup() + LOGGER.info("++++++++ " + time.asctime() + " ++++++++") + try: + return _execute(config, output, state) + except PtfError as error: + LOGGER.critical(str(error)) + error._ptf_logged = True + raise + finally: + had_exception = sys.exc_info()[0] is not None + cleanup_errors = [] + try: + if state is not None: + cleanup_errors.extend(state.close_resources()) + if saved_sigint_handler is not _MISSING: + try: + signal.signal(signal.SIGINT, saved_sigint_handler) + except Exception as error: + LOGGER.exception("Failed to restore the SIGINT handler") + cleanup_errors.append(error) + if session is not None: + try: + session.close() + except Exception as error: + cleanup_errors.append(error) + finally: + _active_run_state = None + try: + if state is not None: + try: + state.restore() + except Exception as error: + cleanup_errors.append(error) + finally: + _RUN_LOCK.release() + if cleanup_errors and not had_exception: + error = PtfError( + "PTF cleanup failed: %s" + % "; ".join(str(error) for error in cleanup_errors) + ) + raise error + + +def _execute(config, output, state): + # type: (PtfConfig) -> int + # The actual test run. This function assumes that the global ptf.config + # is populated and that logging is set up. It returns the exit code. + + # Import after logging is configured. This silences the scapy error + # logs from the import of packet.py, and logs the warnings of ptf + # correctly. + packet_module = sys.modules.get("ptf.packet") + requested_packet_config = { + name: ptf.config[name] + for name in ( + "disable_ipv6", + "disable_vxlan", + "disable_erspan", + "disable_geneve", + "disable_mpls", + "disable_nvgre", + "disable_igmp", + "disable_rocev2", + ) + } + if packet_module is not None: + loaded_packet_module = getattr( + packet_module, "_packet_manipulation_module", None + ) + loaded_packet_config = getattr(packet_module, "_packet_config", None) + if ( + loaded_packet_module != ptf.config["packet_manipulation_module"] + or loaded_packet_config != requested_packet_config + ): + raise PtfError( + "the requested packet configuration differs from the one already " + "loaded in this process; start a new PTF process" + ) + else: + backend_module = sys.modules.get(ptf.config["packet_manipulation_module"]) + backend_config = getattr(backend_module, "_ptf_packet_config", None) + if backend_config is not None and backend_config != requested_packet_config: + raise PtfError( + "the requested packet configuration differs from the packet backend " + "already loaded in this process; start a new PTF process" + ) + + testutils = importlib.import_module("ptf.testutils") + + state.capture_testutils(testutils) + + # Parse the test parameters and log them. Do this before the test + # modules are imported: a test may read its parameters at import + # time. + testutils.TEST_PARAMS = parse_test_params(config.test_behavior.test_params) + testutils.PORT_INFO = dict(ptf.config["port_info"]) + testutils.MINSIZE = ptf.config["minsize"] + testutils.skipped_test_count = 0 + testutils.FILTERS.clear() + ptfutils.default_timeout = ptf.config["default_timeout"] + ptfutils.default_negative_timeout = ptf.config["default_negative_timeout"] + + test_specs = list(config.test_selection.test_specs) + if ptf.config["test_file"] != None: + with open(ptf.config["test_file"], "r") as f: + for line in f: + line, _, _ = line.partition("#") # remove comments + line = line.strip() + if line: + test_specs.append(line) + if test_specs == []: + test_specs = ["standard"] + + test_modules = load_test_modules() + + # Check if test list is requested; display and return if so + if ptf.config["list"]: + _print_test_list(test_modules, output.stdout) + return 0 + + test_modules = prune_tests(test_specs, test_modules) + + # Check if test list is requested; display and return if so + if ptf.config["list_test_names"]: + for modname, (mod, tests) in test_modules.items(): + for testname, test in tests.items(): + print("%s.%s" % (modname, testname), file=output.stdout) + return 0 + + # Generate the test suite + test_suite = [] + for modname, (mod, tests) in test_modules.items(): + for testname, test in tests.items(): + test_suite.append(test()) + + if ptf.config["shard_id"] < 0 or ptf.config["shard_id"] >= ptf.config["num_shards"]: + raise PtfError( + "shard id should be equal or greater than 0 and lower than number of shards" + ) + test_suite = test_suite[ptf.config["shard_id"] :: ptf.config["num_shards"]] + + if ptf.config["test_order"] == "lexico": + test_suite.sort() + elif ptf.config["test_order"] == "rand": + seed = ptf.config["test_order_seed"] + random.seed(seed) + random.shuffle(test_suite) + + if threading.current_thread() is not threading.main_thread(): + for test in test_suite: + timeout = getattr(test, "_testtimeout", None) + if timeout is None: + timeout = ptf.config["test_case_timeout"] + if timeout and timeout > 0: + raise PtfError("test-case timeouts require the main Python thread") + + test_suite = [ + apply_test_timeout(test, ptf.config["test_case_timeout"]) for test in test_suite + ] + test_suite = unittest.TestSuite(test_suite) + + if ptf.config["platform_dir"] is None: + from ptf import platforms + + ptf.config["platform_dir"] = os.path.dirname( + os.path.abspath(platforms.__file__) + ) + + # Allow platforms to import each other + state.add_path(ptf.config["platform_dir"]) + state.track_root(ptf.config["platform_dir"]) + + # Load the platform module + platform_name = ptf.config["platform"] + LOGGER.info("Importing platform: " + platform_name) + + # TODO(antonin): put this check in platforms/nn.py ? + if platform_name == "nn": + try: + import pynng # noqa: F401 pylint: disable=unused-import + except ImportError: + raise PtfError("Cannot use 'nn' platform if pynng package is not installed") + + platform_mod = None + try: + platform_mod = import_module(ptf.config["platform_dir"], platform_name) + except: + LOGGER.warning("Failed to import " + platform_name + " platform module") + raise + state.platform_module = platform_mod + + try: + platform_mod.platform_config_update(ptf.config) + except: + LOGGER.warning("Could not run platform host configuration") + raise + + if ptf.config["port_map"] is None: + raise PtfError("Interface port map was not defined by the platform. Exiting.") + + LOGGER.debug("Configuration: " + str(ptf.config)) + LOGGER.info("port map: " + str(ptf.config["port_map"])) + + if os.getuid() != 0 and not ptf.config["allow_user"] and platform_name != "nn": + raise PtfError( + "Super-user privileges required. Please re-run with sudo or as root." + ) + + if ptf.config["random_seed"] is not None: + LOGGER.info("Random seed: %d" % ptf.config["random_seed"]) + random.seed(ptf.config["random_seed"]) + else: + # Generate random seed and report to log file + seed = random.randrange(100000000) + LOGGER.info("Autogen random seed: %d" % seed) + random.seed(seed) + + profiler = profiler_setup() + try: + if ptf.config["port_map"]: + dataplane = importlib.import_module("ptf.dataplane") + + # Set up the dataplane only when the selected platform exposes ports. + state.dataplane = dataplane.DataPlane(ptf.config) + ptf.dataplane_instance = state.dataplane + pcap_setup() + for port_id, ifname in ptf.config["port_map"].items(): + device, port = port_id + ptf.dataplane_instance.port_add(ifname, device, port) + else: + ptf.dataplane_instance = None + + LOGGER.info("*** TEST RUN START: " + time.asctime()) + if ptf.config["xunit"]: + try: + import xmlrunner # fail-fast if module missing + except ImportError: + raise + test_runner = xmlrunner.XMLTestRunner( + output=ptf.config["xunit_dir"], + outsuffix="", + verbosity=2, + failfast=ptf.config["failfast"], + stream=output.stderr, + ) + else: + test_runner = unittest.TextTestRunner( + verbosity=2, + failfast=ptf.config["failfast"], + stream=output.stderr, + ) + result = test_runner.run(test_suite) + if ptf.config["xunit"]: + # The XML result files are only written once the run completes. + ptfutils.chown_to_invoking_user(ptf.config["xunit_dir"], recursive=True) + run_failures = result.failures + run_errors = result.errors + run_timeouts = [] + for case in result.errors: + traceback_str = case[1] + # TODO: hacky? could not think of a better way + if "raise Timeout.TimeoutError()" in traceback_str: + LOGGER.info("Test case failed because of timeout") + run_timeouts.append(case) + + ptf.open_logfile("main") + testutils.skipped_test_count = len(getattr(result, "skipped", ())) + if testutils.skipped_test_count > 0: + ts = " tests" + if testutils.skipped_test_count == 1: + ts = " test" + LOGGER.info("Skipped " + str(testutils.skipped_test_count) + ts) + print( + "Skipped " + str(testutils.skipped_test_count) + ts, + file=output.stdout, + ) + LOGGER.info("*** TEST RUN END : " + time.asctime()) + + if run_failures or run_errors: + print(file=output.stdout) + print("******************************************", file=output.stdout) + print("ATTENTION: SOME TESTS DID NOT PASS!!!", file=output.stdout) + if (not ptf.config["xunit"]) and run_failures: + print(file=output.stdout) + print("The following tests failed:", file=output.stdout) + print( + ", ".join([f[0].__class__.__name__ for f in run_failures]), + file=output.stdout, + ) + if (not ptf.config["xunit"]) and run_errors: + print(file=output.stdout) + print("The following tests errored:", file=output.stdout) + print( + ", ".join([f[0].__class__.__name__ for f in run_errors]), + file=output.stdout, + ) + if (not ptf.config["xunit"]) and run_timeouts: + print(file=output.stdout) + print( + "The following tests errored because of a timeout:", + file=output.stdout, + ) + print( + ", ".join([f[0].__class__.__name__ for f in run_timeouts]), + file=output.stdout, + ) + print(file=output.stdout) + print("******************************************", file=output.stdout) + return 1 + if testutils.skipped_test_count > 0 and ptf.config["fail_skipped"]: + print(file=output.stdout) + print("******************************************", file=output.stdout) + print( + "ATTENTION: %d TESTS WERE SKIPPED!!!" % testutils.skipped_test_count, + file=output.stdout, + ) + print("******************************************", file=output.stdout) + print(file=output.stdout) + return 1 + return 0 + finally: + profiler_teardown(profiler) + + +if __name__ == "__main__": + # Run PTF tests from a serialized PtfConfig, without the command line + # parser of the ptf binary: + # python -m ptf.runner + # Use this entry point to start PTF as a Python process, for example + # inside a network namespace, and to keep the configuration structured. + import argparse + + parser = argparse.ArgumentParser( + prog="python -m ptf.runner", + description="Run PTF tests described by a PtfConfig JSON file " + "(see ptf.runner.PtfConfig.to_json).", + ) + parser.add_argument( + "config_file", help="Path to the PtfConfig JSON file, or - to read stdin" + ) + args = parser.parse_args() + if args.config_file == "-": + _config = PtfConfig.from_json(sys.stdin.read()) + else: + with open(args.config_file, "r") as f: + _config = PtfConfig.from_json(f.read()) + try: + _rc = run( + _config, + output=RunOutput(capture_root_logging=True), + _manage_signals=True, + ) + except PtfError as error: + if not getattr(error, "_ptf_logged", False): + print("PTF error: %s" % error, file=sys.stderr) + _rc = 1 + # A normal exit can hang when non-daemon threads are still active; + # see ptf.cli.main. + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/ptf/testutils.py b/src/ptf/testutils.py index 5200231..e192633 100755 --- a/src/ptf/testutils.py +++ b/src/ptf/testutils.py @@ -19,7 +19,8 @@ import ptf.dataplane import ptf.parse import ptf.ptfutils -from io import StringIO + +logger = logging.getLogger(__name__) global skipped_test_count skipped_test_count = 0 @@ -40,7 +41,7 @@ def reset_filters(): - FILTERS = [] + FILTERS.clear() # Needs to be a callable @@ -602,7 +603,7 @@ def simple_geneve_packet( @param inner_frame The inner Ethernet frame """ if packet.GENEVE is None: - logging.error( + logger.error( "A GENEVE packet was requested but GENEVE is not supported by your Scapy. See README for more information" ) return None @@ -726,7 +727,7 @@ def simple_nvgre_packet( this packet other than that it is a valid ethernet/IP/NVGRE frame. """ if packet.NVGRE is None: - logging.error( + logger.error( "A NVGRE packet was requested but NVGRE is not supported by your Scapy. See README for more information" ) return None @@ -853,7 +854,7 @@ def simple_vxlan_packet( this packet other than that it is a valid ethernet/IP/UDP/VXLAN frame. """ if packet.VXLAN is None: - logging.error( + logger.error( "A VXLAN packet was requested but VXLAN is not supported by your Scapy. See README for more information" ) return None @@ -1366,7 +1367,7 @@ def simple_gre_erspan_packet( this packet other than that it is a valid ethernet/IP/GRE/ERSPAN frame. """ if packet.GRE is None or packet.ERSPAN is None: - logging.error( + logger.error( "A GRE/ERSPAN packet was requested but GRE or ERSPAN is not supported by your Scapy. See README for more information" ) return None @@ -1519,7 +1520,7 @@ def ipv4_erspan_pkt( @param inner_frame payload of the GRE packet """ if packet.GRE is None or packet.ERSPAN is None or packet.ERSPAN_III is None: - logging.error( + logger.error( "A GRE/ERSPAN packet was requested but GRE or ERSPAN is not supported by your Scapy. See README for more information" ) return None @@ -1671,7 +1672,7 @@ def ipv4_erspan_platform_pkt( or packet.ERSPAN_III is None or packet.PlatformSpecific is None ): - logging.error( + logger.error( "A GRE/ERSPAN packet was requested but GRE or ERSPAN is not supported by your Scapy. See README for more information" ) return None @@ -2532,7 +2533,7 @@ def simple_mpls_packet( """ if packet.MPLS is None: - logging.error( + logger.error( "A MPLS packet was requested but MPLS is not supported by your Scapy. See README for more information" ) return None @@ -2685,7 +2686,7 @@ def simple_igmp_packet( this packet other than that it is a valid ethernet/IP/IGMP frame. """ if packet.IGMP is None: - logging.error( + logger.error( "An IGMP packet was requested but IGMP is not supported by your Scapy. See README for more information" ) return None @@ -3062,7 +3063,7 @@ def get_egr_list(parent, ports, how_many, exclude_list=[]): count += 1 if count >= how_many: return egr_ports - logging.debug("Could not generate enough egress ports for test") + logger.debug("Could not generate enough egress ports for test") return [] @@ -3159,17 +3160,10 @@ def inspect_packet(pkt): Wrapper around scapy's show() method. @returns A string showing the dissected packet. """ - out = None - backup = sys.stdout try: - tmp = StringIO() - sys.stdout = tmp - pkt.show2() - out = tmp.getvalue() - tmp.close() - finally: - sys.stdout = backup - return out + return pkt.show2(dump=True) + except TypeError: + return packet.format_packet(pkt) def nonstandard(cls): @@ -3297,7 +3291,7 @@ def verify_packet(test, pkt, port_id, timeout=None): if not timeout: timeout = ptf.ptfutils.default_timeout device, port = port_to_tuple(port_id) - logging.debug("Checking for pkt on device %d, port %d", device, port) + logger.debug("Checking for pkt on device %d, port %d", device, port) result = dp_poll( test, device_number=device, port_number=port, timeout=timeout, exp_pkt=pkt ) @@ -3317,7 +3311,7 @@ def verify_no_packet(test, pkt, port_id, timeout=None): if timeout is None: timeout = ptf.ptfutils.default_negative_timeout device, port = port_to_tuple(port_id) - logging.debug("Negative check for pkt on device %d, port %d", device, port) + logger.debug("Negative check for pkt on device %d, port %d", device, port) result = dp_poll( test, device_number=device, port_number=port, exp_pkt=pkt, timeout=timeout ) @@ -3338,7 +3332,7 @@ def verify_no_other_packets(test, device_number=0, timeout=None): return if timeout is None: timeout = ptf.ptfutils.default_negative_timeout - logging.debug( + logger.debug( "Checking for unexpected packets on all ports of device %d" % device_number ) result = dp_poll(test, device_number=device_number, timeout=timeout) @@ -3426,7 +3420,7 @@ def verify_packets_any( if device != device_number: continue if port in ports: - logging.debug("Checking for pkt on device %d, port %d", device_number, port) + logger.debug("Checking for pkt on device %d, port %d", device_number, port) result = dp_poll( test, device_number=device, @@ -3477,7 +3471,7 @@ def verify_packet_any_port( timeout = ptf.ptfutils.default_timeout if not n_timeout: n_timeout = ptf.ptfutils.default_negative_timeout - logging.debug("Checking for pkt on device %d, port %r", device_number, ports) + logger.debug("Checking for pkt on device %d, port %r", device_number, ports) result = dp_poll(test, device_number=device_number, timeout=timeout, exp_pkt=pkt) verify_no_other_packets(test, device_number=device_number, timeout=n_timeout) @@ -3531,7 +3525,7 @@ def verify_any_packet_any_port( received = False match_index = 0 - logging.debug("Checking for pkt on device %d, port %r", device_number, ports) + logger.debug("Checking for pkt on device %d, port %r", device_number, ports) result = dp_poll(test, device_number=device_number, timeout=timeout) if isinstance(result, test.dataplane.PollSuccess) and result.port in ports: @@ -3586,7 +3580,7 @@ def verify_each_packet_on_each_port( if not n_timeout: n_timeout = ptf.ptfutils.default_negative_timeout for port, pkt in zip(ports, pkts): - logging.debug("Checking for pkt on device %d, port %d", device_number, port) + logger.debug("Checking for pkt on device %d, port %d", device_number, port) result = dp_poll( test, device_number=device_number, @@ -3646,7 +3640,7 @@ def verify_each_packet_on_multiple_port_lists( ) if rcv_device != device_number: continue - logging.debug("Checking for pkt on device %d, port %d", device_number, port) + logger.debug("Checking for pkt on device %d, port %d", device_number, port) if ptf.dataplane.match_exp_pkt(pkt, rcv_pkt): pkt_cnt += 1 rcv_ports.add(port_list.index(rcv_port)) @@ -3666,7 +3660,7 @@ def verify_packet_prefix(test, pkt, port, len, device_number=0, timeout=None): """ Check that an expected packet is received """ - logging.debug("Checking for pkt on port %r", port) + logger.debug("Checking for pkt on port %r", port) if timeout is None: timeout = ptf.ptfutils.default_timeout result = test.dataplane.poll( @@ -3815,7 +3809,7 @@ def simple_rocev2_packet( """ if packet.BTH is None: - logging.error( + logger.error( "A ROCEv2 packet was requested but ROCEv2 is not supported by your Scapy. See README for more information" ) return None @@ -3968,7 +3962,7 @@ def simple_rocev2v6_packet( """ if packet.BTH is None: - logging.error( + logger.error( "A ROCEv2 packet was requested but ROCEv2 is not supported by your Scapy. See README for more information" ) return None diff --git a/utests/specs/import_state.py b/utests/specs/import_state.py new file mode 100644 index 0000000..d7c5d9c --- /dev/null +++ b/utests/specs/import_state.py @@ -0,0 +1,14 @@ +# Copyright 2026 The P4 Language Consortium +# SPDX-License-Identifier: Apache-2.0 + +from ptf.base_tests import BaseTest +from ptf.testutils import test_param_get + +IMPORT_VALUE = test_param_get("import_value", default=-1) + + +class ImportStateProbe(BaseTest): + _nonstandard = True + + def runTest(self): + print(">>>import_value={}".format(IMPORT_VALUE)) diff --git a/utests/specs/isolation.py b/utests/specs/isolation.py new file mode 100644 index 0000000..5e53cea --- /dev/null +++ b/utests/specs/isolation.py @@ -0,0 +1,22 @@ +# Copyright 2026 The P4 Language Consortium +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from ptf.base_tests import BaseTest + + +@unittest.skip("intentional skip") +class Skipped(BaseTest): + _nonstandard = True + + def runTest(self): + pass + + +class Timed(BaseTest): + _nonstandard = True + _testtimeout = 1 + + def runTest(self): + pass diff --git a/utests/tests/ptf/test_dataplane_lifecycle.py b/utests/tests/ptf/test_dataplane_lifecycle.py new file mode 100644 index 0000000..9189c76 --- /dev/null +++ b/utests/tests/ptf/test_dataplane_lifecycle.py @@ -0,0 +1,112 @@ +# Copyright 2026 The P4 Language Consortium +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from bf_pktpy.ptf import packet_pktpy +from ptf import dataplane + + +class FakePort: + instances = [] + + def __init__(self, interface, device, port, config): + self.interface = interface + self.closed = 0 + self.instances.append(self) + + def close(self): + self.closed += 1 + + +def make_dataplane(monkeypatch): + monkeypatch.setattr(dataplane.DataPlane, "start", lambda self: None) + return dataplane.DataPlane( + {"platform": "fake", "dataplane": {"portclass": FakePort}, "qlen": 1} + ) + + +def test_port_replacement_and_removal_close_resources(monkeypatch): + FakePort.instances = [] + plane = make_dataplane(monkeypatch) + plane.port_add("first", 0, 1) + first = FakePort.instances[-1] + plane.port_add("second", 0, 1) + second = FakePort.instances[-1] + assert first.closed == 1 + assert plane.port_remove(0, 1) + assert second.closed == 1 + plane.kill() + plane.kill() + + +def test_kill_is_idempotent_and_closes_every_port(monkeypatch): + FakePort.instances = [] + plane = make_dataplane(monkeypatch) + plane.port_add("one", 0, 1) + plane.port_add("two", 0, 2) + plane.kill() + assert [port.closed for port in FakePort.instances] == [1, 1] + assert plane.ports == {} + plane.kill() + assert [port.closed for port in FakePort.instances] == [1, 1] + + +def test_kill_closes_remaining_resources_after_port_error(monkeypatch): + class BrokenPort(FakePort): + def close(self): + super().close() + if self.interface == "broken": + raise RuntimeError("close failed") + + monkeypatch.setattr(dataplane.DataPlane, "start", lambda self: None) + plane = dataplane.DataPlane( + {"platform": "fake", "dataplane": {"portclass": BrokenPort}, "qlen": 1} + ) + plane.port_add("broken", 0, 1) + plane.port_add("healthy", 0, 2) + + with pytest.raises(RuntimeError, match="cleanup failed"): + plane.kill() + assert [port.closed for port in BrokenPort.instances[-2:]] == [1, 1] + assert plane.ports == {} + assert plane.waker.pipe_rd is None + + +def test_nn_source_closes_after_its_final_port(monkeypatch): + class FakeSource: + instances = [] + + def __init__(self, *args): + self.ports = set() + self.removed = [] + self.closed = 0 + self.instances.append(self) + + def port_add(self, port): + self.ports.add(port) + + def port_remove(self, port): + if port in self.ports: + self.ports.remove(port) + self.removed.append(port) + + def close(self): + self.closed += 1 + + monkeypatch.setattr(dataplane, "DataPlanePacketSourceNN", FakeSource) + dataplane.DataPlanePortNN.packet_injecters.clear() + first = dataplane.DataPlanePortNN("ipc:///tmp/test", 0, 1) + second = dataplane.DataPlanePortNN("ipc:///tmp/test", 0, 2) + source = FakeSource.instances[0] + first.close() + assert source.closed == 0 + second.close() + assert source.removed == [1, 2] + assert source.closed == 1 + assert dataplane.DataPlanePortNN.packet_injecters == {} + + +def test_bf_pktpy_hexdump_formatter_returns_text(): + result = packet_pktpy.format_hexdump(packet_pktpy.Ether()) + assert isinstance(result, str) diff --git a/utests/tests/ptf/test_ptfutils.py b/utests/tests/ptf/test_ptfutils.py index 50fe5ad..cfaf5da 100644 --- a/utests/tests/ptf/test_ptfutils.py +++ b/utests/tests/ptf/test_ptfutils.py @@ -3,11 +3,12 @@ import logging import os +import signal import pytest import ptf -from ptf.ptfutils import chown_to_invoking_user +from ptf.ptfutils import EventDescriptor, Timeout, chown_to_invoking_user INVOKING_UID = 1234 INVOKING_GID = 5678 @@ -163,3 +164,32 @@ def test_open_logfile_hands_the_log_to_the_invoking_user( assert logfile.exists() assert chown_calls == [(str(logfile), INVOKING_UID, INVOKING_GID)] + + +def test_event_descriptor_close_is_idempotent(): + descriptor = EventDescriptor() + descriptor.close() + descriptor.close() + + +@pytest.mark.skipif(not hasattr(signal, "SIGALRM"), reason="requires SIGALRM") +def test_timeout_restores_previous_signal_state(): + previous_handler = signal.getsignal(signal.SIGALRM) + previous_timer = signal.getitimer(signal.ITIMER_REAL) + + def handler(signum, frame): + pass + + try: + signal.signal(signal.SIGALRM, handler) + signal.setitimer(signal.ITIMER_REAL, 60) + with Timeout(1): + pass + remaining, interval = signal.getitimer(signal.ITIMER_REAL) + assert signal.getsignal(signal.SIGALRM) is handler + assert 55 < remaining <= 60 + assert interval == 0 + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + signal.setitimer(signal.ITIMER_REAL, *previous_timer) diff --git a/utests/tests/ptf/test_runner.py b/utests/tests/ptf/test_runner.py new file mode 100644 index 0000000..780e77e --- /dev/null +++ b/utests/tests/ptf/test_runner.py @@ -0,0 +1,550 @@ +# Copyright 2026 The P4 Language Consortium +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the ptf.runner library API. The API runs PTF tests inside +the current process, without the ptf binary.""" + +import subprocess +import sys +import io +import logging +import random +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from ptf import runner + +TESTDIR = "utests/specs" + + +def make_config(**kwargs): + log_file = kwargs.pop("log_file", "test_runner_ptf.log") + return runner.PtfConfig( + allow_user=True, + logging=runner.LoggingOptions(log_file=log_file), + platform=runner.PlatformOptions(platform="dummy"), + **kwargs, + ) + + +def parse_params(out): + params = {} + for line in out.splitlines(): + if not line.startswith(">>>"): + continue + line = line[3:] + if line == "None": + return None + k, v = line.split("=") + params[k] = int(v) + return params + + +def test_run_in_process_with_string_test_params(capsys): + config = make_config( + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["test.TestParamsGet"], + ), + test_behavior=runner.TestBehaviorOptions(test_params="k1=9;k2=18"), + ) + assert runner.run(config) == 0 + assert parse_params(capsys.readouterr().out) == {"k1": 9, "k2": 18} + + +def test_run_in_process_with_dict_test_params(capsys): + config = make_config( + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["test.TestParamGet"], + ), + test_behavior=runner.TestBehaviorOptions(test_params={"k1": 42}), + ) + assert runner.run(config) == 0 + assert parse_params(capsys.readouterr().out) == {"k1": 42} + + +def test_run_in_process_with_no_test_params(capsys): + config = make_config( + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["test.TestParamsGet"], + ), + ) + assert runner.run(config) == 0 + assert parse_params(capsys.readouterr().out) is None + + +def test_run_in_process_with_flat_dict_config(capsys): + # run() also accepts a dictionary in the flat ptf.config format. + config = { + "test_dir": TESTDIR, + "test_specs": ["test.TestParamGet"], + "platform": "dummy", + "allow_user": True, + "log_file": "test_runner_ptf.log", + "test_params": "k1=7", + } + assert runner.run(config) == 0 + assert parse_params(capsys.readouterr().out) == {"k1": 7} + + +def test_run_in_process_list_tests(capsys): + config = make_config( + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + assert runner.run(config) == 0 + out = capsys.readouterr().out + assert "Module test:" in out + assert "TestParamsGet" in out + assert "modules shown with a total of" in out + + +def test_run_in_process_list_test_names(capsys): + config = make_config( + list_test_names=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + assert runner.run(config) == 0 + out = capsys.readouterr().out + assert "test.TestParamGet" in out + assert "fixtures.ModuleFixtureProbeOne" in out + + +def test_run_in_process_unknown_test_spec(capsys): + config = make_config( + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["does-not-exist"], + ), + ) + with pytest.raises(runner.PtfError, match="did not match any tests"): + runner.run(config) + + +def test_run_in_process_invalid_test_dir(): + config = make_config( + test_selection=runner.TestSelectionOptions(test_dir="not-a-dir"), + ) + with pytest.raises(runner.PtfError, match="invalid test directory"): + runner.run(config) + + +def test_run_in_process_restores_logging_handlers(): + root = logging.getLogger() + saved_handlers = list(root.handlers) + saved_level = root.level + saved_disable = logging.root.manager.disable + config = make_config( + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["test.TestParamGet"], + ), + ) + assert runner.run(config) == 0 + assert list(root.handlers) == saved_handlers + assert root.level == saved_level + assert logging.root.manager.disable == saved_disable + + +def test_run_keeps_caller_file_handler_usable(tmp_path): + root = logging.getLogger() + logfile = tmp_path / "caller.log" + handler = logging.FileHandler(logfile, mode="w") + root.addHandler(handler) + try: + root.warning("before") + config = make_config( + log_file=str(tmp_path / "ptf.log"), + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + assert runner.run(config) == 0 + assert handler.stream is not None + root.warning("after") + handler.flush() + assert logfile.read_text().splitlines() == ["before", "after"] + finally: + root.removeHandler(handler) + handler.close() + + +def test_run_preserves_caller_handler_using_ptf_log_path(tmp_path): + root = logging.getLogger() + logfile = tmp_path / "shared.log" + handler = logging.FileHandler(logfile, mode="w") + root.addHandler(handler) + try: + root.warning("before") + config = make_config( + log_file=str(logfile), + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + assert runner.run(config) == 0 + root.warning("after") + handler.flush() + contents = logfile.read_text() + assert "before" in contents + assert "after" in contents + finally: + root.removeHandler(handler) + handler.close() + + +def test_run_forwards_logs_and_uses_supplied_streams(tmp_path): + stdout = io.StringIO() + stderr = io.StringIO() + records = [] + + class RecordHandler(logging.Handler): + def emit(self, record): + records.append(record) + + observer = logging.getLogger("ptf-test-observer") + saved_level = observer.level + saved_propagate = observer.propagate + observer.propagate = False + observer.setLevel(logging.DEBUG) + observer_handler = RecordHandler() + observer.addHandler(observer_handler) + try: + config = make_config( + log_file=str(tmp_path / "ptf.log"), + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["test.TestParamGet"], + ), + ) + assert ( + runner.run( + config, + output=runner.RunOutput(stdout=stdout, stderr=stderr, logger=observer), + ) + == 0 + ) + finally: + observer.removeHandler(observer_handler) + observer.setLevel(saved_level) + observer.propagate = saved_propagate + assert ">>>k1=-1" not in stdout.getvalue() # Test-owned stdout is not redirected. + assert "test.TestParamGet" in stderr.getvalue() + assert any("TEST RUN START" in record.getMessage() for record in records) + + +def test_run_rejects_recursive_output_logger(tmp_path): + config = make_config( + log_file=str(tmp_path / "ptf.log"), + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + with pytest.raises(ValueError, match="must not propagate"): + runner.run( + config, + output=runner.RunOutput(logger=logging.getLogger("ptf.consumer")), + ) + + +def test_run_can_disable_ptf_log_artifacts(tmp_path, monkeypatch): + test_dir = str(Path(TESTDIR).resolve()) + monkeypatch.chdir(tmp_path) + config = make_config( + log_file=None, + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=test_dir), + ) + + assert runner.run(config) == 0 + assert not list(tmp_path.glob("*.log")) + assert not list(tmp_path.glob("*.pcap")) + + +def test_run_restores_process_state(tmp_path, monkeypatch): + import ptf + + original_config = ptf.config + original_config.clear() + original_config["sentinel"] = {"value": 1} + original_path = sys.path + original_path_contents = list(sys.path) + random.seed(12345) + original_random_state = random.getstate() + + config = make_config( + log_file=str(tmp_path / "ptf.log"), + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + pypath=[str(tmp_path)], + ) + assert runner.run(config) == 0 + + assert ptf.config is original_config + assert ptf.config == {"sentinel": {"value": 1}} + assert sys.path is original_path + assert sys.path == original_path_contents + assert random.getstate() == original_random_state + + +def test_test_modules_are_fresh_for_each_run(tmp_path, capsys): + def run_with(value): + config = make_config( + log_file=str(tmp_path / "ptf-{}.log".format(value)), + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["import_state.ImportStateProbe"], + ), + test_behavior=runner.TestBehaviorOptions( + test_params={"import_value": value} + ), + ) + assert runner.run(config) == 0 + return capsys.readouterr().out + + assert ">>>import_value=1" in run_with(1) + assert ">>>import_value=2" in run_with(2) + + +def test_run_can_execute_on_a_worker_thread(tmp_path): + config = make_config( + log_file=str(tmp_path / "ptf.log"), + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + with ThreadPoolExecutor(max_workers=1) as pool: + assert pool.submit(runner.run, config).result() == 0 + + +def test_zero_timeout_can_execute_on_a_worker_thread(tmp_path): + config = make_config( + log_file=str(tmp_path / "ptf.log"), + list_tests=True, + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + test_behavior=runner.TestBehaviorOptions(test_case_timeout=0), + ) + with ThreadPoolExecutor(max_workers=1) as pool: + assert pool.submit(runner.run, config).result() == 0 + + +def test_decorated_timeout_is_rejected_on_a_worker_thread(tmp_path): + config = make_config( + log_file=str(tmp_path / "ptf.log"), + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["isolation.Timed"], + ), + ) + with ThreadPoolExecutor(max_workers=1) as pool: + with pytest.raises(runner.PtfError, match="main Python thread"): + pool.submit(runner.run, config).result() + + +def test_overlapping_in_process_runs_are_rejected(tmp_path, monkeypatch): + entered = threading.Event() + release = threading.Event() + + def blocking_execute(config, output, state): + entered.set() + release.wait(5) + return 0 + + monkeypatch.setattr(runner, "_execute", blocking_execute) + config = make_config( + log_file=str(tmp_path / "ptf.log"), + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(runner.run, config) + assert entered.wait(5) + with pytest.raises(runner.PtfError, match="another in-process PTF run"): + runner.run(config) + release.set() + assert future.result() == 0 + + +def test_cleanup_failure_changes_success_to_error(tmp_path, monkeypatch): + class BrokenDataplane: + def stop_pcap(self): + pass + + def kill(self): + raise RuntimeError("close failed") + + def execute(config, output, state): + state.dataplane = BrokenDataplane() + return 0 + + monkeypatch.setattr(runner, "_execute", execute) + config = make_config( + log_file=str(tmp_path / "ptf.log"), + test_selection=runner.TestSelectionOptions(test_dir=TESTDIR), + ) + with pytest.raises(runner.PtfError, match="cleanup failed"): + runner.run(config) + + +@pytest.mark.parametrize("fail_skipped, expected", [(False, 0), (True, 1)]) +def test_skips_use_unittest_result(tmp_path, fail_skipped, expected): + config = make_config( + log_file=str(tmp_path / "ptf.log"), + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["isolation.Skipped"], + ), + test_behavior=runner.TestBehaviorOptions(fail_skipped=fail_skipped), + ) + assert runner.run(config) == expected + + +def test_ptf_config_json_roundtrip(): + config = runner.PtfConfig( + allow_user=True, + pypath=["/some/path"], + test_selection=runner.TestSelectionOptions( + test_dir="tests", + test_specs=["test.Foo", "^group"], + test_order="rand", + test_order_seed=1234, + ), + platform=runner.PlatformOptions( + platform="nn", + device_sockets=[ + runner.DeviceSocket(0, {1, 2, 5, 6, 7, 8}, "ipc:///tmp/p.ipc") + ], + interfaces=[runner.Interface(0, 1, "eth1"), runner.Interface(1, 2, "eth2")], + port_info={1: {"mac": "aa:bb:cc:dd:ee:ff"}}, + ), + test_behavior=runner.TestBehaviorOptions( + test_params={"k1": "abc", "k2": 21}, + random_seed=42, + ), + ) + rebuilt = runner.PtfConfig.from_json(config.to_json()) + assert rebuilt == config + + +def test_ptf_config_from_dict_defaults(): + # A partial flat dictionary must produce the default configuration, + # plus the given keys. + rebuilt = runner.PtfConfig.from_dict({"test_dir": "tests"}) + assert rebuilt == runner.PtfConfig( + test_selection=runner.TestSelectionOptions(test_dir="tests") + ) + + +def test_ptf_config_to_dict_legacy_shape(): + config = runner.PtfConfig( + pypath=["/some/path"], + platform=runner.PlatformOptions( + interfaces=[runner.Interface(0, 1, "eth1")], + device_sockets=[runner.DeviceSocket(0, {1, 2}, "tcp://1.2.3.4:1")], + ), + test_behavior=runner.TestBehaviorOptions( + test_params={"k1": "abc"}, + ), + ) + d = config.to_dict() + # interfaces and device_sockets keep their legacy tuple/set shapes + assert d["interfaces"] == [(0, 1, "eth1")] + assert d["device_sockets"] == [(0, {1, 2}, "tcp://1.2.3.4:1")] + assert d["pypath"] == ["/some/path"] + # dict test params are rendered in the legacy string syntax + assert d["test_params"] == "k1='abc'" + assert d["port_map"] is None + assert d["packet_manipulation_module"] is None + + +def test_flat_config_preserves_platform_extensions(): + portclass = object() + config = runner.PtfConfig.from_dict( + {"test_dir": "tests", "dataplane": {"portclass": portclass}} + ) + assert config.extra_config == {"dataplane": {"portclass": portclass}} + assert config.to_dict()["dataplane"]["portclass"] is portclass + + +def test_json_rejects_non_json_platform_extensions(): + config = runner.PtfConfig(extra_config={"extension": {1, 2}}) + with pytest.raises(TypeError): + config.to_json() + + +def test_preimported_packet_backend_config_is_rejected(tmp_path): + script = """ +import ptf +import ptf.packet_scapy +from ptf import runner + +config = runner.PtfConfig( + allow_user=True, + list_tests=True, + logging=runner.LoggingOptions(log_file=None), + platform=runner.PlatformOptions(platform="dummy"), + test_selection=runner.TestSelectionOptions(test_dir="utests/specs"), + test_behavior=runner.TestBehaviorOptions(disable_ipv6=True), +) +try: + runner.run(config) +except runner.PtfError as error: + print(error) +else: + raise SystemExit("configuration was not rejected") +""" + result = subprocess.run( + [sys.executable, "-c", script], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + assert result.returncode == 0, result.stdout + assert "packet backend already loaded" in result.stdout + + +def test_python_dash_m_entry_point(tmp_path): + # python -m ptf must behave like the ptf binary. + r = subprocess.run( + [ + sys.executable, + "-m", + "ptf", + "--test-dir", + TESTDIR, + "--platform", + "dummy", + "--allow-user", + "--log-file", + str(tmp_path / "ptf.log"), + "test.TestParamGet", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + input=None, + universal_newlines=True, + ) + assert r.returncode == 0 + # No --test-params: test_param_get falls back to its default (-1). + assert ">>>k1=-1" in r.stdout + + +def test_serialized_config_stdin_entry_point(tmp_path): + config = make_config( + log_file=str(tmp_path / "ptf.log"), + test_selection=runner.TestSelectionOptions( + test_dir=TESTDIR, + test_specs=["test.TestParamGet"], + ), + ) + result = subprocess.run( + [sys.executable, "-m", "ptf.runner", "-"], + input=config.to_json(), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + assert result.returncode == 0, result.stdout + assert ">>>k1=-1" in result.stdout