From c29c1516e32d4e3c21dd9d795e3ac79f3c616727 Mon Sep 17 00:00:00 2001 From: tamaralyne <46680674+tamaralyne@users.noreply.github.com> Date: Thu, 16 Oct 2025 15:44:13 -0400 Subject: [PATCH] Revert "merge in release/0.9.4" --- .github/workflows/full.yml | 111 -- .github/workflows/pull_request.yml | 92 +- .github/workflows/push.yml | 7 +- .gitignore | 2 - .mypy-check-paths | 14 - .pre-commit-config.yaml | 2 +- CHANGELOG.rst | 66 - Makefile | 78 +- README.contribute.rst | 230 +-- README.report.rst | 18 + README.rst | 188 +- VERSION | 2 +- docs/_static/custom.css | 18 +- docs/arch.rst | 219 +- docs/asyncio.rst | 186 -- docs/conf.py | 11 +- docs/debug.rst | 158 +- docs/deprecated.rst | 3 - docs/dhcp-server-detector.rst | 137 -- docs/fixtures.rst | 4 - docs/index.rst | 18 +- docs/ipdb.rst | 6 + docs/ipdb_toc.rst | 13 + docs/iproute.rst | 56 +- docs/iproute_intro.rst | 91 - docs/iproute_linux.rst | 20 - docs/iproute_netns.rst | 85 - docs/iproute_platforms.rst.disabled | 20 - docs/iproute_responses.rst | 134 -- docs/iproute_tc.rst.disabled | 24 - docs/ndb_auth.rst | 34 + docs/ndb_probes.rst | 6 - docs/ndb_toc.rst | 2 +- docs/netns.rst | 16 +- docs/{nlsocket.rst.disabled => nlsocket.rst} | 0 docs/parser.rst | 88 +- docs/plan9.rst | 20 - docs/pyroute2-dhcp-client.rst | 118 +- docs/{remote.rst.disabled => remote.rst} | 0 docs/threading.rst | 72 - docs/usage.rst | 132 +- examples/devlink/devlink_monitor.py | 1 + examples/ethtool/ethtool-ioctl_get_infos.py | 4 +- examples/ethtool/ethtool_get_infos.py | 1 - examples/generic/Makefile | 1 + examples/generic/README.rst | 27 - examples/generic/netl.c | 105 +- examples/generic/netl.py | 38 +- examples/ipq.py | 5 +- examples/iproute/ip_monitor.py | 3 +- examples/iproute/socketcan.py | 4 +- examples/ipset.py | 3 +- examples/kobject_uevent.py | 1 - examples/lab/README.rst | 15 + examples/lab/iproute_get_addr/README.rst | 7 + examples/lab/iproute_get_addr/check.py | 11 + examples/lab/iproute_get_addr/setup.py | 4 + examples/lab/iproute_get_addr/task.py | 8 + examples/lab/iproute_get_attr/README.rst | 7 + examples/lab/iproute_get_attr/check.py | 9 + examples/lab/iproute_get_attr/setup.py | 4 + examples/lab/iproute_get_attr/task.py | 12 + examples/ndb/create_bond.py | 1 + examples/ndb/create_interface.py | 7 +- examples/ndb/create_vlan.py | 3 +- examples/ndb/keystone_auth.py | 112 ++ examples/ndb/radius_auth.py | 81 + examples/nftables_sets.py | 35 +- examples/policy/policy.py | 1 - examples/processes/pmonitor.py | 1 - examples/processes/taskstats.py | 2 - examples/pyroute2-cli/comments | 17 + examples/pyroute2-cli/create_bridge | 70 + examples/pyroute2-cli/create_dummy | 31 + examples/pyroute2-cli/dump_lo | 5 + examples/wifi/nl80211_interface_type.py | 4 +- examples/wifi/nl80211_monitor.py | 1 + examples/wifi/nl80211_scan_dump.py | 9 +- examples/wifi/nl80211_set_type.py | 5 +- lab/Makefile | 20 + .../__init__.py => lab/_static/classic.css | 0 lab/_static/custom.css | 349 ++++ lab/_static/fixup.js | 10 + lab/_static/lab.js | 139 ++ lab/_templates/conf.js | 6 + lab/_templates/form_template.html | 15 + lab/_templates/layout.html | 30 + lab/conf.py | 23 + lab/index.rst | 20 + lab/iproute.rst | 74 + noxfile.py | 252 +-- pr2modules/__init__.py | 24 + pyproject.minimal.toml | 107 - pyproject.toml | 86 - pyroute2/__init__.py | 152 +- pyroute2/cli/__init__.py | 148 ++ pyroute2/cli/auth/__init__.py | 0 pyroute2/cli/auth/auth_keystone.py | 33 + pyroute2/cli/auth/auth_radius.py | 28 + pyroute2/cli/console.py | 107 + pyroute2/cli/parser.py | 204 ++ pyroute2/cli/server.py | 108 + pyroute2/cli/session.py | 256 +++ pyroute2/common.py | 377 +++- pyroute2/compat.py | 19 - pyroute2/config/__init__.py | 61 +- pyroute2/config/asyncio.py | 4 - pyroute2/conntrack.py | 117 +- pyroute2/decoder/args.py | 26 - pyroute2/decoder/loader.py | 316 --- pyroute2/decoder/main.py | 189 -- pyroute2/devlink.py | 89 +- pyroute2/dhcp/__init__.py | 295 ++- pyroute2/dhcp/cli.py | 185 -- pyroute2/dhcp/client.py | 738 +------ pyroute2/dhcp/dhcp4msg.py | 124 +- pyroute2/dhcp/dhcp4socket.py | 240 +-- pyroute2/dhcp/enums/__init__.py | 1 - pyroute2/dhcp/enums/bootp.py | 21 - pyroute2/dhcp/enums/dhcp.py | 201 -- pyroute2/dhcp/fsm.py | 69 - pyroute2/dhcp/hooks.py | 201 -- pyroute2/dhcp/iface_status.py | 104 - pyroute2/dhcp/leases.py | 247 --- pyroute2/dhcp/messages.py | 180 -- pyroute2/dhcp/server_detector.py | 234 --- pyroute2/dhcp/timers.py | 73 - pyroute2/dhcp/xids.py | 85 - pyroute2/ethtool/ethtool.py | 35 - pyroute2/ethtool/ioctl.py | 42 +- pyroute2/ext/bpf.py | 89 - pyroute2/ext/rawsocket.py | 121 +- pyroute2/fixtures/__init__.py | 132 -- pyroute2/fixtures/doctest.py | 24 - pyroute2/fixtures/iproute/__init__.py | 402 ---- pyroute2/fixtures/ndb/__init__.py | 19 - pyroute2/fixtures/plan9/__init__.py | 48 - pyroute2/ipdb/__init__.py | 283 --- pyroute2/ipdb/exceptions.py | 18 + pyroute2/ipdb/interfaces.py | 1500 ++++++++++++++ pyroute2/ipdb/linkedset.py | 307 +++ pyroute2/ipdb/main.py | 1525 ++++++++++++++ pyroute2/ipdb/routes.py | 1323 +++++++++++++ pyroute2/ipdb/rules.py | 307 +++ pyroute2/ipdb/transactional.py | 530 +++++ pyroute2/ipdb/utils.py | 9 + pyroute2/iproute/__init__.py | 169 +- pyroute2/iproute/bsd.py | 17 +- pyroute2/iproute/darwin.py | 478 ----- pyroute2/iproute/ipmock.py | 620 ++---- pyroute2/iproute/linux.py | 1628 ++++++--------- pyroute2/iproute/parsers.py | 54 +- pyroute2/iproute/windows.py | 12 +- pyroute2/ipset.py | 4 +- pyroute2/ipvs.py | 204 -- pyroute2/iwutil.py | 374 ++-- pyroute2/lab.py | 30 + pyroute2/loader.py | 43 + pyroute2/minimal.py | 124 +- pyroute2/ndb/cli.py | 86 + pyroute2/ndb/cluster.py | 39 + pyroute2/ndb/main.py | 178 +- pyroute2/ndb/noipdb.py | 198 ++ pyroute2/ndb/objects/__init__.py | 320 ++- pyroute2/ndb/objects/address.py | 93 +- pyroute2/ndb/objects/interface.py | 413 ++-- pyroute2/ndb/objects/neighbour.py | 47 +- pyroute2/ndb/objects/netns.py | 18 +- pyroute2/ndb/objects/probe.py | 82 - pyroute2/ndb/objects/route.py | 280 ++- pyroute2/ndb/objects/rule.py | 6 +- pyroute2/ndb/query.py | 170 ++ pyroute2/ndb/report.py | 131 +- pyroute2/ndb/schema.py | 348 ++-- pyroute2/ndb/source.py | 283 ++- pyroute2/ndb/sync_api.py | 206 -- pyroute2/ndb/task_manager.py | 340 ++-- pyroute2/ndb/transport.py | 210 ++ pyroute2/ndb/view.py | 107 +- pyroute2/netlink/__init__.py | 240 +-- pyroute2/netlink/connector/cn_proc.py | 13 +- pyroute2/netlink/core.py | 843 -------- pyroute2/netlink/coredata.py | 54 - pyroute2/netlink/devlink/__init__.py | 20 +- pyroute2/netlink/diag/__init__.py | 2 +- pyroute2/netlink/event/__init__.py | 46 +- pyroute2/netlink/event/acpi_event.py | 12 +- pyroute2/netlink/event/dquot.py | 10 +- pyroute2/netlink/event/thermal.py | 8 +- pyroute2/netlink/generic/__init__.py | 70 +- pyroute2/netlink/generic/ethtool.py | 100 +- pyroute2/netlink/generic/ipvs.py | 133 -- pyroute2/netlink/generic/l2tp.py | 180 +- pyroute2/netlink/generic/mptcp.py | 39 +- pyroute2/netlink/generic/wireguard.py | 99 +- pyroute2/netlink/marshal.py | 144 -- pyroute2/netlink/nfnetlink/ipset.py | 2 +- pyroute2/netlink/nfnetlink/nfctsocket.py | 168 +- pyroute2/netlink/nfnetlink/nftsocket.py | 250 +-- pyroute2/netlink/nl80211/__init__.py | 39 +- pyroute2/netlink/nlsocket.py | 1764 +++++++++++------ pyroute2/netlink/proxy.py | 76 + pyroute2/netlink/rt_files.py | 176 -- pyroute2/netlink/rtnl/__init__.py | 7 - pyroute2/netlink/rtnl/ifaddrmsg.py | 3 - pyroute2/netlink/rtnl/ifinfmsg/__init__.py | 12 +- pyroute2/netlink/rtnl/ifinfmsg/compat.py | 382 ++++ pyroute2/netlink/rtnl/ifinfmsg/proxy.py | 123 ++ pyroute2/netlink/rtnl/ifinfmsg/sync.py | 70 + pyroute2/netlink/rtnl/ifinfmsg/tuntap.py | 14 +- pyroute2/netlink/rtnl/iprsocket.py | 508 ++--- pyroute2/netlink/rtnl/marshal.py | 2 - pyroute2/netlink/rtnl/probe_msg.py | 133 -- pyroute2/netlink/rtnl/riprsocket.py | 16 + pyroute2/netlink/rtnl/rtmsg.py | 10 - pyroute2/netlink/rtnl/tcmsg/act_bpf.py | 5 - pyroute2/netlink/rtnl/tcmsg/cls_basic.py | 10 +- pyroute2/netlink/rtnl/tcmsg/cls_flow.py | 10 +- pyroute2/netlink/rtnl/tcmsg/cls_fw.py | 10 +- pyroute2/netlink/rtnl/tcmsg/cls_matchall.py | 10 +- pyroute2/netlink/rtnl/tcmsg/cls_u32.py | 8 +- pyroute2/netlink/rtnl/tcmsg/common_ematch.py | 6 +- pyroute2/netlink/rtnl/tcmsg/sched_bpf.py | 12 +- pyroute2/netlink/rtnl/tcmsg/sched_cake.py | 8 +- pyroute2/netlink/rtnl/tcmsg/sched_clsact.py | 6 +- pyroute2/netlink/rtnl/tcmsg/sched_htb.py | 30 +- pyroute2/netlink/rtnl/tcmsg/sched_ingress.py | 6 +- pyroute2/netlink/rtnl/tcmsg/sched_tbf.py | 5 - pyroute2/netlink/rtnl/tcmsg/sched_template.py | 4 +- pyroute2/netlink/taskstats/__init__.py | 264 +-- pyroute2/netlink/uevent/__init__.py | 76 +- pyroute2/netns/__init__.py | 139 +- pyroute2/nftables/expressions.py | 16 +- pyroute2/nftables/main.py | 85 +- pyroute2/nslink/nslink.py | 231 +++ pyroute2/plan9/__init__.py | 430 ---- pyroute2/plan9/client.py | 204 -- pyroute2/plan9/filesystem.py | 236 --- pyroute2/plan9/ipc.py | 51 - pyroute2/plan9/plan9socket.py | 9 - pyroute2/plan9/server.py | 471 ----- pyroute2/process.py | 245 --- pyroute2/protocols/__init__.py | 125 +- pyroute2/remote/__init__.py | 8 + pyroute2/remote/__main__.py | 5 + pyroute2/remote/iproute.py | 149 ++ pyroute2/remote/shell.py | 72 + pyroute2/remote/transport.py | 369 ++++ pyroute2/requests/address.py | 21 +- pyroute2/requests/bridge.py | 37 +- pyroute2/requests/common.py | 13 +- pyroute2/requests/link.py | 21 +- pyroute2/requests/main.py | 128 +- pyroute2/requests/neighbour.py | 11 +- pyroute2/requests/probe.py | 23 - pyroute2/requests/route.py | 85 +- pyroute2/requests/rule.py | 12 +- pyroute2/requests/tc.py | 76 - pyroute2/statsd.py | 63 - pyroute2/wiset.py | 4 +- requirements.dev.txt | 8 + requirements.docs.txt | 7 + requirements.repo.txt | 4 + setup.cfg | 43 + setup.minimal.cfg | 63 + setup.py | 3 + stubs/.gitkeep | 0 tests/README.md | 44 + tests/decoder/README.md | 64 + tests/decoder/decoder.py | 41 + tests/net_tools.py | 194 -- tests/test_ci/conftest.py | 37 - tests/test_ci/test_fixtures.py | 60 - tests/test_core/conftest.py | 5 - tests/test_core/test_check_tid.py | 33 - tests/test_core/test_event_loop.py | 196 -- tests/test_core/test_ndb/test_repl.py | 193 -- tests/test_core/test_ndb/test_sync_api.py | 77 - tests/test_core/test_plan9/test_basic.py | 31 - tests/test_core/test_socket_create.py | 23 - tests/test_core/test_statsd.py | 109 - tests/test_decoder/nl0.json | 1591 --------------- tests/test_decoder/nl0.pcap | Bin 20884 -> 0 bytes tests/test_decoder/test_pcap.py | 51 - tests/test_integration/conftest.py | 1 - tests/test_integration/test_kuryr.py | 91 +- tests/test_integration/test_lnst.py | 140 +- tests/test_integration/test_octavia.py | 98 - tests/test_integration/test_os_vif.py | 42 - tests/test_integration/test_ovn_bgp_agent.py | 30 - tests/test_lab/conftest.py | 1 + tests/test_lab/test_code_blocks.py | 26 + tests/test_limits/conftest.py | 1 - tests/test_limits/test_nl.py | 85 +- tests/test_limits/test_stress.py | 100 +- tests/test_linux/conftest.py | 22 +- .../fixtures/dhcp_servers/__init__.py | 162 -- .../fixtures/dhcp_servers/dnsmasq.py | 79 - .../test_linux/fixtures/dhcp_servers/mock.py | 61 - .../fixtures/dhcp_servers/udhcpd.py | 101 - tests/test_linux/fixtures/interfaces.py | 85 - tests/test_linux/fixtures/pcap_files.py | 79 - tests/test_linux/fixtures/rt_file.py | 15 - tests/test_linux/pr2test/context_manager.py | 15 +- tests/test_linux/pr2test/tools.py | 3 +- tests/test_linux/test_devlink.py | 29 +- .../test_android_reboot_request.pcap | Bin 386 -> 0 bytes .../test_android_tethering_renew.pcap | Bin 754 -> 0 bytes .../test_decode_simple_lease_process.pcap | Bin 2200 -> 0 bytes .../test_huawei_discover_option_148.pcap | Bin 833 -> 0 bytes .../test_invalid_client_id_option.pcap | Bin 382 -> 0 bytes .../test_invalid_router_option.pcap | Bin 395 -> 0 bytes .../test_netatmo_discover_request.pcap | Bin 1122 -> 0 bytes .../test_parser/test_truncated_packet.pcap | Bin 630 -> 0 bytes .../test_washing_machine_request.pcap | Bin 390 -> 0 bytes .../test_parser/test_wii_discover.pcap | Bin 382 -> 0 bytes .../test_ack_invalid_request_state.pcap | Bin 740 -> 0 bytes .../test_unit/test_get_and_renew_lease.pcap | Bin 1098 -> 0 bytes .../test_unit/test_init_reboot_nak.pcap | Bin 1098 -> 0 bytes .../test_unit/test_offer_wrong_xid.pcap | 1 - .../test_unit/test_requesting_timeout.pcap | Bin 382 -> 0 bytes .../test_unit/test_truncated_packet.pcap | Bin 406 -> 0 bytes .../test_unexpected_dhcp_message.pcap | Bin 382 -> 0 bytes .../test_unit/test_unknown_message.pcap | Bin 414 -> 0 bytes tests/test_linux/test_dhcp/conftest.py | 100 - tests/test_linux/test_dhcp/test_cli.py | 248 --- tests/test_linux/test_dhcp/test_encode.py | 42 - tests/test_linux/test_dhcp/test_hooks.py | 199 -- .../test_linux/test_dhcp/test_integration.py | 391 ---- tests/test_linux/test_dhcp/test_parser.py | 438 ---- .../test_dhcp/test_server_detector.py | 269 --- tests/test_linux/test_dhcp/test_unit.py | 437 ---- tests/test_linux/test_diag.py | 13 +- tests/test_linux/test_dquot/test_dquot.py | 10 + tests/test_linux/test_ethtool.py | 4 +- tests/test_linux/test_generic/test_l2tp.py | 11 +- tests/test_linux/test_ipdb.py | 88 + tests/test_linux/test_ipr/test_basic.py | 12 +- tests/test_linux/test_ipr/test_link.py | 18 +- tests/test_linux/test_ipr/test_match.py | 2 +- tests/test_linux/test_ipr/test_netns.py | 36 - tests/test_linux/test_ipr/test_probe.py | 77 - tests/test_linux/test_ipr/test_route.py | 20 - tests/test_linux/test_ipr/test_rule.py | 219 +- tests/test_linux/test_ipr/test_vlan.py | 2 +- tests/test_linux/test_ipvs.py | 57 - tests/test_linux/test_iwutil.py | 52 +- tests/test_linux/test_ndb/test_altnames.py | 1 - tests/test_linux/test_ndb/test_backup.py | 9 +- tests/test_linux/test_ndb/test_examples.py | 15 + tests/test_linux/test_ndb/test_netns.py | 42 +- tests/test_linux/test_ndb/test_probe.py | 57 - tests/test_linux/test_ndb/test_routes.py | 4 +- tests/test_linux/test_ndb/test_sources.py | 7 +- tests/test_linux/test_raw/test_dhcp.py | 68 + tests/test_linux/test_remote.py | 14 + tests/test_linux/test_tc/test_basic.py | 16 - tests/test_linux/test_tc/test_bpf.py | 12 +- tests/test_linux/test_tc/test_classful.py | 4 +- tests/test_linux/test_tc/test_htb.py | 68 +- tests/test_minimal/conftest.py | 28 - .../test_minimal/test_ipr/test_addr_async.py | 53 - tests/test_minimal/test_ipr/test_addr_sync.py | 50 - .../test_ipr/test_ensure_async.py | 93 - .../test_minimal/test_ipr/test_ensure_sync.py | 83 - .../test_minimal/test_ipr/test_link_async.py | 41 - tests/test_minimal/test_ipr/test_link_sync.py | 33 - .../test_minimal/test_ipr/test_route_async.py | 40 - .../test_ipr/test_route_dump_async.py | 112 -- .../test_ipr/test_route_dump_sync.py | 106 - .../test_minimal/test_ipr/test_route_sync.py | 32 - .../test_minimal/test_ipr/test_rule_async.py | 57 - tests/test_minimal/test_ipr/test_rule_sync.py | 47 - tests/test_minimal/test_ipr/test_tc_async.py | 258 --- tests/test_minimal/test_ipr/test_tc_sync.py | 237 --- tests/test_minimal/test_iproute.py | 54 + .../test_nfctsocket/test_basic_async.py | 80 - .../test_nfctsocket/test_basic_sync.py | 75 - .../test_nfctsocket/test_conntrack_async.py | 148 -- .../test_nfctsocket/test_conntrack_sync.py | 144 -- .../test_nftables/test_expressions.py | 70 - .../test_nftables/test_nftsocket.py | 37 - tests/test_neutron/test_ip_lib.py | 12 +- tests/test_process/test_basic.py | 81 - tests/test_process/test_catastrophe.py | 49 - tests/test_repo/test_noxfile.py | 30 +- tests/test_repo/test_version.py | 13 +- tests/test_unit/test_addr_pool.py | 35 + tests/test_unit/test_common.py | 66 +- .../test_iproute_match/test_match.py | 6 +- tests/test_unit/test_nlmsg/test_marshal.py | 3 - .../test_nlmsg/uevent_kernel_backlight.dump | 45 - .../test_nlmsg/uevent_udevd_backlight.dump | 28 - tests/test_unit/test_requests/common.py | 2 +- tests/test_unit/test_requests/test_address.py | 133 -- tests/test_unit/test_requests/test_link.py | 33 - .../test_unit/test_requests/test_neighbour.py | 14 +- tests/utils.py | 6 + util/make_lab_templates.py | 39 + 399 files changed, 17321 insertions(+), 25504 deletions(-) delete mode 100644 .github/workflows/full.yml delete mode 100644 .mypy-check-paths delete mode 100644 docs/asyncio.rst delete mode 100644 docs/deprecated.rst delete mode 100644 docs/dhcp-server-detector.rst delete mode 100644 docs/fixtures.rst create mode 100644 docs/ipdb.rst create mode 100644 docs/ipdb_toc.rst delete mode 100644 docs/iproute_intro.rst delete mode 100644 docs/iproute_linux.rst delete mode 100644 docs/iproute_netns.rst delete mode 100644 docs/iproute_platforms.rst.disabled delete mode 100644 docs/iproute_responses.rst delete mode 100644 docs/iproute_tc.rst.disabled create mode 100644 docs/ndb_auth.rst delete mode 100644 docs/ndb_probes.rst rename docs/{nlsocket.rst.disabled => nlsocket.rst} (100%) delete mode 100644 docs/plan9.rst rename docs/{remote.rst.disabled => remote.rst} (100%) delete mode 100644 docs/threading.rst delete mode 100644 examples/generic/README.rst create mode 100644 examples/lab/README.rst create mode 100644 examples/lab/iproute_get_addr/README.rst create mode 100644 examples/lab/iproute_get_addr/check.py create mode 100644 examples/lab/iproute_get_addr/setup.py create mode 100644 examples/lab/iproute_get_addr/task.py create mode 100644 examples/lab/iproute_get_attr/README.rst create mode 100644 examples/lab/iproute_get_attr/check.py create mode 100644 examples/lab/iproute_get_attr/setup.py create mode 100644 examples/lab/iproute_get_attr/task.py create mode 100644 examples/ndb/keystone_auth.py create mode 100644 examples/ndb/radius_auth.py create mode 100644 examples/pyroute2-cli/comments create mode 100644 examples/pyroute2-cli/create_bridge create mode 100644 examples/pyroute2-cli/create_dummy create mode 100644 examples/pyroute2-cli/dump_lo create mode 100644 lab/Makefile rename pyroute2/decoder/__init__.py => lab/_static/classic.css (100%) create mode 100644 lab/_static/custom.css create mode 100644 lab/_static/fixup.js create mode 100644 lab/_static/lab.js create mode 100644 lab/_templates/conf.js create mode 100644 lab/_templates/form_template.html create mode 100644 lab/_templates/layout.html create mode 100644 lab/conf.py create mode 100644 lab/index.rst create mode 100644 lab/iproute.rst create mode 100644 pr2modules/__init__.py delete mode 100644 pyproject.minimal.toml create mode 100644 pyroute2/cli/__init__.py create mode 100644 pyroute2/cli/auth/__init__.py create mode 100644 pyroute2/cli/auth/auth_keystone.py create mode 100644 pyroute2/cli/auth/auth_radius.py create mode 100644 pyroute2/cli/console.py create mode 100644 pyroute2/cli/parser.py create mode 100644 pyroute2/cli/server.py create mode 100644 pyroute2/cli/session.py delete mode 100644 pyroute2/compat.py delete mode 100644 pyroute2/decoder/args.py delete mode 100644 pyroute2/decoder/loader.py delete mode 100644 pyroute2/decoder/main.py delete mode 100644 pyroute2/dhcp/cli.py delete mode 100644 pyroute2/dhcp/enums/__init__.py delete mode 100644 pyroute2/dhcp/enums/bootp.py delete mode 100644 pyroute2/dhcp/enums/dhcp.py delete mode 100644 pyroute2/dhcp/fsm.py delete mode 100644 pyroute2/dhcp/hooks.py delete mode 100644 pyroute2/dhcp/iface_status.py delete mode 100644 pyroute2/dhcp/leases.py delete mode 100644 pyroute2/dhcp/messages.py delete mode 100644 pyroute2/dhcp/server_detector.py delete mode 100644 pyroute2/dhcp/timers.py delete mode 100644 pyroute2/dhcp/xids.py delete mode 100644 pyroute2/ext/bpf.py delete mode 100644 pyroute2/fixtures/__init__.py delete mode 100644 pyroute2/fixtures/doctest.py delete mode 100644 pyroute2/fixtures/iproute/__init__.py delete mode 100644 pyroute2/fixtures/ndb/__init__.py delete mode 100644 pyroute2/fixtures/plan9/__init__.py create mode 100644 pyroute2/ipdb/exceptions.py create mode 100644 pyroute2/ipdb/interfaces.py create mode 100644 pyroute2/ipdb/linkedset.py create mode 100644 pyroute2/ipdb/main.py create mode 100644 pyroute2/ipdb/routes.py create mode 100644 pyroute2/ipdb/rules.py create mode 100644 pyroute2/ipdb/transactional.py create mode 100644 pyroute2/ipdb/utils.py delete mode 100644 pyroute2/iproute/darwin.py delete mode 100644 pyroute2/ipvs.py create mode 100644 pyroute2/lab.py create mode 100644 pyroute2/loader.py create mode 100755 pyroute2/ndb/cli.py create mode 100644 pyroute2/ndb/cluster.py create mode 100644 pyroute2/ndb/noipdb.py delete mode 100644 pyroute2/ndb/objects/probe.py create mode 100644 pyroute2/ndb/query.py delete mode 100644 pyroute2/ndb/sync_api.py create mode 100644 pyroute2/ndb/transport.py delete mode 100644 pyroute2/netlink/core.py delete mode 100644 pyroute2/netlink/coredata.py delete mode 100644 pyroute2/netlink/generic/ipvs.py delete mode 100644 pyroute2/netlink/marshal.py create mode 100644 pyroute2/netlink/proxy.py delete mode 100644 pyroute2/netlink/rt_files.py create mode 100644 pyroute2/netlink/rtnl/ifinfmsg/compat.py create mode 100644 pyroute2/netlink/rtnl/ifinfmsg/proxy.py create mode 100644 pyroute2/netlink/rtnl/ifinfmsg/sync.py delete mode 100644 pyroute2/netlink/rtnl/probe_msg.py create mode 100644 pyroute2/netlink/rtnl/riprsocket.py create mode 100644 pyroute2/nslink/nslink.py delete mode 100644 pyroute2/plan9/__init__.py delete mode 100644 pyroute2/plan9/client.py delete mode 100644 pyroute2/plan9/filesystem.py delete mode 100644 pyroute2/plan9/ipc.py delete mode 100644 pyroute2/plan9/plan9socket.py delete mode 100644 pyroute2/plan9/server.py delete mode 100644 pyroute2/process.py create mode 100644 pyroute2/remote/__init__.py create mode 100644 pyroute2/remote/__main__.py create mode 100644 pyroute2/remote/iproute.py create mode 100644 pyroute2/remote/shell.py create mode 100644 pyroute2/remote/transport.py delete mode 100644 pyroute2/requests/probe.py delete mode 100644 pyroute2/requests/tc.py delete mode 100644 pyroute2/statsd.py create mode 100644 requirements.dev.txt create mode 100644 requirements.docs.txt create mode 100644 requirements.repo.txt create mode 100644 setup.cfg create mode 100644 setup.minimal.cfg create mode 100644 setup.py create mode 100644 stubs/.gitkeep create mode 100644 tests/README.md create mode 100644 tests/decoder/README.md create mode 100644 tests/decoder/decoder.py delete mode 100644 tests/net_tools.py delete mode 100644 tests/test_ci/conftest.py delete mode 100644 tests/test_ci/test_fixtures.py delete mode 100644 tests/test_core/conftest.py delete mode 100644 tests/test_core/test_check_tid.py delete mode 100644 tests/test_core/test_event_loop.py delete mode 100644 tests/test_core/test_ndb/test_repl.py delete mode 100644 tests/test_core/test_ndb/test_sync_api.py delete mode 100644 tests/test_core/test_plan9/test_basic.py delete mode 100644 tests/test_core/test_socket_create.py delete mode 100644 tests/test_core/test_statsd.py delete mode 100644 tests/test_decoder/nl0.json delete mode 100644 tests/test_decoder/nl0.pcap delete mode 100644 tests/test_decoder/test_pcap.py delete mode 100644 tests/test_integration/conftest.py delete mode 100644 tests/test_integration/test_octavia.py delete mode 100644 tests/test_integration/test_os_vif.py delete mode 100644 tests/test_integration/test_ovn_bgp_agent.py create mode 100644 tests/test_lab/conftest.py create mode 100644 tests/test_lab/test_code_blocks.py delete mode 100644 tests/test_limits/conftest.py delete mode 100644 tests/test_linux/fixtures/dhcp_servers/__init__.py delete mode 100644 tests/test_linux/fixtures/dhcp_servers/dnsmasq.py delete mode 100644 tests/test_linux/fixtures/dhcp_servers/mock.py delete mode 100644 tests/test_linux/fixtures/dhcp_servers/udhcpd.py delete mode 100644 tests/test_linux/fixtures/interfaces.py delete mode 100644 tests/test_linux/fixtures/pcap_files.py delete mode 100644 tests/test_linux/fixtures/rt_file.py delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_android_reboot_request.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_android_tethering_renew.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_decode_simple_lease_process.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_huawei_discover_option_148.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_invalid_client_id_option.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_invalid_router_option.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_netatmo_discover_request.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_truncated_packet.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_washing_machine_request.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_parser/test_wii_discover.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_unit/test_ack_invalid_request_state.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_unit/test_get_and_renew_lease.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_unit/test_init_reboot_nak.pcap delete mode 120000 tests/test_linux/test_dhcp/captures/test_unit/test_offer_wrong_xid.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_unit/test_requesting_timeout.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_unit/test_truncated_packet.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_unit/test_unexpected_dhcp_message.pcap delete mode 100644 tests/test_linux/test_dhcp/captures/test_unit/test_unknown_message.pcap delete mode 100644 tests/test_linux/test_dhcp/conftest.py delete mode 100644 tests/test_linux/test_dhcp/test_cli.py delete mode 100644 tests/test_linux/test_dhcp/test_encode.py delete mode 100644 tests/test_linux/test_dhcp/test_hooks.py delete mode 100644 tests/test_linux/test_dhcp/test_integration.py delete mode 100644 tests/test_linux/test_dhcp/test_parser.py delete mode 100644 tests/test_linux/test_dhcp/test_server_detector.py delete mode 100644 tests/test_linux/test_dhcp/test_unit.py create mode 100644 tests/test_linux/test_ipdb.py delete mode 100644 tests/test_linux/test_ipr/test_probe.py delete mode 100644 tests/test_linux/test_ipvs.py delete mode 100644 tests/test_linux/test_ndb/test_probe.py create mode 100644 tests/test_linux/test_raw/test_dhcp.py create mode 100644 tests/test_linux/test_remote.py delete mode 100644 tests/test_minimal/conftest.py delete mode 100644 tests/test_minimal/test_ipr/test_addr_async.py delete mode 100644 tests/test_minimal/test_ipr/test_addr_sync.py delete mode 100644 tests/test_minimal/test_ipr/test_ensure_async.py delete mode 100644 tests/test_minimal/test_ipr/test_ensure_sync.py delete mode 100644 tests/test_minimal/test_ipr/test_link_async.py delete mode 100644 tests/test_minimal/test_ipr/test_link_sync.py delete mode 100644 tests/test_minimal/test_ipr/test_route_async.py delete mode 100644 tests/test_minimal/test_ipr/test_route_dump_async.py delete mode 100644 tests/test_minimal/test_ipr/test_route_dump_sync.py delete mode 100644 tests/test_minimal/test_ipr/test_route_sync.py delete mode 100644 tests/test_minimal/test_ipr/test_rule_async.py delete mode 100644 tests/test_minimal/test_ipr/test_rule_sync.py delete mode 100644 tests/test_minimal/test_ipr/test_tc_async.py delete mode 100644 tests/test_minimal/test_ipr/test_tc_sync.py create mode 100644 tests/test_minimal/test_iproute.py delete mode 100644 tests/test_minimal/test_nfctsocket/test_basic_async.py delete mode 100644 tests/test_minimal/test_nfctsocket/test_basic_sync.py delete mode 100644 tests/test_minimal/test_nfctsocket/test_conntrack_async.py delete mode 100644 tests/test_minimal/test_nfctsocket/test_conntrack_sync.py delete mode 100644 tests/test_minimal/test_nftables/test_expressions.py delete mode 100644 tests/test_minimal/test_nftables/test_nftsocket.py delete mode 100644 tests/test_process/test_basic.py delete mode 100644 tests/test_process/test_catastrophe.py delete mode 100644 tests/test_unit/test_nlmsg/uevent_kernel_backlight.dump delete mode 100644 tests/test_unit/test_nlmsg/uevent_udevd_backlight.dump create mode 100755 util/make_lab_templates.py diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml deleted file mode 100644 index d02ba613f..000000000 --- a/.github/workflows/full.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Main CI full - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - linter: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.14] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=linter-${{ matrix.python }} - repo: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=repo - unit: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=unit - docs: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=docs - neutron: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=neutron - integration: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=integration - process: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=process - limits: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=limits - minimal: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=minimal - ci-self: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.10, python3.11, python3.12, python3.13, python3.14] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=ci-self-${{ matrix.python }} - fedora: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.10, python3.11, python3.12, python3.13, python3.14] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=linux-${{ matrix.python }} - ubuntu: - runs-on: Ubuntu - strategy: - matrix: - python: [python3.10, python3.11, python3.12] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=linux-${{ matrix.python }} - fedora-core: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.10, python3.11, python3.12, python3.13, python3.14] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=core-${{ matrix.python }} - ubuntu-core: - runs-on: Ubuntu - strategy: - matrix: - python: [python3.10, python3.11, python3.12] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=core-${{ matrix.python }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 10dc179b4..77a03694d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1,10 +1,8 @@ -name: Main CI short +name: Main CI on: pull_request: - branches: - - master - - 'release/**' + branches: [ master ] workflow_dispatch: permissions: @@ -12,16 +10,13 @@ permissions: jobs: linter: - runs-on: Fedora - strategy: - matrix: - python: [python3.14] + runs-on: code steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - - run: make nox session=linter-${{ matrix.python }} + - run: make nox session=linter repo: - runs-on: Linux + runs-on: Python3.8 steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 @@ -32,6 +27,12 @@ jobs: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - run: make nox session=unit + lab: + runs-on: Linux + steps: + - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE + - uses: actions/checkout@v4 + - run: make nox session=lab docs: runs-on: Linux steps: @@ -39,10 +40,7 @@ jobs: - uses: actions/checkout@v4 - run: make nox session=docs neutron: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.13] + runs-on: Python3.8 steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 @@ -52,70 +50,28 @@ jobs: steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - - run: sudo make nox session=integration - process: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=process - limits: - runs-on: Linux - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=limits + - run: make nox session=integration minimal: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.14] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: sudo make nox session=minimal-${{ matrix.python }} - ci-self: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.14] - steps: - - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v4 - - run: make nox session=ci-self-${{ matrix.python }} - fedora: - runs-on: Fedora - strategy: - matrix: - python: [python3.10, python3.14] + runs-on: Linux steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - - run: sudo make nox session=linux-${{ matrix.python }} - ubuntu: - runs-on: Ubuntu - strategy: - matrix: - python: [python3.10, python3.12] + - run: sudo make nox session=minimal + linux-fedora-38: + runs-on: linux-fedora-38 steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - - run: sudo make nox session=linux-${{ matrix.python }} - fedora-core: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.14] + - run: sudo make nox session=linux + linux-ubuntu-22: + runs-on: linux-ubuntu-22 steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - - run: sudo make nox session=core-${{ matrix.python }} - ubuntu-core: - runs-on: Ubuntu - strategy: - matrix: - python: [python3.10, python3.12] + - run: sudo make nox session=linux + openbsd: + runs-on: OpenBSD steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - - run: sudo make nox session=core-${{ matrix.python }} + - run: /home/github/openbsd_runner.sh diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 97248ab0f..ba0551bd7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -10,11 +10,8 @@ permissions: jobs: build: - runs-on: Fedora - strategy: - matrix: - python: [python3.9, python3.14] + runs-on: code steps: - run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v4 - - run: make nox session=linter-${{ matrix.python }} + - run: make nox session=linter diff --git a/.gitignore b/.gitignore index 28061e33a..256312a05 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,3 @@ benchmark.log venv .venv .nox* -tests/*.db -tests/*.json diff --git a/.mypy-check-paths b/.mypy-check-paths deleted file mode 100644 index f2c4842e2..000000000 --- a/.mypy-check-paths +++ /dev/null @@ -1,14 +0,0 @@ -pyroute2/common.py -pyroute2/fixtures -pyroute2/netlink/core.py -pyroute2/netlink/coredata.py -pyroute2/netlink/nlsocket.py -pyroute2/netlink/uevent/__init__.py -pyroute2/netlink/rtnl/iprsocket.py -pyroute2/netlink/rtnl/ifinfmsg/tuntap.py -pyroute2/netlink/rt_files.py -pyroute2/netns/ -pyroute2/plan9/client.py -pyroute2/process.py -pyroute2/statsd.py -pyroute2/ndb/sync_api.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90abbf24b..014400a68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -files: '^(noxfile.py|pyroute2|pr2modules|util|examples|docs/conf.py|examples/pyroute2-cli|tests)' +files: '^(noxfile.py|pyroute2|pr2modules|util|examlpes/ndb|docs/conf.py|examples/pyroute2-cli|tests)' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ae41cbd66..6d54094fb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,72 +1,6 @@ Changelog ========= -* 0.9.4 - * iwutil: fix get_interfaces_dict -* 0.9.3 - * packaging: use only pyproject files - * packaging: update pyroute2-minimal modules list - * fixtures: split out ndb fixture - * ndb: use AsyncIPRoute as RTNL source - * ndb: deprecate and remove NDB cli - * async versions for: - * acpi events - * conntrack - * devlink - * diag - * dquot - * ethtool - * event - * generic - * ipvs - * iwutil - * l2tp - * mptcp - * nl80211 - * taskstats - * thermal events - * wireguard - * iproute: rt_files management - * examples: update kernel module example -* 0.9.2 - * iproute: basic Darwin support - * nftables: asynchronous API - * plan9: Inode class context manager API - * plan9: support call on read - * code: provide again .epid -* 0.9.1 - * nftables: fix comment size -* 0.9.1rc2 - * nftables: fix work with the new core, add basic masq and comment support - * nlsocket: fix batch requests - * dhcp client: add InterfaceNotFound exception - * ethtool: use byref instead of cpointer -* 0.9.1rc1 - * core: rewritten using asyncio, and the async API is now first class - * core: provide compatibility synchronous API classes - * telemetry: a basic internal implementation using statsd protocol - * dhcp: a huge upgrade, all the protocol module reimplemented using asyncio - * 9p: a basic client / server / fs implementation (no 9p2000.L yet) - * fixtures: added public pytest fixtures - * NetNS: obsoleted, all the functionality merged into the core - * proxy: deprecated and removed, see NetlinkRequest - * netns: provide unified socket creation function, netns.create_socket() - * process: separate class to manage external processes - -.. warning:: - - 0.8.1 is the final minor thread-based version of the library. All updates to - the synchronous core will follow as 0.8.x. The asynchronous core will follow - as 0.9.x and up. - -* 0.8.1 - * ndb: fix route replace - * ndb: fix af_bridge_fdb indexes - * rtnl: probe API - * tc: htb rate64 - * ipvs: basic support - * decoder: support pcap format - * ethtool: get/set channels * 0.7.12 * cli: move readline import to the script * ci: update tools versions diff --git a/Makefile b/Makefile index 64b7f8698..b1ff640ab 100644 --- a/Makefile +++ b/Makefile @@ -5,22 +5,14 @@ # python ?= $(shell util/find_python.sh) platform := $(shell uname -s) -releaseTag ?= $(shell git describe --tags --abbrev=0) -releaseDescription := $(shell git tag -l -n1 ${releaseTag} | sed 's/[0-9. ]\+//') -noxboot ?= ~/.venv-boot define nox {\ which nox 2>/dev/null || {\ - test -d ${noxboot} && \ - {\ - . ${noxboot}/bin/activate;\ - } || {\ - ${python} -m venv ${noxboot};\ - . ${noxboot}/bin/activate;\ - pip install --upgrade pip;\ - pip install nox;\ - };\ + python -m venv ~/.venv-boot/;\ + . ~/.venv-boot/bin/activate;\ + pip install --upgrade pip;\ + pip install nox;\ };\ nox $(1) -- '${noxconfig}';\ } @@ -35,21 +27,27 @@ all: @echo \* install -- install lib into the system @echo -.PHONY: git-clean -git-clean: - git clean -d -f -x - git remote prune origin - git branch --merged | grep -vE '(^\*| master )' >/tmp/merged-branches && \ - ( xargs git branch -d /dev/null | awk -f util/imports_dict.awk + for module in $(call list_modules); do \ + echo $$module; \ + findimports -n $$module/pr2modules/ 2>/dev/null | awk -f util/imports_dict.awk | awk '{printf("\t"$$0"\n")}'; \ + done + +# deprecated: +epydoc clean-version update-version force-version README.md setup.ini develop: + @echo Deprecated target, see README.make.md diff --git a/README.contribute.rst b/README.contribute.rst index 17aab6ff4..17885e69f 100644 --- a/README.contribute.rst +++ b/README.contribute.rst @@ -1,199 +1,91 @@ .. devcontribute: -Project Contribution Guide +Project contribution guide ========================== -Setup the Environment ---------------------- +Step 1: setup the environment +----------------------------- -.. code-block:: bash +Linux ++++++ - # Make sure you have the following installed: - # bash - # git - # python - # GNU make, sed, awk +.. code-block:: sh - # Clone the repository - git clone ${pyroute2_git_url} - cd pyroute2 + # make sure you have installed: + # bash + # git + # python + # GNU make, sed, awk - # Run the test suite - sudo make test + # clone the repo + git clone ${pyroute2_git_url} + cd pyroute2 + # run the test suite + make test -Test-Driven Development ------------------------ - -It is best practice that every change is covered by tests. To make -things easier, write the tests first and then implement your feature. - -The test modules are located in the `/tests/` folder and are run -using nox. You can add your tests to an existing test module or create -your own module if it requires a specific environment that is not yet -covered. In that case, add a new session to `noxfile.py`. - -The project’s convention is to use pytest, but it is not a strict -requirement. If testing your feature requires a specific framework or -uses the standard library’s `unittest` module, it can be discussed -in the pull request. - -However, tests must always be run using a nox session. - - -Code Linting ------------- - -The code must be formatted according to the project’s code style. -The simplest way is to run `make format`, which will automatically -reformat the code. - -.. note:: - The code must pass `make format` in order to be merged. - -In addition to code style checks, additional linting is performed. -It is best practice to add type annotations and ensure that the -modified code passes mypy linting. If you do so, add the affected -file or module to `.mypy-check-paths` so that subsequent changes -will not undo your work. - - -Run Specific Test Modules --------------------------- - -It is possible to run a specific test session instead of running -all default sessions: - -.. code-block:: bash - - make test session=unit # run only tests/test_unit - make test session=core-python3.14 # run tests/test_core with Python 3.14 - -See `noxfile.py` for the defined sessions and the Python version matrix. - -Each test session is run in a separate Python process within its own -virtual environment, while test modules within the same session share -the process and virtual environment. +OpenBSD ++++++++ -Thus, testing integration with libraries such as eventlet, or testing -multiprocessing start methods (spawn, fork, etc.), requires a separate -session since such process setup can only be performed once. +.. code-block:: sh -To run a specific test module within a session, use the `noxconfig` argument: + # install required tools + pkg_add bash git gmake gsed python rust -.. code-block:: bash + # clone the repo + git clone ${pyroute_git_url} + cd pyroute2 - # Run tests/test_linux/test_ndb/test_mpls.py - make test session=linux-python3.14 \ - noxconfig='{"sub": "test_ndb/test_mpls.py"}' + # run the test suite + gmake test +Step 2: plan and implement the change +------------------------------------- -Test Session Configuration ---------------------------- +The best practice is that any change should be covered by tests. +The test suite is in the `/tests/` folder and is run by `nox`. You +can add your tests to an existing tests module, or create your +own module, if it requires some specific environment that is not +covered yet. In the latter case add a new session to `noxfile.py`. -Test session configuration can be provided via the `noxconfig` argument -in JSON format. Available options: +The project is designed to work on the bare standard library. +But some embedded environments strip even the stdlib, removing +modules like sqlite3. -**coverage** (bool, default: false) - Generate a coverage report. +So to run pyroute2 even in such environments, the project provides +two packages, `pyroute2` and `pyroute2.minimal`, with the latter +providing a minimal distribution, with no sqlite3 or pickle. -**exitfirst** (bool, default: true) - Exit the session after the first failure (true) or run all tests and - report failures at the end (false). +Modules `pyroute2` and `pyroute2.minimal` are mutually exclusive. -**fail_on_warnings** (bool, default: false) - Fail on Python warnings, such as DeprecationWarning. +Each module provides it's own pypi package. +More details: https://github.com/svinota/pyroute2/discussions/786 -**fast** (bool, default: false) - Reuse an existing virtual environment for the session (true) or purge - the virtual environment and create a new one from scratch (false). - This option might be useful when running tests offline, but it may - leave artifacts in the source tree. It is disabled by default and - requires at least one run with `"fast": false` to properly set up - the virtual environment. Use with caution. - -**pdb** (bool, default: false) - Launch the pdb debugger shell on errors. - -**sub** (str, default: none) - Select the module to run within the session. - -**test_prefix** (str, default: "tests") - Override the test sessions directory. - -**timeout** (int, default: 60) - Test case timeout in seconds. - -**verbose** (bool, default: true) - If true, print every test case name and description (if available); - if false, print only a dot for each test case. - -**Example:** - -.. code-block:: bash - - make test \ - session=core-python3.14 \ - noxconfig='{ \ - "exitfirst": false, \ - "verbose": false, \ - "timeout": 10, \ - "pdb": true, \ - "sub": "test_plan9/test_basic.py" \ - }' - - -Running with nox ----------------- - -The `make test` command creates a virtual environment and installs nox -within it. However, you can also use your system’s installation of nox. -The previous example would then look as follows: - -.. code-block:: bash - - nox \ - -e core-python3.14 -- \ - '{ \ - "exitfirst": false, \ - "verbose": false, \ - "timeout": 10, \ - "pdb": true, \ - "sub": "test_plan9/test_basic.py" \ - }' - - -Project Dependencies --------------------- - -The project is designed to work with only the standard library, which -is a strict requirement. Some embedded environments even strip the -standard library, removing modules such as sqlite3. +Step 3: test the change +----------------------- -To support such environments, the project provides two packages: `pyroute2` -and `pyroute2.minimal`. The latter offers a minimal distribution without -modules such as sqlite3 or pickle. +Assume the environment is already set up on the step 1: -The `pyroute2` and `pyroute2.minimal` packages are mutually exclusive. -Each provides its own PyPI package. +.. code-block:: sh + # run code linter + make format -Submit a Pull Request ---------------------- + # run test suite, some tests may require root + make test -The primary repository for the project is on GitHub. All pull requests -are welcome. +Step 4: submit a PR +------------------- -The code must meet the following requirements: +The primary repo for the project is on Github. All the PRs +are more than welcome there. -- The library must work on Python ≥ 3.9. -- The code must pass `make format`. -- The code must not break existing unit and functional tests (run via `sudo make test`). -- The use of ctypes must not cause the library to fail on SELinux. +Requirements to a PR +++++++++++++++++++++ +The code must comply some requirements: -IRC Channel ------------ -The project has an IRC channel, **#pyroute2**, on the Libera.Chat network. -While the team is not guaranteed to be online 24/7, you can still discuss -your questions there. +* the library **must** work on Python >= 3.9 +* the code **must** pass `make format` +* the code **must** not break existing unit and functional tests (`make test`) +* the `ctypes` usage **must not** break the library on SELinux diff --git a/README.report.rst b/README.report.rst index 0bfc44f30..a0368364f 100644 --- a/README.report.rst +++ b/README.report.rst @@ -10,3 +10,21 @@ with your report: * Linux kernel version * Python version * Specific environment, if used -- gevent, eventlet etc. + +The project provides a script to print the system summary: + +.. code-block:: sh + + pyroute2-test-platform + +Please keep in mind, that this command will try to create +and delete different interface types, and this requires +root access. + +It is possible also to run the test in your code: + +.. code-block:: python + + from pprint import pprint + from pyroute2.config.test_platform import TestCapsRtnl + pprint(TestCapsRtnl().collect()) diff --git a/README.rst b/README.rst index 065d76239..495cf0f81 100644 --- a/README.rst +++ b/README.rst @@ -1,16 +1,10 @@ Pyroute2 ======== -Pyroute2 is a pure Python networking framework. The core requires only Python +Pyroute2 is a pure Python **netlink** library. The core requires only Python stdlib, no 3rd party libraries. The library was started as an RTNL protocol -implementation, so the name is **pyroute2**, but now it supports several -protocols, including non-netlink. Here are some supported netlink families -and protocols: - -* **dhcp** --- dynamic host configuration protocol for IPv4 -* **9p2000** --- Plan9 file system protocol - -Netlink: +implementation, so the name is **pyroute2**, but now it supports many netlink +protocols. Some supported netlink families and protocols: * **rtnl**, network settings --- addresses, routes, traffic controls * **nfnetlink** --- netfilter API @@ -51,22 +45,104 @@ and standard system tools. Other platforms are not supported. -IPRoute -- synchronous RTNL API -------------------------------- +NDB -- high level RTNL API +-------------------------- + +Key features: + +* Data integrity +* Transactions with commit/rollback changes +* State synchronization +* Multiple sources, including netns and remote systems + +A "Hello world" example: + +.. code-block:: python + + from pyroute2 import NDB + + with NDB() as ndb: + with ndb.interfaces['eth0'] as eth0: + # set one parameter + eth0.set(state='down') + eth0.commit() # make sure that the interface is down + # or multiple parameters at once + eth0.set(ifname='hello_world!', state='up') + eth0.commit() # rename, bring up and wait for success + # --> <-- here you can be sure that the interface is up & renamed + +More examples: + +.. code-block:: python + + from pyroute2 import NDB + + ndb = NDB(log='debug') + + for record in ndb.interfaces.summary(): + print(record.ifname, record.address, record.state) + + if_dump = ndb.interfaces.dump() + if_dump.select_records(state='up') + if_dump.select_fields('index', 'ifname', 'kind') + for line in if_dump.format('json'): + print(line) + + addr_summary = ndb.addresses.summary() + addr_summary.select_records(ifname='eth0') + for line in addr_summary.format('csv'): + print(line) + + with ndb.interfaces.create(ifname='br0', kind='bridge') as br0: + br0.add_port('eth0') + br0.add_port('eth1') + br0.add_ip('10.0.0.1/24') + br0.add_ip('192.168.0.1/24') + br0.set( + br_stp_state=1, # set STP on + br_group_fwd_mask=0x4000, # set LLDP forwarding + state='up', # bring the interface up + ) + # --> <-- commit() will be run by the context manager + + # operate on netns: + ndb.sources.add(netns='testns') # connect to a namespace + + with ( + ndb.interfaces.create( + ifname='veth0', # create veth + kind='veth', + peer={ + 'ifname': 'eth0', # setup peer + 'net_ns_fd': 'testns', # in a namespace + }, + state='up', + ) + ) as veth0: + veth0.add_ip(address='172.16.230.1', prefixlen=24) + + with ndb.interfaces.wait( + target='testns', ifname='eth0' + ) as peer: # wait for the peer + peer.set(state='up') # bring it up + peer.add_ip('172.16.230.2/24') # add address + +IPRoute -- Low level RTNL API +----------------------------- -Low-level **IPRoute** utility --- Linux network configuration, this -class is almost a 1-to-1 RTNL mapping. There are no implicit +Low-level **IPRoute** utility --- Linux network configuration. +The **IPRoute** class is a 1-to-1 RTNL mapping. There are no implicit interface lookups and so on. -Get notifications about network settings changes: +Get notifications about network settings changes with IPRoute: .. code-block:: python from pyroute2 import IPRoute - with IPRoute() as ipr: - ipr.bind() # <--- start listening for RTNL broadcasts - for message in ipr.get(): # receive the broadcasts + # With IPRoute objects you have to call bind() manually + ipr.bind() + for message in ipr.get(): print(message) More examples: @@ -99,45 +175,10 @@ More examples: # release Netlink socket ip.close() -AsyncIPRoute -- asynchronous RTNL API -------------------------------------- - -While `IPRoute` provides a synchronous RTNL API, it is actually build -around the asyncio-based core. - -The same example as above can look like that: - -.. code-block:: python - - import asyncio - - from pyroute2 import AsyncIPRoute - - async def main(): - # get access to the netlink socket - ipr = AsyncIPRoute() - - # print interfaces - async for link in await ipr.get_links(): - print(link) - - # create VETH pair and move v0p1 to netns 'test' - await ipr.link('add', ifname='v0p0', peer='v0p1', kind='veth') - - # wait for the devices: - peer, veth = await ipr.poll( - ipr.link, 'dump', timeout=5, ifname=lambda x: x in ('v0p0', 'v0p1') - ) - await ipr.link('set', index=peer['index'], net_ns_fd='test') - ... - ipr.close() - - asyncio.run(main()) - -Please notice that `.close()` is synchronous in any case. +Network namespace examples +-------------------------- -Network namespace management ----------------------------- +Network namespace manipulation: .. code-block:: python @@ -170,40 +211,15 @@ List interfaces in some **netns**: .. code-block:: python - from pyroute2 import IPRoute + from pyroute2 import NetNS + from pprint import pprint - with IPRoute(netns='test') as ipr: - for link in ipr.get_links(): - print(link) + ns = NetNS('test') + pprint(ns.get_links()) + ns.close() More details and samples see in the documentation. -NDB -- high level RTNL API --------------------------- - -Key features: - -* Data integrity -* Transactions with commit/rollback changes -* State synchronization -* Multiple sources, including netns and remote systems - -A "Hello world" example: - -.. code-block:: python - - from pyroute2 import NDB - - with NDB() as ndb: - with ndb.interfaces['eth0'] as eth0: - # set one parameter - eth0.set(state='down') - eth0.commit() # make sure that the interface is down - # or multiple parameters at once - eth0.set(ifname='hello_world!', state='up') - eth0.commit() # rename, bring up and wait for success - # --> <-- here you can be sure that the interface is up & renamed - Installation ------------ @@ -230,7 +246,7 @@ Using source, requires make and nox Requirements ------------ -Python >= 3.9 +Python >= 3.6 Links ----- diff --git a/VERSION b/VERSION index a602fc9e2..88a7b2285 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.4 +0.7.12 diff --git a/docs/_static/custom.css b/docs/_static/custom.css index dbc7a6765..ff77f9a5f 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -232,20 +232,15 @@ pre { color: #333333; line-height: 120%; border: 1px solid #ac9; + border-left: none; + border-right: none; border-radius: 10px; } -div.highlight-python > div.highlight > pre { - padding-bottom: 2em; -} - div.highlight-none > div.highlight > pre { - background-color: #fffeed; -} - -div.highlight-python + div.highlight-none { - margin-top: -2em; - padding-left: 1em; + border: none; + margin-top: -1.5em; + padding-top: 1.5em; } tt { @@ -283,9 +278,6 @@ div.viewcode-block:target { dl.method > dt { margin-bottom: 1em; padding: 1em; - border-left: solid 1px #a00000; - border-bottom: solid 1px #a00000; - border-radius: 10px; } dl.class > dt { diff --git a/docs/arch.rst b/docs/arch.rst index f89a064fb..387983a1f 100644 --- a/docs/arch.rst +++ b/docs/arch.rst @@ -9,77 +9,124 @@ Sockets The idea behind the pyroute2 framework is pretty simple. The library provides socket objects, that have: -* shortcuts to establish higher level connections (netlink, 9p, ...) -* extra methods to run queries and protocols - -The library core is based on asyncio, and provides both asynchronous -and synchronous versions of API, where the synchronous one is a -wrapper around the async code. This way we avoid keeping two separate -codebase while providing the legacy sync API. - -The library provides asynchronous sockets: - -.. inheritance-diagram:: pyroute2.dhcp.dhcp4socket.AsyncDHCP4Socket - pyroute2.AsyncAcpiEventSocket - pyroute2.AsyncConntrack - pyroute2.AsyncDL - pyroute2.AsyncDQuotSocket - pyroute2.AsyncDevlinkSocket - pyroute2.AsyncEventSocket - pyroute2.AsyncGenericNetlinkSocket - pyroute2.AsyncIPRSocket - pyroute2.AsyncIPRoute - pyroute2.AsyncIPVSSocket - pyroute2.AsyncIW - pyroute2.AsyncL2tp - pyroute2.AsyncMPTCP - pyroute2.AsyncNFCTSocket - pyroute2.AsyncNFTSocket - pyroute2.AsyncNL80211 - pyroute2.AsyncNlEthtool - pyroute2.AsyncTaskStats - pyroute2.AsyncThermalEventSocket - pyroute2.AsyncWireGuard - pyroute2.Plan9ClientSocket - pyroute2.Plan9ServerSocket - pyroute2.dhcp.dhcp4socket.AsyncDHCP4Socket - :parts: 1 +* shortcuts to establish netlink connections +* extra methods to run netlink queries +* some magic to handle packet bursts +* another magic to transparently mangle netlink messages + +In other sense any netlink socket is just an ordinary socket +with `fileno()`, `recv()`, `sendto()` etc. Of course, one +can use it in `poll()`. -And synchronous sockets: - -.. inheritance-diagram:: pyroute2.AcpiEventSocket - pyroute2.Conntrack - pyroute2.DL - pyroute2.DQuotSocket - pyroute2.DevlinkSocket - pyroute2.DiagSocket - pyroute2.EventSocket - pyroute2.GenericNetlinkSocket - pyroute2.IPBatch - pyroute2.IPQSocket - pyroute2.IPRSocket - pyroute2.IPRoute - pyroute2.IPSet - pyroute2.IPVS - pyroute2.IPVSSocket - pyroute2.IW - pyroute2.L2tp - pyroute2.MPTCP - pyroute2.NFCTSocket - pyroute2.NFTSocket - pyroute2.NL80211 - pyroute2.NetNS - pyroute2.NlEthtool - pyroute2.ProcEventSocket - pyroute2.RawIPRoute - pyroute2.TaskStats - pyroute2.ThermalEventSocket - pyroute2.UeventSocket - pyroute2.WireGuard +There is an inheritance diagram of Linux netlink sockets, provided +by the library: + +.. inheritance-diagram:: pyroute2.iproute.linux.IPRoute + pyroute2.iproute.linux.IPBatch + pyroute2.iproute.linux.RawIPRoute + pyroute2.iwutil.IW + pyroute2.ipset.IPSet + pyroute2.netlink.uevent.UeventSocket + pyroute2.netlink.taskstats.TaskStats + pyroute2.netlink.generic.wireguard.WireGuard + pyroute2.netlink.generic.ethtool.NlEthtool + pyroute2.netlink.ipq.IPQSocket + pyroute2.netlink.nfnetlink.nfctsocket.NFCTSocket + pyroute2.netlink.nfnetlink.nftsocket.NFTSocket + pyroute2.netlink.event.EventSocket + pyroute2.netlink.event.acpi_event.AcpiEventSocket + pyroute2.netlink.event.dquot.DQuotSocket + pyroute2.netlink.event.thermal.ThermalEventSocket + pyroute2.netlink.devlink.DevlinkSocket + pyroute2.netlink.diag.DiagSocket + pyroute2.remote.RemoteIPRoute + pyroute2.remote.transport.RemoteSocket + pyroute2.remote.shell.ShellIPR + pyroute2.nslink.nslink.NetNS :parts: 1 -Not all the synchronous sockets have got their asynchronous counterpart yet, -but this work is ongoing. +under the hood +-------------- + +Let's assume we use an `IPRoute` object to get the +interface list of the system:: + + from pyroute2 import IPRoute + ipr = IPRoute() + ipr.get_links() + ipr.close() + +The `get_links()` method is provided by the `IPRouteMixin` +class. It chooses the message to send (`ifinfmsg`), prepares +required fields and passes it to the next layer:: + + result.extend(self.nlm_request(msg, RTM_GETLINK, msg_flags)) + +The `nlm_request()` is a method of the `NetlinkSocketBase` class. +It wraps the pair request/response in one method. The request +is done via `put()`, response comes with `get()`. These +methods hide under the hood the asynchronous nature of the +netlink protocol, where the response can come whenever -- +the time and packet order are not guaranteed. But one can +use the `sequence_number` field of a netlink message to +match responses, and the pair `put()/get()` does it. + +cache thread +------------ + +Sometimes it is preferable to get incoming messages asap +and parse them only when there is time for that. For that +case the `NetlinkSocketBase` provides a possibility to start a +dedicated cache thread, that will collect and queue incoming +messages as they arrive. The thread doesn't affect the +socket behaviour: it will behave exactly in the same way, +the only difference is that `recv()` will return already +cached in the userspace message. To start the thread, +one should call `bind()` with `async_cache=True`:: + + ipr = IPRoute() + ipr.bind(async_cache=True) + ... # do some stuff + ipr.close() + +message mangling +---------------- + +An interesting feature of the `IPRSocketBase` is a netlink +proxy code, that allows to register callbacks for different +message types. The callback API is simple. The callback +must accept the message as a binary data, and must return +a dictionary with two keys, `verdict` and `data`. The +verdict can be: + + * for `sendto()`: `forward`, `return` or `error` + * for `recv()`: `forward` or `error` + +E.g.:: + + msg = ifinfmsg(data) + msg.decode() + ... # mangle msg + msg.reset() + msg.encode() + return {'verdict': 'forward', + 'data': msg.buf.getvalue()} + +The `error` verdict raises an exception from `data`. The +`forward` verdict causes the `data` to be passed. The +`return` verdict is valid only in `sendto()` callbacks and +means that the `data` should not be passed to the kernel, +but instead it must be returned to the user. + +This magic allows the library to transparently support +ovs, teamd, tuntap calls via netlink. The corresponding +callbacks transparently route the call to an external +utility or to `ioctl()` API. + +How to register callbacks, see `IPRSocketBase` init. +The `_sproxy` serves `sendto()` mangling, the `_rproxy` +serves the `recv()` mangling. Later this API can become +public. Netlink messages ================ @@ -164,6 +211,42 @@ kernel. The PF_ROUTE messages: pyroute2.bsd.pf_route.ifma_msg :parts: 1 +IPDB +==== + +The `IPDB` module implements high-level logic to manage +some of the system network settings. It is completely +agnostic to the netlink object's nature, the only requirement +is that the netlink transport must provide RTNL API. + +So, using proper mixin classes one can create a custom +RTNL-compatible transport. E.g., this way `IPDB` can work +over `NetNS` objects, providing the network management +within some network namespace — while itself it runs in the +main namespace. + +The `IPDB` architecture is not too complicated, but it +implements some useful transaction magic, see `commit()` +methods of the `Transactional` objects. + +.. inheritance-diagram:: pyroute2.ipdb.main.IPDB + pyroute2.ipdb.interfaces.Interface + pyroute2.ipdb.linkedset.LinkedSet + pyroute2.ipdb.linkedset.IPaddrSet + pyroute2.ipdb.routes.NextHopSet + pyroute2.ipdb.routes.Via + pyroute2.ipdb.routes.Encap + pyroute2.ipdb.routes.Metrics + pyroute2.ipdb.routes.BaseRoute + pyroute2.ipdb.routes.Route + pyroute2.ipdb.routes.MPLSRoute + pyroute2.ipdb.routes.RoutingTable + pyroute2.ipdb.routes.MPLSTable + pyroute2.ipdb.routes.RoutingTableSet + pyroute2.ipdb.rules.Rule + pyroute2.ipdb.rules.RulesDict + :parts: 1 + Internet protocols ================== diff --git a/docs/asyncio.rst b/docs/asyncio.rst deleted file mode 100644 index 0b74c38bf..000000000 --- a/docs/asyncio.rst +++ /dev/null @@ -1,186 +0,0 @@ -.. _asyncio: - -Library core -============ - -Intro ------ - -Starting from version 0.9.1, pyroute2 is built on an asynchronous core. -This decision was driven not only by long-standing user requests but -also by years of challenges in refactoring the synchronous core. - -The complexity lies in the netlink protocol itself. Packets arriving -through the socket can be unordered, and multi-packet responses to -different requests may overlap. In addition, the socket also receives -broadcast netlink packets from the kernel as well as broadcast responses -initiated by other netlink users. - -The old core was designed to meet the following requirements: - -* The core must be thread-safe. -* No designated management thread; the current reader thread should - buffer extra packets and pass the buffer to the next reader upon - exit. -* No implicit background threads; threads should only be started - upon explicit user request. - -While this approach was fast enough, it resulted in a custom event loop -implementation with multiple overlapping locks, making the core code -extremely difficult to maintain. - -The current core is built on top of `asyncio`: - -* The netlink socket is managed by `asyncio`. -* Raw data reception methods are no longer available to the user. -* All synchronous APIs are now wrappers around the asynchronous API. - -As a result, the asynchronous API has become a first-class citizen in -the project, and the code required to reassemble netlink responses has -been reduced by 80%. - -AsyncCoreSocket ---------------- - -.. aafig:: - :scale: 80 - :textual: - :rounded: - - \ - +------------------+ | - | socket +---+ | - +------------------+ | +-------------------+ | - +---+ asyncio transport | | - +------------------+ | +---------+---------+ | - | asyncio protocol +---+ | | - +------------------+ +---------+---------+ | - | packets queue | \ class AsyncCoreSocket - +---------+---------+ / - +------------------+ | | - | msg reassemble +<----------------+ | - +--------+---------+ | - | | - | ... async get() | - | | - v | - / - -Important `AsyncCoreSocket` components: - -* `AsyncCoreSocket.socket` -- thread-local socket-like object managed - by `.endpoint` -* `AsyncCoreSocket.transport` -- thread-local `asyncio.Transport` -* `AsyncCoreSocket.protocol` -- thread-local `asyncio.Protocol` -* `AsyncCoreSocket.msg_queue` -- thread-local `asyncio` queue for data - received from the socket -* `AsyncCoreSocket.enqueue()` -- a synchronous routine to enqueue - packets into `.msg_queue`, used by the `transport` in the `protocol` -* `AsyncCoreSocket.get()` -- an asynchronous routine for retrieving - packets from the queue and reassembling responses -* `AsyncCoreSocket.marshal` -- a protocol-specific marshal for parsing - binary data into netlink messages - -.. testcode:: - :hide: - - import asyncio - import inspect - - from pyroute2 import IPRoute - from pyroute2.netlink.core import CoreMessageQueue - from pyroute2.netlink.marshal import Marshal - - with IPRoute() as ipr: - # AsyncCoreSocket.socket, compatibility, management - assert callable(ipr.asyncore.socket.recv) - assert callable(ipr.asyncore.socket.send) - assert callable(ipr.asyncore.socket.recvmsg) - assert callable(ipr.asyncore.socket.sendmsg) - assert callable(ipr.asyncore.socket.bind) - assert ipr.asyncore.transport._sock == ipr.asyncore.socket - - # AsyncCoreSocket.endpoint - assert isinstance(ipr.asyncore.transport, asyncio.Transport) - assert isinstance(ipr.asyncore.protocol, asyncio.Protocol) - - # msg_queue - assert isinstance(ipr.asyncore.msg_queue, CoreMessageQueue) - - # enqueue() - e_flags = ipr.asyncore.enqueue.__code__.co_flags - assert callable(ipr.asyncore.enqueue) - assert not e_flags & inspect.CO_ASYNC_GENERATOR - - # get() - g_flags = ipr.asyncore.get.__code__.co_flags - assert callable(ipr.asyncore.get) - assert g_flags & inspect.CO_ASYNC_GENERATOR - - # marshal - assert isinstance(ipr.asyncore.marshal, Marshal) - - -Synchronous code ----------------- - -`CoreSocket` is the synchronous version of `AsyncCoreSocket` implemented -using wrappers. Since it is merely a wrapper, it also operates on the -`asyncio` event loop. - -.. testcode:: - :hide: - - from pyroute2.netlink.core import AsyncCoreSocket, CoreSocket - - with CoreSocket() as cs: - assert isinstance(cs.asyncore, AsyncCoreSocket) - assert not isinstance(cs, AsyncCoreSocket) - assert not issubclass(CoreSocket, AsyncCoreSocket) - -`CoreSocket`, as well as other synchronous API classes, uses composition -instead of inheritance. The asynchronous API is available then as -`.asyncore` property. - -.. aafig:: - :scale: 80 - :textual: - :rounded: - - \ - +------------------+ | - | AsyncCoreSocket + | - +--------+---------+ | - | \ class CoreSocket - v / - +--------+---------+ | - | SyncAPI | | - +--------+---------+ | - | / - v - - -An example of a synchronous wrapper method: - -.. - The working directory to build the docs is - {git_root}/.nox-{user}/{nox_target}/tmp/{docs_sources} - - In order to include sources from the git, one should - step back all the way until {git_root}, thus - ../../../../ - -.. literalinclude:: ../../../../pyroute2/netlink/nlsocket.py - :caption: pyroute2.netlink.nlsocket: class NetlinkSocket - :pyobject: NetlinkSocket.get - :linenos: - :lineno-match: - -Synchronous APIs are provided for backward compatibility, and will remain -a part of the library. - -All synchronous components are built either on top of `CoreSocket`, -such as `GenericNetlinkSocket`, or using custom wrappers, like in -`IPRoute`. The plan is to refactor all components to provide an asynchronous -API, keeping the synchronous API for compatibility with existing projects -that use pyroute2. diff --git a/docs/conf.py b/docs/conf.py index 8acad8814..f325b6c98 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,7 @@ source_suffix = '.rst' master_doc = 'index' project = u'pyroute2' -copyright = u'pyroute2 team' +copyright = u'Peter Saveliev and PyRoute2 team' release = pyroute2.__version__ @@ -42,14 +42,7 @@ 'pyroute2-dhcp-client', 'pyroute2-dhcp-client', 'pyroute2 dhcp client', - ['Étienne Noss', 'Peter Saveliev'], - 1, - ), - ( - 'dhcp-server-detector', - 'dhcp-server-detector', - 'dhcp server detector', - ['Étienne Noss'], + ['Peter Saveliev'], 1, ), ] diff --git a/docs/debug.rst b/docs/debug.rst index c3254a1e2..756087174 100644 --- a/docs/debug.rst +++ b/docs/debug.rst @@ -1,15 +1,153 @@ .. debug: -Netlink debugging howto ------------------------ +Netlink debug howto +------------------- -pyroute2-decoder -================ +Dump data +========= -.. automodule:: pyroute2.decoder.main - :members: +Either run the required command via `strace`, or attach to the running +process with `strace -p`. Use `-s {int}` argument to make sure that all +the messages are dumped. The `-x` argument instructs `strace` to produce +output in the hex format that can be passed to the pyroute2 decoder:: -filter functions -================ -.. automodule:: pyroute2.decoder.loader - :members: MatchOps + $ strace -e trace=network -x -s 16384 ip ro + socket(PF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, NETLINK_ROUTE) = 3 + setsockopt(3, SOL_SOCKET, SO_SNDBUF, [32768], 4) = 0 + setsockopt(3, SOL_SOCKET, SO_RCVBUF, [1048576], 4) = 0 + bind(3, {sa_family=AF_NETLINK, pid=0, groups=00000000}, 12) = 0 + getsockname(3, {sa_family=AF_NETLINK, pid=28616, groups=00000000}, [12]) = 0 + sendto(3, "\x28\x00\x00\x00\x1a\x00\x01\x03 [skip] ", 40, 0, NULL, 0) = 40 + recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, + msg_iov(1)=[{"\x3c\x00\x00\x00\x18 [skip]", 16384}], + msg_controllen=0, msg_flags=0}, 0) = 480 + socket(PF_LOCAL, SOCK_DGRAM|SOCK_CLOEXEC, 0) = 4 + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 + recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, + msg_iov(1)=[{"\x14\x00\x00\x00\x03 [skip]", 16384}], + msg_controllen=0, msg_flags=0}, 0) = 20 + +++ exited with 0 +++ + +Now you can copy `send…()` and `recv…()` buffer strings to a file. + +Strace compatibility note +========================= + +Starting with version 4.13, `strace` parses Netlink message headers and +displays them in their parsed form instead of displaying the whole buffer in +its raw form. The rest of the buffer is still shown, but due to it being +incomplete, the method mentioned above doesn't work anymore. + +For the time being, the easiest workaround is probably to use an older strace +version as it only depends on libc6. + +Decode data +=========== + +The decoder is not provided with rpm or pip packages, so you should +have a local git repo of the project:: + + $ git clone + $ cd pyroute2 + +Now run the decoder:: + + $ export PYTHONPATH=`pwd` + $ python tests/decoder/decoder.py + +E.g. for the route dump in the file `rt.dump` the command line +should be:: + + $ python tests/decoder/decoder.py \ + pyroute2.netlink.rtnl.rtmsg.rtmsg \ + rt.dump + +**Why should I specify the message class?** Why there is no marshalling +in the decoder script? 'Cause it is intended to be used with different +netlink protocols, not only RTNL, but also nl80211, nfnetlink etc. +There is no common marshalling for all the netlink protocols. + +**How to specify the message class?** All the netlink protocols are +defined under `pyroute2/netlink/`, e.g. `rtmsg` module is +`pyroute2/netlink/rtnl/rtmsg.py`. Thereafter you should specify the +class inside the module, since there can be several classes. In the +`rtmsg` case the line will be `pyroute.netlink.rtnl.rtmsg.rtmsg` or, +more friendly to the bash autocomplete, `pyroute2/netlink/rtnl/rtmsg.rtmsg`. +Notice, that the class you have to specify with dot anyways. + +**What is the data file format?** Rules are as follows: + +* The data dump should be in a hex format. Two possible variants are: + `\\x00\\x01\\x02\\x03` or `00:01:02:03`. +* There can be several packets in the same file. They should be of the + same type. +* Spaces and line ends are ignored, so you can format the dump as you + want. +* The `#` symbol starts a comment until the end of the line. +* The `#!` symbols start a comment until the end of the file. + +Example:: + + # ifinfmsg headers + # + # nlmsg header + \x84\x00\x00\x00 # length + \x10\x00 # type + \x05\x06 # flags + \x49\x61\x03\x55 # sequence number + \x00\x00\x00\x00 # pid + # RTNL header + \x00\x00 # ifi_family + \x00\x00 # ifi_type + \x00\x00\x00\x00 # ifi_index + \x00\x00\x00\x00 # ifi_flags + \x00\x00\x00\x00 # ifi_change + # ... + + +Compile data +============ + +Starting with 0.4.1, the library provides `BatchSocket` class, that +only compiles and collects requests instead of sending them to the +kernel. E.g., it is used by `IPBatch`, that combines `BatchSocket` +with `IPRouteMixin`, providing RTNL compiler:: + + $ python3 + Python 3.4.3 (default, Mar 31 2016, 20:42:37) + [GCC 5.3.1 20151207 (Red Hat 5.3.1-2)] on linux + Type "help", "copyright", "credits" or "license" for more information. + # import all the stuff + >>> from pyroute2 import IPBatch + >>> from pyroute2.common import hexdump + # create the compiler + >>> ipb = IPBatch() + # compile requests into one buffer + >>> ipb.link("add", index=550, kind="dummy", ifname="test") + >>> ipb.link("set", index=550, state="up") + >>> ipb.addr("add", index=550, address="10.0.0.2", mask=24) + # inspect the buffer + >>> hexdump(ipb.batch) + '3c:00:00:00:10:00:05:06:00:00:00:00:a2:7c:00:00:00:00:00:00: + 26:02:00:00:00:00:00:00:00:00:00:00:09:00:03:00:74:65:73:74: + 00:00:00:00:10:00:12:00:0a:00:01:00:64:75:6d:6d:79:00:00:00: + 20:00:00:00:13:00:05:06:00:00:00:00:a2:7c:00:00:00:00:00:00: + 26:02:00:00:01:00:00:00:01:00:00:00:28:00:00:00:14:00:05:06: + 00:00:00:00:a2:7c:00:00:02:18:00:00:26:02:00:00:08:00:01:00: + 0a:00:00:02:08:00:02:00:0a:00:00:02' + # reset the buffer + >>> ipb.reset() + +Pls notice, that in Python2 you should use `hexdump(str(ipb.batch))` +instead of `hexdump(ipb.batch)`. + +The data, compiled by `IPBatch` can be used either to run batch +requests, when one `send()` call sends several messages at once, or +to produce binary buffers to test your own netlink parsers. Or just +to dump some data to be sent later and probably even on another host:: + + >>> ipr = IPRoute() + >>> ipr.sendto(ipb.batch, (0, 0)) + +The compiler always produces requests with `sequence_number == 0`, +so if there will be any responses, they can be handled as broadcasts. diff --git a/docs/deprecated.rst b/docs/deprecated.rst deleted file mode 100644 index dce49d3e2..000000000 --- a/docs/deprecated.rst +++ /dev/null @@ -1,3 +0,0 @@ -.. _deprecated: - -.. automodule:: pyroute2.ipdb diff --git a/docs/dhcp-server-detector.rst b/docs/dhcp-server-detector.rst deleted file mode 100644 index e4f2b7dbc..000000000 --- a/docs/dhcp-server-detector.rst +++ /dev/null @@ -1,137 +0,0 @@ -dhcp-server-detector -==================== - -Synopsis --------- - - **dhcp-server-detector [options] interface [interface ...]** - -Description ------------ - -**dhcp-server-detector** is a DHCP server detection tool based on pyroute2. - -It can be used to determine quickly if one or more DHCP server(s) are -responding on one or more network interfaces. -For example, it can be used to detect suspected "rogue" servers, or simply to -probe servers and obtain information about the offered options. - -**dhcp-server-detector** sends `DISCOVER` messages on the specified -interface(s) and prints eventual matching `OFFER`:s, and some metadata, -as JSON. - -Detection behavior (duration, intervals, ...) can be controlled with a few -options. - -Available options ------------------ - ---duration , -d - Number of seconds spent collecting responses, defaults to 30. - ---interval , -i - Number of seconds between each `DISCOVER` message, per interface. - Defaults to 4. - ---source-port , -s - Source port to bind to, defaults to 68. - It is highly unlikely that you'll ever get any response with another port. - ---exit-on-first-offer, -1 - Exit as soon as a response is received. - ---log-level - Logging level to use: `DEBUG`, `INFO`, `WARNING`, `ERROR`. - The default is `WARNING`. - Set to `INFO` to log sent & received messages. - - -Output format -------------- - -Each time an `OFFER` is received, a JSON object is printed to stdout. -It contains the following data: - -- `interface`, the interface on which the message was received, -- `message`, a full dump of the `OFFER`, with - - `dhcp`: decoded bootp & dhcp data - - `eth_src`, `eth_dst`: Link-layer source & destination addresses - - `ip_src`, `ip_dst`: IPv4 source & destination addresses - - `sport`, `dport`: UDP source & destination ports. - -In proper DHCP, `eth_dst` & `ip_dst` are always broadcast addresses, -and the UDP source & dest. ports are always 67 and 68. - - -.. code-block:: bash - - # dhcp-server-detector -1 wlp61s0 - { - "interface": "wlp61s0", - "message": { - "dhcp": { - "op": 2, - "htype": 1, - "hlen": 6, - "hops": 0, - "xid": 2900208454, - "secs": 0, - "flags": 32768, - "ciaddr": "0.0.0.0", - "yiaddr": "192.168.94.166", - "siaddr": "0.0.0.0", - "giaddr": "0.0.0.0", - "chaddr": "a0:a4:c5:93:ac:60", - "sname": "", - "file": "", - "cookie": "63:82:53:63", - "options": { - "message_type": 2, - "server_id": "192.168.94.254", - "lease_time": 43200, - "subnet_mask": "255.255.255.0", - "router": [ - "192.168.94.254" - ], - "name_server": [ - "192.168.94.254" - ], - "broadcast_address": "192.168.94.255" - } - }, - "eth_src": "14:0c:76:62:51:64", - "eth_dst": "ff:ff:ff:ff:ff:ff", - "ip_src": "192.168.94.254", - "ip_dst": "255.255.255.255", - "sport": 67, - "dport": 68 - } - } - - -Exit codes ----------- - -The programs always exits with `0` if at least one `OFFER` was received -in the configured duration, `1` otherwise. - -Along with its JSON output, it means it can be used easily in scripts, like: - -.. code-block:: bash - - # prints a line for every interface on which a DHCP server is detected. - # waits 1 second for each interface. - for ifname in $(ip --json l | jq -r '.[].ifname'); do - if dhcp-server-detector -d 1 -1 $ifname > /dev/null; then - echo "DHCP server found on $ifname" - fi - done - -or: - -.. code-block:: bash - - # does the same as the script above, but polls all interfaces in parallel - # for 3s max before exiting - dhcp-server-detector -d 3 $(ip --json l | jq -r '.[].ifname') |\ - jq -r .interface diff --git a/docs/fixtures.rst b/docs/fixtures.rst deleted file mode 100644 index 40f36ea11..000000000 --- a/docs/fixtures.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. _fixtures: - -.. automodule:: pyroute2.fixtures - :members: diff --git a/docs/index.rst b/docs/index.rst index c761e1022..9cc3c46e2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,12 +20,8 @@ Usage :maxdepth: 2 usage - asyncio - threading iproute ndb - fixtures - plan9 wiset ipset netns @@ -49,7 +45,6 @@ Man pages pyroute2-cli pyroute2-dhcp-client - dhcp-server-detector Development ----------- @@ -61,14 +56,23 @@ Development arch parser netlink + nlsocket + +Experimental +------------ + +.. toctree:: + :maxdepth: 1 + + remote Deprecated ---------- .. toctree:: - :maxdepth: 1 + :maxdepth: 1 - deprecated + ipdb_toc Indices and tables ================== diff --git a/docs/ipdb.rst b/docs/ipdb.rst new file mode 100644 index 000000000..ebf77d02a --- /dev/null +++ b/docs/ipdb.rst @@ -0,0 +1,6 @@ +.. _ipdb: + +.. automodule:: pyroute2.ipdb.main + :members: + + diff --git a/docs/ipdb_toc.rst b/docs/ipdb_toc.rst new file mode 100644 index 000000000..f450cfe71 --- /dev/null +++ b/docs/ipdb_toc.rst @@ -0,0 +1,13 @@ +.. ipdbtoc: + +IPDB module +=========== + +.. warning:: + The IPDB module is deprecated and obsoleted by NDB. Please consider + using NDB instead. + +.. toctree:: + :maxdepth: 2 + + ipdb diff --git a/docs/iproute.rst b/docs/iproute.rst index 4e07d4eb9..eb8a570f0 100644 --- a/docs/iproute.rst +++ b/docs/iproute.rst @@ -1,17 +1,49 @@ .. _iproute: -IPRoute and related modules -=========================== +IPRoute module +============== -.. toctree:: - :maxdepth: 2 +.. automodule:: pyroute2.iproute + :members: - iproute_intro - iproute_netns - iproute_responses - iproute_linux +BSD systems +----------- -.. - excluded chapters: - iproute_platforms - iproute_tc +.. automodule:: pyroute2.iproute.bsd + +Windows systems +--------------- + +.. automodule:: pyroute2.iproute.windows + +.. autoclass:: pyroute2.iproute.windows.IPRoute + :members: + +Linux systems +------------- + +.. automodule:: pyroute2.iproute.linux + :members: + +Queueing disciplines +-------------------- + +.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_drr + :members: + +.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_choke + :members: + +.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_clsact + :members: + +.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_hfsc + :members: + +.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_htb + :members: + +Filters +------- + +.. automodule:: pyroute2.netlink.rtnl.tcmsg.cls_u32 diff --git a/docs/iproute_intro.rst b/docs/iproute_intro.rst deleted file mode 100644 index b7a5a0cef..000000000 --- a/docs/iproute_intro.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. _iproute_intro: - -.. testsetup:: * - - from pyroute2 import config - - config.mock_netlink = True - - -RTNL classes ------------- - -**Arguments** to the constructors: - -port: `Optional[int]` - An integer to be used together with `pid` in the `bind()` - call, `epid = pid + (port << 22)`. - -pid: `Optional[int]` - Value to be used as the base in while calling `bind()` - -fileno: `Optional[int]` - An open file descriptor to construct the socket from. - -sndbuf: `int` - Send buffer limit in bytes. - -rcvbuf: `int` - Receive buffer limit in bytes. - -rcvsize: `int` - Maximum recieve packet size. - -all_ns: `bool` - Turns on `NETLINK_LISTEN_ALL_NSID` on the socket. - -async_qsize - Deprecated. - -nlm_generator - Deprecated. - -target: `str` - Target field (string) to be provided in the header. Useful - when working with sockets in multiple network namespaces. - -ext_ack: `bool` - Extended ACK controls reporting additional error or warning - info in `NLMSG_ERROR` and `NLMSG_DONE` messages. - -strict_check: `bool` - Controls strict input field checking. By default kernel does - not validate the input fields, silently ignoring possible - issues, that may lead to regressions in the future. - -groups: `int` (default: `pyroute2.netlink.rtnl.RTMGRP_DEFAULTS`) - Groups to subscribe when calling `bind()`, see `pyroute2.netlink.rtnl` - -nlm_echo: `bool` - Return the request fields in the response. - -use_socket: `Optional[socket.socket]` - An existing socket object to run the protocol on. - -netns: `str` - Network namespace to use. - -flags: `int` (default: `os.O_CREAT`) - Flags to use when calling `netns.create()`. By default the - library will create netns if it doesn't exist, and reuse if - it does. In order to fail when the network namespace already - exists, you should provide `flags=0`. - -libc: `Optional[ctypes.CDLL]` - If you want the socket to use specific libc object when managing - network namespaces, you can use this argument. - -use_event_loop: `Optional[asyncio.AbstractEventLoop]` - Use an existing asyncio event loop. - -**RTNL classes**: - -.. autoclass:: pyroute2.AsyncIPRSocket - -.. autoclass:: pyroute2.IPRSocket - -.. autoclass:: pyroute2.AsyncIPRoute - -.. autoclass:: pyroute2.IPRoute - -.. autoclass:: pyroute2.NetNS diff --git a/docs/iproute_linux.rst b/docs/iproute_linux.rst deleted file mode 100644 index f68951a60..000000000 --- a/docs/iproute_linux.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _iproute_linux: - -.. testsetup:: * - - from pyroute2 import config, IPRoute - config.mock_netlink = True - ipr = IPRoute() - -.. testcleanup:: * - - ipr.close() - - -Linux systems -------------- - -.. automodule:: pyroute2.iproute.linux - -.. autoclass:: pyroute2.iproute.linux.RTNL_API - :members: diff --git a/docs/iproute_netns.rst b/docs/iproute_netns.rst deleted file mode 100644 index 3d0959562..000000000 --- a/docs/iproute_netns.rst +++ /dev/null @@ -1,85 +0,0 @@ -.. _iproute_netns: - - -.. testsetup:: * - - from pyroute2 import config - config.mock_netlink = True - -Using namespaces ----------------- - -The code to init a netlink socket in a network namespace was -moved to the library core. To run a socket within a netns, simply -pass `netns` argument to the socket init: - -.. testcode:: netns00 - - import asyncio - - from pyroute2 import AsyncIPRoute - - - async def main(): - async with AsyncIPRoute(netns="test") as ipr: - print(f"current netns: {ipr.status['netns']}") - - asyncio.run(main()) - -.. testoutput:: netns00 - - current netns: test - -It is possible to use the old `NetNS` class, it is now just a compatibility -wrapper for the new API: - -.. testcode:: netns01 - - from pyroute2 import NetNS - - with NetNS("test") as ns: - print(f"current netns: {ns.status['netns']}") - -.. testoutput:: netns01 - - current netns: test - -Flags also might be used with any constructor. The default flags are -`os.O_CREAT`, which means that the network namespace will be created -if doesn't exist. Using 0 as `flags` value means that the constructor -will fail, if the network namespace doesn't exist already: - -.. testcode:: netns02 - - from pyroute2 import IPRoute - - try: - ipr = IPRoute(netns="foo", flags=0) - except FileNotFoundError: - print("netns doesn't exist, refuse to start") - -.. testoutput:: netns02 - - netns doesn't exist, refuse to start - -The init routine works now as follows: - -* fork a child using `pyroute2.config.child_process_mode`, which can be - either `"fork"` (default) for `os.fork()` or `"mp"` for - `multiprocessing.Process()` (safer and slower). -* start a socket in the child -* send the socket FD back to the parent -* init a socket in the parent using the FD from the child -* exit the child - -.. testcode:: netns-config - :hide: - - from pyroute2 import config - assert isinstance(config.child_process_mode, str) - -An important note about `pyroute2.config.child_process_mode`: while the -`"fork"` mode might be significantly faster than `"mp"` on some setups -and versions, it is not threadsafe, and you will get warnings from Python -when using it in multithreaded applications. The socket init routine is -written to be safe even under these circumstances, but ye warned. diff --git a/docs/iproute_platforms.rst.disabled b/docs/iproute_platforms.rst.disabled deleted file mode 100644 index 063f0f56c..000000000 --- a/docs/iproute_platforms.rst.disabled +++ /dev/null @@ -1,20 +0,0 @@ -.. _iproute_platforms: - -Non-RTNL platforms ------------------- - -`pyroute2` offers experimental support for platforms that do not -provide Netlink/RTNL, including BSD systems. - -BSD systems -~~~~~~~~~~~ - -.. automodule:: pyroute2.iproute.bsd - -Windows systems -~~~~~~~~~~~~~~~ - -.. automodule:: pyroute2.iproute.windows - -.. autoclass:: pyroute2.iproute.windows.IPRoute - :members: diff --git a/docs/iproute_responses.rst b/docs/iproute_responses.rst deleted file mode 100644 index 70639f0f5..000000000 --- a/docs/iproute_responses.rst +++ /dev/null @@ -1,134 +0,0 @@ -.. _iproute_responses: - -.. testsetup:: * - - from pyroute2 import config - - config.mock_netlink = True - -NLMSG_ERROR responses ---------------------- - -Some kernel subsystems return `NLMSG_ERROR` in response to any request. -This is acceptable as long as `nlmsg["header"]["error"]` is `None`. -If it is not `None`, an exception will be raised by the parser. - -If you receive an `NLMSG_ERROR` message instead of an exception, -it means `error == 0`, which is equivalent to `$? == 0` in bash. - -How to work with messages -------------------------- - -Every netlink message contains a header, fields, and NLAs -(netlink attributes). Each NLA is itself a netlink message -(see "recursion"). - -The library parses messages according to this structure. -Each RTNL message includes the following: - -* `nlmsg['header']` -- parsed header -* `nlmsg['attrs']` -- NLA chain (parsed on demand) -* data fields, e.g. `nlmsg['flags']` etc. -* `nlmsg.header` -- the header fields spec -* `nlmsg.fields` -- the data fields spec -* `nlmsg.nla_map` -- NLA spec - -.. - Test the attributes above: - -.. testcode:: - :hide: - - from pyroute2 import IPRoute - - with IPRoute() as ipr: - msg = tuple(ipr.link('dump'))[0] - assert isinstance(msg['header'], dict) - assert msg['header']['sequence_number'] > 0 - assert isinstance(msg['attrs'], list) - assert isinstance(msg.header, tuple) - assert isinstance(msg.fields, tuple) - assert isinstance(msg.nla_map, tuple) - assert len(msg['attrs']) > 0 - assert len(msg.header) == 5 - assert len(msg.fields) > 0 - assert len(msg.nla_map) > 0 - -One key feature of the parser is that NLAs are parsed -only on demand, i.e., when accessed. This prevents -unnecessary CPU usage. - -The NLA chain is a list-like structure rather than a -dictionary because the netlink standard does not require -NLAs to be unique within a single message:: - - {'attrs': [('IFLA_IFNAME', 'lo'), # [1] - ('IFLA_TXQLEN', 1), - ('IFLA_OPERSTATE', 'UNKNOWN'), - ('IFLA_LINKMODE', 0), - ('IFLA_MTU', 65536), - ('IFLA_GROUP', 0), - ('IFLA_PROMISCUITY', 0), - ('IFLA_NUM_TX_QUEUES', 1), - ('IFLA_NUM_RX_QUEUES', 1), - ('IFLA_CARRIER', 1), - ...], - 'change': 0, - 'event': 'RTM_NEWLINK', # [2] - 'family': 0, - 'flags': 65609, - 'header': {'error': None, # [3] - 'flags': 2, - 'length': 1180, - 'pid': 28233, - 'sequence_number': 257, # [4] - 'type': 16}, # [5] - 'ifi_type': 772, - 'index': 1} - - # [1] every NLA is parsed upon access - # [2] this field is injected by the RTNL parser - # [3] if not None, an exception will be raised - # [4] more details in the netlink description - # [5] 16 == RTM_NEWLINK - -To access fields or NLAs, use the `.get()` method. To retrieve -nested NLAs, pass a tuple of NLA names to the `.get()` call to -navigate through the hierarchy: - -.. testcode:: - - from pyroute2 import IPRoute - - with IPRoute() as ipr: - lo = tuple(ipr.link("get", index=1))[0] - # get a field - assert lo.get("index") == 1 - # get an NLA - assert lo.get("ifname") == "lo" - # get a nested NLA - assert lo.get(("stats64", "rx_bytes")) == 43309665 - -If an NLA with the specified name is not present in the chain, -`.get()` returns None. To retrieve a list of all NLAs with the -specified name, use `.get_attrs()`. - -Below is an example demonstrating the usage of `.get()` and -`.get_attrs()` with an NLA hierarchy:: - - # for macvlan interfaces there may be several - # IFLA_MACVLAN_MACADDR NLA provided, so use - # get_attrs() to get all the list, not only - # the first one - - (msg - .get('IFLA_LINKINFO') # one NLA - .get('IFLA_INFO_DATA') # one NLA - .get_attrs('IFLA_MACVLAN_MACADDR')) # a list of - -.. - FIXME! test the example above - -The protocol itself does not impose a limit on the number of NLAs -of the same type within a single message. This is why we cannot -represent them as a dictionary, unlike with `PF_ROUTE` messages. diff --git a/docs/iproute_tc.rst.disabled b/docs/iproute_tc.rst.disabled deleted file mode 100644 index 2042177c6..000000000 --- a/docs/iproute_tc.rst.disabled +++ /dev/null @@ -1,24 +0,0 @@ -.. _iproute_tc: - -Queueing disciplines --------------------- - -.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_drr - :members: - -.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_choke - :members: - -.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_clsact - :members: - -.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_hfsc - :members: - -.. automodule:: pyroute2.netlink.rtnl.tcmsg.sched_htb - :members: - -Filters -------- - -.. automodule:: pyroute2.netlink.rtnl.tcmsg.cls_u32 diff --git a/docs/ndb_auth.rst b/docs/ndb_auth.rst new file mode 100644 index 000000000..2d28a9b70 --- /dev/null +++ b/docs/ndb_auth.rst @@ -0,0 +1,34 @@ +.. _ndbauth: + +Authorization plugins +===================== + +.. automodule:: pyroute2.ndb.auth_manager + +Usecase: OpenStack Keystone auth +-------------------------------- + +Say we have a public service that provides access to NDB instance via +HTTP, and authenticates users via Keystone. Then the auth flow could be: + +1. Accept a connection from a client +2. Create custom auth manager object A +3. A.__init__() validates X-Auth-Token against Keystone (Authentication) +4. A.check() checks that X-Auth-Token is not expired (Authorization) +5. The auth result is being logged (Accounting) + +An example AuthManager with OpenStack APIv3 support you may find in the +`/examples/ndb/` directory. + +.. literalinclude:: ../examples/ndb/keystone_auth.py + :language: python + :caption: keystone_auth.py + :name: keystone_auth + +Usecase: RADIUS auth +-------------------- + +.. literalinclude:: ../examples/ndb/radius_auth.py + :language: python + :caption: radius_auth.py + :name: radius_auth diff --git a/docs/ndb_probes.rst b/docs/ndb_probes.rst deleted file mode 100644 index 8c7240dc0..000000000 --- a/docs/ndb_probes.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. ndbprobes: - -Network probes -============== - -.. automodule:: pyroute2.ndb.objects.probe diff --git a/docs/ndb_toc.rst b/docs/ndb_toc.rst index c2345fd46..812ba7ddd 100644 --- a/docs/ndb_toc.rst +++ b/docs/ndb_toc.rst @@ -11,7 +11,7 @@ ndb_interfaces ndb_addresses ndb_routes - ndb_probes ndb_schema ndb_sources ndb_debug + ndb_auth diff --git a/docs/netns.rst b/docs/netns.rst index b4feefccf..e17ebf758 100644 --- a/docs/netns.rst +++ b/docs/netns.rst @@ -4,16 +4,10 @@ NetNS management ================ .. automodule:: pyroute2.netns - :members: create, - remove, - attach, - setns, - pushns, - popns, - dropns, - listnetns, - ns_pids, - pid_to_ns + :members: + +.. automodule:: pyroute2.nslink + :members: -.. automodule:: pyroute2.nslink.nspopen +.. automodule:: pyroute2.NSPopen :members: diff --git a/docs/nlsocket.rst.disabled b/docs/nlsocket.rst similarity index 100% rename from docs/nlsocket.rst.disabled rename to docs/nlsocket.rst diff --git a/docs/parser.rst b/docs/parser.rst index 67772835f..b070020ce 100644 --- a/docs/parser.rst +++ b/docs/parser.rst @@ -7,24 +7,30 @@ Netlink parser data flow ======================== -NetlinkRequest --------------- +NetlinkSocketBase: receive the data +----------------------------------- -In order to run a query and get the response to it, `AsyncNetlinkSocket` -utilizes `NetlinkRequest` class that calculates required message flags, -allocates the sequence number, completes the query message, and encodes it. +When `NetlinkSocketBase` receives the data from a netlink socket, it can do it +in two ways: -When the message is about to being sent, `NetlinkRequest` first tries a -proxy, if any registered, and if `NetlinkRequest.proxy()` returns True -then `NetlinkRequest` stops processing, otherwise sends it using the -underlying socket. +1. get data directly with `socket.recv()` or `socket.recv_into()` +2. run a buffer thread that receives the data asap and leaves in the + `buffer_queue` to be consumed later by `recv()` or `recv_into()` -`NetlinkRequest.response()` collects all the response packets for its -sequence number, buffered so far, and returns an async iterator over -the arrived response messages. +`NetlinkSocketBase` implements these two receive methods, that choose +the data source -- directly from the socket or from `buffer_queue` -- +depending on the `buffer_thread` property: -Marshal: get and run parsers ----------------------------- +**pyroute2.netlink.nlsocket.NetlinkSocketBase** + +.. code-include:: :func:`pyroute2.netlink.nlsocket.NetlinkSocketBase.recv` + :language: python + +.. code-include:: :func:`pyroute2.netlink.nlsocket.NetlinkSocketBase.recv_into` + :language: python + +.. code-include:: :func:`pyroute2.netlink.nlsocket.NetlinkSocketBase.buffer_thread_routine` + :language: python .. aafig:: :scale: 80 @@ -66,6 +72,9 @@ Marshal: get and run parsers | v +Marshal: get and run parsers +---------------------------- + Marshal should choose a proper parser depending on the `key`, `flags` and `sequence_number`. By default it uses only `nlmsg->type` as the `key` and `nlmsg->flags`, and there are several ways to customize getting parsers. @@ -189,11 +198,10 @@ Mandatory message fields, expected by NetlinkSocketBase methods: Per-request parsers ------------------- -To assign a custom parser to a request/response communication, it is -enough to provide the parser function to the `NetlinkRequest` object. - -An example is `IPRoute.get_default_routes()`, which could be slow on -systems with huge amounts of routes. +Sometimes, it may be reasonable to handle a particular response with a +specific parser rather than a generic one. An example is +`IPRoute.get_default_routes()`, which could be slow on systems with +huge amounts of routes. Instead of parsing every route record as `rtmsg`, this method assigns a specific parser to its request. The custom parser doesn't parse records @@ -210,9 +218,35 @@ then parses only matched records with the standard routine: .. code-include:: :func:`pyroute2.iproute.parsers.default_routes` :language: python +To assign a custom parser to a request/response communication, you should +know first `sequence_number`, be it allocated dynamically with +`NetlinkSocketBase.addr_pool.alloc()` or assigned statically. Then you +can create a record in `NetlinkSocketBase.seq_map`: -NetlinkRequest: pick correct messages -------------------------------------- +.. code-block:: python + + # + def my_parser(data, offset, length): + ... + return parsed_message + + msg_seq = nlsocket.addr_pool.alloc() + msg = nlmsg() + msg['header'] = { + 'type': my_type, + 'flags': NLM_F_REQUEST | NLM_F_ACK, + 'sequence_number': msg_seq, + } + msg['data'] = my_data + msg.encode() + nlsocket.seq_map[msg_seq] = my_parser + nlsocket.sendto(msg.data, (0, 0)) + for reponse_message in nlsocket.get(msg_seq=msg_seq): + handle(response_message) + + +NetlinkSocketBase: pick correct messages +---------------------------------------- The netlink protocol is asynchronous, so responses to several requests may come simultaneously. Also the kernel may send broadcast messages that are @@ -263,10 +297,14 @@ The message flow on the diagram features `sequence_number == 0` broadcasts and further you can run a request with `sequence_number == 2` before the final response with `sequence_number == 1` comes. -To handle that, pyroute2 protocol objects buffer all the messages, and -`NetlinkRequest` only gets the reponse. +To handle that, `NetlinkSocketBase.get()` buffers all the irrelevant messages, +returns ones with only the requested `sequence_number`, and uses locks to wait +on the resource. + +The current implementation is relatively complicated and will be changed in +the future. -**pyroute2.netlink.nlsocket.NetlinkRequest** +**pyroute2.netlink.nlsocket.NetlinkSocketBase** -.. code-include:: :func:`pyroute2.netlink.nlsocket.NetlinkRequest.response` +.. code-include:: :func:`pyroute2.netlink.nlsocket.NetlinkSocketBase.get` :language: python diff --git a/docs/plan9.rst b/docs/plan9.rst deleted file mode 100644 index 6db37e454..000000000 --- a/docs/plan9.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _plan9: - -.. testsetup:: - - import asyncio - - from pyroute2.plan9.server import Plan9ServerSocket - from pyroute2.plan9.client import Plan9ClientSocket - - -Plan9 9p2000 protocol -===================== - -The library provides basic asynchronous 9p2000 implementation. - -.. autoclass:: pyroute2.plan9.server.Plan9ServerSocket - :members: - -.. autoclass:: pyroute2.plan9.client.Plan9ClientSocket - :members: diff --git a/docs/pyroute2-dhcp-client.rst b/docs/pyroute2-dhcp-client.rst index 9c08728c5..914358e68 100644 --- a/docs/pyroute2-dhcp-client.rst +++ b/docs/pyroute2-dhcp-client.rst @@ -4,121 +4,49 @@ pyroute2-dhcp-client Synopsis -------- - **pyroute2-dhcp-client** [OPTION]... + **pyroute2-dhcp-client** Description ----------- -**pyroute2-dhcp-client** is a client based on the pyroute2 DHCP implementation. - -It is mostly a simple CLI wrapper around the -`pyroute2.dhcp.client.AsyncDHCPClient` class, which has more flexibility and -configuration options (see `pyroute2.dhcp.client.ClientConfig`). -Some options are not (yet) configurable through the commandline, such as the -client and vendor IDs, the hostname and the requested parameters. - -Its default behavior is to try to acquire a lease for the passed interface, -configuring its IP address and gateway, as long as it is running. -On exit, it releases its lease and remove the associated IP from the interface. -If the interface goes down during the client's lifetime, or is not up when -starting, the client waits until it is up again. - -System configuration based on lease options (i.e. IP/gateway configuration) -is done through hooks. The default hooks add the obtained IP to the interface -and set the default gateway, but that can be configured, and you can ask the -client to run your own custom hooks (see `pyroute2.dhcp.hooks`). - -The client can also be started in "one-shot" mode (see the `-x` option), where -it will exit as soon as a lease is obtained. - -Available options: ------------------- - ---lease-type - Class to use for leases. Must be a subclass of `pyroute2.dhcp.leases.Lease`. - ---hook - Hooks to run on lease events, see `pyroute2.dhcp.hooks`. - ---disable-hooks - Disable all hooks. - ---exit-on-timeout , -x - Wait for max N seconds for a lease, exit if none could be obtained. - ---log-level - Logging level to use: DEBUG, INFO, WARNING, ERROR. The default is INFO. - ---no-release, -R - Do not send a DHCPRELEASE on exit. - ---write-pidfile, -p - Write a pid file in the working directory. - -Leases ------- - -The default behavior is to write the latest obtained lease to a JSON file named -after the interface in the client's working directory. - -On startup, if the client finds such a file, it will request the same IP. - -This behavior can be modified with the `--lease-type` option. For example, the -`pyroute2.dhcp.client.leases.JSONStdoutLease` class just writes leases to -standard output and does not persist them to disk. - -Signals -------- - -**pyroute2-dhcp-client** responds to the following signals: - -- `SIGINT` (i.e. Ctrl-C) releases & exits -- `SIGUSR1` triggers a lease renewal (normally triggered automatically at ~50% of the lease time) -- `SIGUSR2` triggers a rebinding (normally triggered automatically at ~87% of the lease time) -- `SIGHUP` forces the current lease to expire and starts looking for a new one +**pyroute2-dhcp-client** is a way too simple DHCP client. The only +option is the network interface to run on. The script prints the +DHCP server response as JSON. Examples -------- .. code-block:: bash - # pyroute2-dhcp-client --disable-hooks -R -x 5 eth0 - ... - # cat eth0.lease.json + # pyroute2-dhcp-client eth0 { - "ack": { "op": 2, "htype": 1, "hlen": 6, "hops": 0, - "xid": 391458644, + "xid": 17, "secs": 0, - "flags": 32768, + "flags": 0, "ciaddr": "0.0.0.0", - "yiaddr": "192.168.124.180", - "siaddr": "192.168.124.1", + "yiaddr": "172.16.1.105", + "siaddr": "172.16.1.1", "giaddr": "0.0.0.0", - "chaddr": "aa:80:fa:2c:49:a2", + "chaddr": "18:56:80:11:ff:a3", "sname": "", "file": "", - "cookie": "63:82:53:63", + "cookie": "63:82:53", "options": { - "message_type": 5, - "server_id": "192.168.124.1", - "lease_time": 3600, - "renewal_time": 1800, - "rebinding_time": 3150, - "subnet_mask": "255.255.255.0", - "broadcast_address": "192.168.124.255", - "router": [ - "192.168.124.1" - ], - "name_server": [ - "192.168.124.1" - ] + "message_type": 5, + "server_id": "172.16.1.1", + "lease_time": 43200, + "renewal_time": 21600, + "rebinding_time": 37800, + "subnet_mask": "255.255.255.0", + "router": [ + "172.16.1.1" + ], + "name_server": [ + "172.16.1.1" + ] } - }, - "interface": "eth0", - "server_mac": "52:54:00:e9:d3:1d", - "obtained": 1740562827.7350075 } diff --git a/docs/remote.rst.disabled b/docs/remote.rst similarity index 100% rename from docs/remote.rst.disabled rename to docs/remote.rst diff --git a/docs/threading.rst b/docs/threading.rst deleted file mode 100644 index b4f0dbe52..000000000 --- a/docs/threading.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. _threading: - -Using the library in threaded environments -========================================== - -Network namespaces ------------------- - -To run a separate socket in one network namespace while keeping the -Python process in another namespace, the library follows these steps: - -1. Spawn a child process. -2. Execute `netns.setns()` in the child. -3. Create a socket. -4. Send the file descriptor back to the parent using `socket.send_fds()`. -5. Terminate the child process. -6. Create a socket in the parent using `socket(fileno=...)`. - -As a result, the parent process obtains a socket belonging to -another network namespace. However, it can be used natively like -any other socket, both synchronously and asynchronously. - -Starting a child process: os.fork() ------------------------------------ - -By default, pyroute2 uses `os.fork()` to create the child process. -In multithreaded environments, `os.fork()` does not recreate threads; -the child process continues only from the thread where `os.fork()` -was called. This can leave the garbage collector in a corrupted state. - -While this is generally not an issue -- since the socket creation routine -stops the garbage collector, and does not rely on shared data -- there -is still some risk. - -To address this, pyroute2 provides a configuration option: -`config.child_process_mode`. The default value is `"fork"`, but you -can change it to `"mp"` to use the `multiprocessing` module for creating -and managing the child process. - -Starting a child process: multiprocessing ------------------------------------------ - -The `multiprocessing` module may or may not rely on `os.fork()`, depending -on the method set via `multiprocessing.set_start_method()`. In Python -versions earlier than 3.14, the default method is `"fork"`, but starting -from Python 3.14, the default is `"spawn"`. - -Using `"spawn"` is safer but significantly slower. Additionally, `"spawn"` -introduces limitations due to pickling: - -* The target function and its arguments must be pickleable. -* Passing lambda functions as the target is not possible. -* The libc instance cannot be passed to the child process. - -Since pyroute2 does not manage the `multiprocessing` start method, the -start method cannot be configured via `config.child_process_mode`. If you -set `config.child_process_mode` to `"mp"`, and need to explicitly specify -the start method, you must call `multiprocessing.set_start_method()` -manually elsewhere in your program. - -Threading and asyncio ---------------------- - -An asyncio event loop can only run in the thread where it was started. -In a multithreaded environment, the library creates a local event loop -and a local netlink socket for each thread that accesses the object. -While this approach is safe, it complicates object termination. -Although an event loop can be stopped from another thread, it cannot -be closed. - -The best solution is to call `close()` in every thread where you -call `bind()`. diff --git a/docs/usage.rst b/docs/usage.rst index 0cb72a66d..81a91880d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,72 +1,26 @@ .. usage: -.. testsetup:: * - - from pyroute2 import config - config.mock_netlink = True - Quickstart ========== -"Hello world", sync API: +Hello, world:: -.. testcode:: + $ pip install pyroute2 + $ cat example.py from pyroute2 import IPRoute + with IPRoute() as ipr: + print([x.get_attr('IFLA_IFNAME') for x in ipr.get_links()]) - def main(): - ipr = IPRoute() - for link in ipr.link("dump"): - print(link.get("ifname"), link.get("state"), link.get("address")) - ipr.close() - - main() - - -.. testoutput:: - - lo up 00:00:00:00:00:00 - eth0 up 52:54:00:72:58:b2 - -"Hello world", async API: - -.. testcode:: - - import asyncio - - from pyroute2 import AsyncIPRoute - - - async def main(): - ipr = AsyncIPRoute() - async for link in await ipr.link("dump"): - print(link.get("ifname"), link.get("state"), link.get("address")) - ipr.close() - - asyncio.run(main()) - - -.. testoutput:: - - lo up 00:00:00:00:00:00 - eth0 up 52:54:00:72:58:b2 - -Netlink sockets ---------------- - -Netlink sockets created with pyroute2 behave similarly to ordinary -socket objects, but there are some key differences in how they -handle data reception. - -At a low level, these sockets are monitored by the asyncio event -loop, which means that direct use of `recv()` or `recvmsg()` is not -supported. If you require such low-level functionality, you would -need to modify the protocol class associated with the socket object. + $ python example.py + ['lo', 'p6p1', 'wlan0', 'virbr0', 'virbr0-nic'] -By default, the lowest-level API available for receiving data with -pyroute2 sockets is the `get()` method. +Sockets +------- -.. testcode:: +In the runtime pyroute2 socket objects behave as normal +sockets. One can use them in the poll/select, one can +call `recv()` and `sendmsg()`:: from pyroute2 import IPRoute @@ -76,28 +30,33 @@ pyroute2 sockets is the `get()` method. # subscribe to broadcast messages ipr.bind() - # wait for parsed data - data = ipr.get() + # wait for data (do not parse it) + data = ipr.recv(65535) + # parse received data + messages = ipr.marshal.parse(data) + + # shortcut: recv() + parse() + # + # (under the hood is much more, but for + # simplicity it's enough to say so) + # + messages = ipr.get() -... but pyroute2 objects have additional high level methods: -.. testcode:: +But pyroute2 objects have a lot of methods, written to +handle specific tasks:: from pyroute2 import IPRoute # RTNL interface with IPRoute() as ipr: - # get IP addresses - for msg in ipr.addr("dump"): - addr = msg.get("address") - mask = msg.get("prefixlen") - print(f"{addr}/{mask}") -.. testoutput:: + # get devices list + ipr.get_links() - 127.0.0.1/8 - 192.168.122.28/24 + # get addresses + ipr.get_addr() Resource release ---------------- @@ -121,15 +80,13 @@ import signature. All other objects are also available for import, but they may change signatures in the next versions. -E.g.: - -.. testcode:: +E.g.:: # Import a pyroute2 class directly. In the next versions # the import signature can be changed, e.g., NetNS from # pyroute2.netns.nslink it can be moved somewhere else. # - from pyroute2.iproute.linux import NetNS + from pyroute2.netns.nslink import NetNS ns = NetNS('test') # Import the same class from root module. This signature @@ -138,3 +95,30 @@ E.g.: # from pyroute2 import NetNS ns = NetNS('test') + +Special cases +============= + +eventlet +-------- + +The eventlet environment conflicts in some way with socket +objects, and pyroute2 provides some workaround for that:: + + # import symbols + # + import eventlet + from pyroute2 import NetNS + from pyroute2.config.eventlet import eventlet_config + + # setup the environment + eventlet.monkey_patch() + eventlet_config() + + # run the code + ns = NetNS('nsname') + ns.get_routes() + ... + +This may help, but not always. In general, the pyroute2 library +is not eventlet-friendly. diff --git a/examples/devlink/devlink_monitor.py b/examples/devlink/devlink_monitor.py index 951e03f19..b586bdf01 100644 --- a/examples/devlink/devlink_monitor.py +++ b/examples/devlink/devlink_monitor.py @@ -1,5 +1,6 @@ from pyroute2.devlink import DL + dl = DL(groups=~0) print(dl.get()) dl.close() diff --git a/examples/ethtool/ethtool-ioctl_get_infos.py b/examples/ethtool/ethtool-ioctl_get_infos.py index 4e2ac2629..cf6f9a33c 100644 --- a/examples/ethtool/ethtool-ioctl_get_infos.py +++ b/examples/ethtool/ethtool-ioctl_get_infos.py @@ -1,6 +1,6 @@ import sys - -from pyroute2.ethtool.ioctl import IoctlEthtool, NotSupportedError +from pyroute2.ethtool.ioctl import IoctlEthtool +from pyroute2.ethtool.ioctl import NotSupportedError if len(sys.argv) != 2: raise Exception("USAGE: {0} IFNAME".format(sys.argv[0])) diff --git a/examples/ethtool/ethtool_get_infos.py b/examples/ethtool/ethtool_get_infos.py index e564b408e..9817325f7 100644 --- a/examples/ethtool/ethtool_get_infos.py +++ b/examples/ethtool/ethtool_get_infos.py @@ -1,5 +1,4 @@ import sys - from pyroute2.ethtool import Ethtool if len(sys.argv) != 2: diff --git a/examples/generic/Makefile b/examples/generic/Makefile index 6e656b3a0..5c516a39b 100644 --- a/examples/generic/Makefile +++ b/examples/generic/Makefile @@ -5,3 +5,4 @@ all: clean: make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean + diff --git a/examples/generic/README.rst b/examples/generic/README.rst deleted file mode 100644 index cc5cca389..000000000 --- a/examples/generic/README.rst +++ /dev/null @@ -1,27 +0,0 @@ -An example echo protocol ------------------------- - -This directory contains an example of an echo protocol implemented -using Generic Netlink. To load the module, first build it with `make`, -then load it using `insmod`. - -.. note:: You must either disable Secure Boot or sign the module to - load it successfully. - -You can use this code as a template for developing your own Generic -Netlink protocols. - -Running the client ------------------- - -To test the module, simply run `netl.py`. It will send a string to the -kernel and print the echoed reply. - -Out of scope ------------- - -The following topics are beyond the scope of this project. Please refer -to external documentation for: - -* How to sign kernel modules if Secure Boot is enabled -* How to install kernel build dependencies for your distribution diff --git a/examples/generic/netl.c b/examples/generic/netl.c index dd5c1cc35..873d816b8 100644 --- a/examples/generic/netl.c +++ b/examples/generic/netl.c @@ -3,7 +3,7 @@ * Use `make` to compile and `insmod` to load the module * * Sergiy Lozovsky - * Peter Saveliev + * Peter V. Saveliev * * Requires kernel 4.10+ */ @@ -12,118 +12,119 @@ #include /* Needed for the macros */ #include -#define EXMPL_GENL_FAMILY_NAME "ECHO_GENL" -#define EXMPL_GENL_VERSION 0x1 - -/* attributes (variables): the index in this enum is used as - * a reference for the type, userspace application - * has to indicate the corresponding type +/* attributes (variables): the index in this enum is used as a reference for the type, + * userspace application has to indicate the corresponding type + * the policy is used for security considerations */ enum { EXMPL_NLA_UNSPEC, - EXMPL_NLA_STR, + EXMPL_NLA_DATA, + EXMPL_NLA_LEN, __EXMPL_NLA_MAX, }; -#define EXMPL_NLA_MAX (__EXMPL_NLA_MAX - 1) - /* ... and the same for commands */ enum { EXMPL_CMD_UNSPEC, - EXMPL_CMD_ECHO, - __EXMPL_CMD_MAX, + EXMPL_CMD_MSG, }; -#define EXMPL_CMD_MAX (__EXMPL_CMD_MAX - 1) - /* attribute policy: defines which attribute has which type (e.g int, char * etc) * possible values defined in net/netlink.h */ static struct nla_policy exmpl_genl_policy[__EXMPL_NLA_MAX] = { - [EXMPL_NLA_STR] = { .type = NLA_NUL_STRING }, + [EXMPL_NLA_DATA] = { .type = NLA_NUL_STRING }, + [EXMPL_NLA_LEN] = { .type = NLA_U32 }, }; -static struct genl_family exmpl_genl_family; +#define VERSION_NR 1 +static struct genl_family exmpl_gnl_family; -static int exmpl_cmd_echo(struct sk_buff *skb, struct genl_info *info) +static int get_length(struct sk_buff *request, struct genl_info *info) { - struct sk_buff *skb_out; + struct sk_buff *reply; + char *buffer; void *msg_head; - const char *msg; - if (!info->attrs[EXMPL_NLA_STR]) + if (info == NULL) return -EINVAL; - msg = nla_data(info->attrs[EXMPL_NLA_STR]); + if (!info->attrs[EXMPL_NLA_DATA]) + return -EINVAL; - pr_info("exmpl_genl: received: %s\n", msg); + /* get the data */ + buffer = nla_data(info->attrs[EXMPL_NLA_DATA]); - skb_out = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL); - if (!skb_out) + /* send a message back*/ + /* allocate some memory, since the size is not yet known use NLMSG_GOODSIZE*/ + reply = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL); + if (reply == NULL) return -ENOMEM; - msg_head = genlmsg_put(skb_out, info->snd_portid, info->snd_seq, - &exmpl_genl_family, 0, EXMPL_CMD_ECHO); - if (!msg_head) { - nlmsg_free(skb_out); + /* start the message */ + msg_head = genlmsg_put_reply(reply, info, &exmpl_gnl_family, 0, info->genlhdr->cmd); + if (msg_head == NULL) { return -ENOMEM; } - if (nla_put_string(skb_out, EXMPL_NLA_STR, msg)) { - nlmsg_free(skb_out); - return -EMSGSIZE; - } + /* add a EXMPL_LEN attribute -- report the data length */ + if (0 != nla_put_u32(reply, EXMPL_NLA_LEN, strlen(buffer))) + return -EINVAL; + + /* finalize the message */ + genlmsg_end(reply, msg_head); + + /* send the message back */ + if (0 != genlmsg_reply(reply, info)) + return -EINVAL; - genlmsg_end(skb_out, msg_head); - return genlmsg_reply(skb_out, info); + return 0; } /* commands: mapping between commands and actual functions*/ -static const struct genl_ops exmpl_genl_ops_echo[] = { +static const struct genl_ops exmpl_gnl_ops_echo[] = { { - .cmd = EXMPL_CMD_ECHO, - .flags = 0, + .cmd = EXMPL_CMD_MSG, .policy = exmpl_genl_policy, - .doit = exmpl_cmd_echo, + .doit = get_length, }, }; /* family definition */ -static struct genl_family exmpl_genl_family __ro_after_init = { - .name = EXMPL_GENL_FAMILY_NAME, //the name of this family, used by userspace application - .version = EXMPL_GENL_VERSION, //version number - .maxattr = EXMPL_NLA_MAX, +static struct genl_family exmpl_gnl_family __ro_after_init = { + .name = "EXMPL_GENL", //the name of this family, used by userspace application + .version = VERSION_NR, //version number + .maxattr = __EXMPL_NLA_MAX - 1, .module = THIS_MODULE, - .ops = exmpl_genl_ops_echo, - .n_ops = ARRAY_SIZE(exmpl_genl_ops_echo), + .ops = exmpl_gnl_ops_echo, + .n_ops = ARRAY_SIZE(exmpl_gnl_ops_echo), }; -static int __init exmpl_genl_init(void) +static int __init exmpl_gnl_init(void) { int rc; - rc = genl_register_family(&exmpl_genl_family); + rc = genl_register_family(&exmpl_gnl_family); if (rc != 0) { printk(KERN_INFO "rkmod: genl_register_family failed %d\n", rc); return 1; } - printk(KERN_INFO "Generic netlink example loaded, protocol version %d\n", EXMPL_GENL_VERSION); + printk(KERN_INFO "Generic netlink example loaded, protocol version %d\n", VERSION_NR); return 0; } -static void __exit exmpl_genl_exit(void) +static void __exit exmpl_gnl_exit(void) { int ret; /*unregister the family*/ - ret = genl_unregister_family(&exmpl_genl_family); + ret = genl_unregister_family(&exmpl_gnl_family); if(ret !=0){ printk("unregister family %i\n",ret); } } -module_init(exmpl_genl_init); -module_exit(exmpl_genl_exit); +module_init(exmpl_gnl_init); +module_exit(exmpl_gnl_exit); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Simple generic netlink echo module"); diff --git a/examples/generic/netl.py b/examples/generic/netl.py index 64a08492f..8f0e36770 100755 --- a/examples/generic/netl.py +++ b/examples/generic/netl.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 -from pyroute2.netlink import NLM_F_REQUEST, genlmsg +import traceback +from pyroute2.netlink import NLM_F_REQUEST +from pyroute2.netlink import genlmsg from pyroute2.netlink.generic import GenericNetlinkSocket -EXMPL_CMD_UNSPEC = 0 -EXMPL_CMD_ECHO = 1 + +RLINK_CMD_UNSPEC = 0 +RLINK_CMD_REQ = 1 class rcmd(genlmsg): @@ -13,20 +16,33 @@ class rcmd(genlmsg): with the kernel module ''' - nla_map = (('EXMPL_NLA_UNSPEC', 'none'), ('EXMPL_NLA_STR', 'asciiz')) + nla_map = ( + ('RLINK_ATTR_UNSPEC', 'none'), + ('RLINK_ATTR_DATA', 'asciiz'), + ('RLINK_ATTR_LEN', 'uint32'), + ) -class Exmpl(GenericNetlinkSocket): +class Rlink(GenericNetlinkSocket): def send_data(self, data): msg = rcmd() - msg['cmd'] = EXMPL_CMD_ECHO + msg['cmd'] = RLINK_CMD_REQ msg['version'] = 1 - msg['attrs'] = [('EXMPL_NLA_STR', data)] + msg['attrs'] = [('RLINK_ATTR_DATA', data)] ret = self.nlm_request(msg, self.prid, msg_flags=NLM_F_REQUEST)[0] - return ret.get_attr('EXMPL_NLA_STR') + return ret.get_attr('RLINK_ATTR_LEN') if __name__ == '__main__': - with Exmpl() as exmpl: - exmpl.bind('ECHO_GENL', rcmd) - print(exmpl.send_data('hello world')) + try: + # create protocol instance + rlink = Rlink() + rlink.bind('EXMPL_GENL', rcmd) + # request a method + print(rlink.send_data('x' * 65000)) + except: + # if there was an error, log it to the console + traceback.print_exc() + finally: + # finally -- release the instance + rlink.close() diff --git a/examples/ipq.py b/examples/ipq.py index 70c537e2d..79949888e 100644 --- a/examples/ipq.py +++ b/examples/ipq.py @@ -1,8 +1,7 @@ -from dpkt.ip import IP - -from pyroute2 import IPQSocket from pyroute2.common import hexdump +from pyroute2 import IPQSocket from pyroute2.netlink.ipq import NF_ACCEPT +from dpkt.ip import IP ip = IPQSocket() ip.bind() diff --git a/examples/iproute/ip_monitor.py b/examples/iproute/ip_monitor.py index 405cd5f1c..58a3d870c 100644 --- a/examples/iproute/ip_monitor.py +++ b/examples/iproute/ip_monitor.py @@ -2,9 +2,8 @@ Simplest example to monitor Netlink events with a Python script. ''' -from pprint import pprint - from pyroute2 import IPRSocket +from pprint import pprint ip = IPRSocket() ip.bind() diff --git a/examples/iproute/socketcan.py b/examples/iproute/socketcan.py index 59af905a6..2b198f1d0 100644 --- a/examples/iproute/socketcan.py +++ b/examples/iproute/socketcan.py @@ -15,9 +15,7 @@ ip_route.link('set', index=idx, state='down') # set CAN birate - ip_route.link( - 'set', index=idx, kind='can', can_bittiming={'bitrate': 250000} - ) + ip_route.link('set', index=idx, kind='can', can_bittiming={'bitrate': 250000 }) # bring can0 interface up ip_route.link('set', index=idx, state='up') diff --git a/examples/ipset.py b/examples/ipset.py index 8330641b0..d6f555ea4 100644 --- a/examples/ipset.py +++ b/examples/ipset.py @@ -1,6 +1,5 @@ import socket - -from pyroute2.ipset import IPSet, PortEntry, PortRange +from pyroute2.ipset import IPSet, PortRange, PortEntry ipset = IPSet() ipset.create("foo", stype="hash:ip") diff --git a/examples/kobject_uevent.py b/examples/kobject_uevent.py index c75d6e810..a024b7249 100644 --- a/examples/kobject_uevent.py +++ b/examples/kobject_uevent.py @@ -1,5 +1,4 @@ from pprint import pprint - from pyroute2 import UeventSocket kus = UeventSocket() diff --git a/examples/lab/README.rst b/examples/lab/README.rst new file mode 100644 index 000000000..e2f3f41fc --- /dev/null +++ b/examples/lab/README.rst @@ -0,0 +1,15 @@ +The pyroute2 lab is a collection if interactive tutorials and code samples. + +The online version of the lab is here: https://lab.pyroute2.org/ + +To run the lab examples in the command line, you can use nox: + +.. code-block:: shell + + # init the environment from scratch, rebuild the project + # and install all the dependencies: + nox -e lab + + # reuse the environment, only build the sphinx project + # and run the tests + nox -e lab -r -- '{"reuse": true}' diff --git a/examples/lab/iproute_get_addr/README.rst b/examples/lab/iproute_get_addr/README.rst new file mode 100644 index 000000000..2ecfb1139 --- /dev/null +++ b/examples/lab/iproute_get_addr/README.rst @@ -0,0 +1,7 @@ +Create an `IPRoute` object and list addresses. + +Conditions: + +* create exactly on `IPRoute()` object +* run `get_addr()` at least once +* run `close()` exactly once diff --git a/examples/lab/iproute_get_addr/check.py b/examples/lab/iproute_get_addr/check.py new file mode 100644 index 000000000..3d9040baf --- /dev/null +++ b/examples/lab/iproute_get_addr/check.py @@ -0,0 +1,11 @@ +from setup import lab +from task import IPRoute + +if len(lab.registry) != 1: + raise AssertionError('expected exactly one IPRoute instance') +if not isinstance(lab.registry[0], IPRoute): + raise AssertionError('expected IPRoute instance') + +ipr = lab.registry[0] +ipr.get_addr.assert_called() +ipr.close.assert_called_once() diff --git a/examples/lab/iproute_get_addr/setup.py b/examples/lab/iproute_get_addr/setup.py new file mode 100644 index 000000000..ee913ab6f --- /dev/null +++ b/examples/lab/iproute_get_addr/setup.py @@ -0,0 +1,4 @@ +from pyroute2 import lab + +lab.use_mock = True +lab.registry = [] diff --git a/examples/lab/iproute_get_addr/task.py b/examples/lab/iproute_get_addr/task.py new file mode 100644 index 000000000..600b88594 --- /dev/null +++ b/examples/lab/iproute_get_addr/task.py @@ -0,0 +1,8 @@ +from pyroute2 import IPRoute + +ipr = IPRoute() + +for msg in ipr.get_addr(): + print(msg) + +ipr.close() diff --git a/examples/lab/iproute_get_attr/README.rst b/examples/lab/iproute_get_attr/README.rst new file mode 100644 index 000000000..62cdfaafb --- /dev/null +++ b/examples/lab/iproute_get_attr/README.rst @@ -0,0 +1,7 @@ +Access netlink message attributes. + +* `get_links()` returns an iterator over link objects +* `msg.get('index')` returns `index` field just like `msg['index']` does +* `msg.get('ifname')` returns `IFLA_IFNAME` value as a string +* `msg.get('af_spec')` returns `IFLA_AF_SPEC` as a dict +* `msg.get(('af_spec', 'af_inet', 'forwarding'))` as an int diff --git a/examples/lab/iproute_get_attr/check.py b/examples/lab/iproute_get_attr/check.py new file mode 100644 index 000000000..f509abdb5 --- /dev/null +++ b/examples/lab/iproute_get_attr/check.py @@ -0,0 +1,9 @@ +from setup import lab +from task import IPRoute + +ipr = lab.registry[0] +if not isinstance(ipr, IPRoute): + raise AssertionError('expected IPRoute instance') + +if not ipr.close.called: + print('\nWARNING: it is recommended to close IPRoute instances') diff --git a/examples/lab/iproute_get_attr/setup.py b/examples/lab/iproute_get_attr/setup.py new file mode 100644 index 000000000..ee913ab6f --- /dev/null +++ b/examples/lab/iproute_get_attr/setup.py @@ -0,0 +1,4 @@ +from pyroute2 import lab + +lab.use_mock = True +lab.registry = [] diff --git a/examples/lab/iproute_get_attr/task.py b/examples/lab/iproute_get_attr/task.py new file mode 100644 index 000000000..7cf2fd377 --- /dev/null +++ b/examples/lab/iproute_get_attr/task.py @@ -0,0 +1,12 @@ +from pyroute2 import IPRoute + +ipr = IPRoute() + +print('iterate network interfaces\n') +for msg in ipr.get_links(): + index = msg.get('index') + ifname = msg.get('ifname') + forwarding = msg.get(('af_spec', 'af_inet', 'forwarding')) + print(f'{index}: {ifname}: forwarding = {forwarding}') + +ipr.close() diff --git a/examples/ndb/create_bond.py b/examples/ndb/create_bond.py index d582a3473..f0d2912a7 100644 --- a/examples/ndb/create_bond.py +++ b/examples/ndb/create_bond.py @@ -8,6 +8,7 @@ with NDB() as ndb: + # The same scheme works for bridge interfaces too: you # can create a bridge interface and assign ports to it # just as below. diff --git a/examples/ndb/create_interface.py b/examples/ndb/create_interface.py index 35f8ce586..65d8e5ff3 100644 --- a/examples/ndb/create_interface.py +++ b/examples/ndb/create_interface.py @@ -1,6 +1,7 @@ from pyroute2 import NDB from pyroute2.common import uifname + with NDB() as ndb: # dummy, bridge and bond interfaces are created in the @@ -18,4 +19,8 @@ .commit() ) print(ndb.interfaces[ifname].show('json')) - (ndb.interfaces[ifname].remove().commit()) + ( + ndb.interfaces[ifname] + .remove() + .commit() + ) diff --git a/examples/ndb/create_vlan.py b/examples/ndb/create_vlan.py index 4e3ad0660..17b8e6522 100644 --- a/examples/ndb/create_vlan.py +++ b/examples/ndb/create_vlan.py @@ -7,6 +7,7 @@ with NDB() as ndb: + ( ndb.interfaces.create(ifname=vlan_host, kind='dummy') .set('state', 'up') @@ -17,7 +18,7 @@ ifname=vlan_interface, kind='vlan', link=ndb.interfaces[vlan_host], - vlan_id=101, + vlan_id=101 ) .set('mtu', 1400) .set('state', 'up') diff --git a/examples/ndb/keystone_auth.py b/examples/ndb/keystone_auth.py new file mode 100644 index 000000000..3d2ea2f92 --- /dev/null +++ b/examples/ndb/keystone_auth.py @@ -0,0 +1,112 @@ +''' +:test:argv:14080769fe05e1f8b837fb43ca0f0ba4 + +A simplest example of a custom AuthManager and its usage +with `AuthProxy` objects. + +Here we authenticate the auth token against Keystone and +allow any NDB operations until it is expired. + +One can get such token with a curl request:: + + $ cat request.json + { "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "name": "admin", + "domain": { "name": "admin_domain" }, + "password": "secret" + } + } + }, + "scope": { + "project": { + "id": "f0af12d451fb4bccbb38217e7f9afe9a" + } + } + } + } + + $ curl -i \ + -H "Content-Type: application/json" \ + -d "@request.json" \ + http://keystone:5000/v3/auth/tokens + +`X-Subject-Token` header in the response will be the token we need. Say we +get `14080769fe05e1f8b837fb43ca0f0ba4` as `X-Subject-Token`. Then you can +run:: + + $ . openstack.rc # <-- your OpenStack APIv3 RC file + $ export PYTHONPATH=`pwd` + $ python3 examples/ndb/keystone_auth.py 14080769fe05e1f8b837fb43ca0f0ba4 + +Using this example you can implement services that export NDB via any RPC, +e.g. HTTP, and use Keystone integration. Same scheme may be used for any +other Auth API, be it RADIUS or like that. + +An example of a simple HTTP service you can find in /cli/pyroute2-cli. +''' + +import os +import sys +import time +from dateutil.parser import parse as isodate +from keystoneauth1.identity import v3 +from keystoneauth1 import session +from keystoneclient.v3 import client as ksclient +from keystoneclient.v3.tokens import TokenManager +from pyroute2 import NDB + + +class OSAuthManager(object): + def __init__(self, token, log): + # create a Keystone password object + auth = v3.Password( + auth_url=os.environ.get('OS_AUTH_URL'), + username=os.environ.get('OS_USERNAME'), + password=os.environ.get('OS_PASSWORD'), + user_domain_name=(os.environ.get('OS_USER_DOMAIN_NAME')), + project_id=os.environ.get('OS_PROJECT_ID'), + ) + # create a session object + sess = session.Session(auth=auth) + # create a token manager + tmanager = TokenManager(ksclient.Client(session=sess)) + # validate the token + keystone_response = tmanager.validate(token) + # init attrs + self.log = log + self.expire = isodate(keystone_response['expires_at']).timestamp() + + def check(self, obj, tag): + # + # totally ignore obj and tag, validate only token expiration + # + # problems to be solved before you use this code in production: + # 1. access levels: read-only, read-write -- match tag + # 2. how to deal with revoked tokens + # + if time.time() > self.expire: + self.log.error('%s permission denied' % (tag,)) + raise PermissionError('keystone token has been expired') + + self.log.info('%s permission granted' % (tag,)) + return True + + +with NDB(log='debug') as ndb: + # create a utility log channel + log = ndb.log.channel('main') + + # create an AuthManager-compatible object + log.info('request keystone auth') + am = OSAuthManager(sys.argv[1], ndb.log.channel('keystone')) + log.info('keystone auth complete, expires %s' % am.expire) + + # create an auth proxy for this particular token + ap = ndb.auth_proxy(am) + + # validate access via that proxy + print(ap.interfaces['lo']) diff --git a/examples/ndb/radius_auth.py b/examples/ndb/radius_auth.py new file mode 100644 index 000000000..0feb4bc0c --- /dev/null +++ b/examples/ndb/radius_auth.py @@ -0,0 +1,81 @@ +''' +:test:argv:testing +:test:argv:secret +:test:environ:RADIUS_SERVER=127.0.0.1 +:test:environ:RADIUS_SECRET=secret + +An example of using RADIUS authentication with NDB. + +In order to run the example you can setup a FreeRADIUS server:: + + # /etc/raddb/clients + client test { + ipaddr = 192.168.122.101 # IP addr of your client + secret = s3cr3t + } + + # /etc/raddb/users + testing Cleartext-Password := "secret" + +Then setup your client:: + + # download RADIUS dictionaries + $ export GITSERVER=https://raw.githubusercontent.com + $ export DICTPATH=pyradius/pyrad/master/example + $ wget $GITSERVER/$DICTPATH/dictionary + $ wget $GITSERVER/$DICTPATH/dictionary.freeradius + + # setup the environment + $ cat radius.rc + export RADIUS_SERVER=192.168.122.1 + export RADIUS_SECRET=s3cr3t + export PYTHONPATH=`pwd` + + $ . radius.rc + $ python3 examples/ndb/radius_auth.py testing secret + +''' + +import os +import sys +from pyrad.client import Client +from pyrad.dictionary import Dictionary +import pyrad.packet +from pyroute2 import NDB + + +class RadiusAuthManager(object): + def __init__(self, user, password, log): + client = Client( + server=os.environ.get('RADIUS_SERVER'), + secret=os.environ.get('RADIUS_SECRET').encode('ascii'), + dict=Dictionary('dictionary'), + ) + req = client.CreateAuthPacket( + code=pyrad.packet.AccessRequest, User_Name=user + ) + req['User-Password'] = req.PwCrypt(password) + reply = client.SendPacket(req) + self.auth = reply.code + self.log = log + + def check(self, obj, tag): + # + self.log.info('%s access' % (tag,)) + return self.auth == pyrad.packet.AccessAccept + + +with NDB(log='debug') as ndb: + # create a utility log channel + log = ndb.log.channel('main') + + # create an AuthManager-compatible object + log.info('request radius auth') + am = RadiusAuthManager(sys.argv[1], sys.argv[2], ndb.log.channel('radius')) + log.info('radius auth complete') + + # create an auth proxy for these credentials + ap = ndb.auth_proxy(am) + + # validate access via that proxy + print(ap.interfaces['lo']) diff --git a/examples/nftables_sets.py b/examples/nftables_sets.py index be7a8c929..670a147aa 100644 --- a/examples/nftables_sets.py +++ b/examples/nftables_sets.py @@ -1,20 +1,15 @@ import time from pyroute2.netlink.nfnetlink.nftsocket import NFPROTO_IPV4 -from pyroute2.nftables.main import NFTables, NFTSetElem +from pyroute2.nftables.main import NFTables +from pyroute2.nftables.main import NFTSetElem def test_ipv4_addr_set(): with NFTables(nfgen_family=NFPROTO_IPV4) as nft: nft.table("add", name="filter") - my_set = nft.sets( - "add", - table="filter", - name="test0", - key_type="ipv4_addr", - comment="my test set", - timeout=0, - ) + my_set = nft.sets("add", table="filter", name="test0", key_type="ipv4_addr", + comment="my test set", timeout=0) # With str nft.set_elems( @@ -32,21 +27,19 @@ def test_ipv4_addr_set(): ) try: - assert { - e.value - for e in nft.set_elems("get", table="filter", set="test0") - } == {"10.2.3.4", "10.4.3.2", "9.9.9.9"} - assert ( - nft.sets("get", table="filter", name="test0").comment - == b"my test set" - ) + assert {e.value for e in nft.set_elems("get", table="filter", set="test0")} == { + "10.2.3.4", + "10.4.3.2", + "9.9.9.9", + } + assert nft.sets("get", table="filter", name="test0").comment == b"my test set" time.sleep(1.2) # timeout for elem 9.9.9.9 (1000ms) - assert { - e.value - for e in nft.set_elems("get", table="filter", set="test0") - } == {"10.2.3.4", "10.4.3.2"} + assert {e.value for e in nft.set_elems("get", table="filter", set="test0")} == { + "10.2.3.4", + "10.4.3.2", + } finally: nft.sets("del", table="filter", name="test0") nft.table("del", name="filter") diff --git a/examples/policy/policy.py b/examples/policy/policy.py index 1a2435eb4..b6b875554 100755 --- a/examples/policy/policy.py +++ b/examples/policy/policy.py @@ -2,7 +2,6 @@ import traceback from pprint import pprint - from pyroute2.netlink.generic import GenericNetlinkSocket if __name__ == '__main__': diff --git a/examples/processes/pmonitor.py b/examples/processes/pmonitor.py index 7050a9543..5b6a46c8e 100644 --- a/examples/processes/pmonitor.py +++ b/examples/processes/pmonitor.py @@ -1,7 +1,6 @@ ''' Monitor process exit ''' - from pyroute2 import TaskStats from pyroute2.common import hexdump diff --git a/examples/processes/taskstats.py b/examples/processes/taskstats.py index 02761bdba..c5dae11f7 100644 --- a/examples/processes/taskstats.py +++ b/examples/processes/taskstats.py @@ -1,9 +1,7 @@ ''' Simple taskstats sample. ''' - import os - from pyroute2 import TaskStats pid = os.getpid() diff --git a/examples/pyroute2-cli/comments b/examples/pyroute2-cli/comments new file mode 100644 index 000000000..55ffb0e10 --- /dev/null +++ b/examples/pyroute2-cli/comments @@ -0,0 +1,17 @@ +#!/usr/bin/env pyroute2-cli +# +! Test mixed comments, both ! and # +# +interfaces # ... tail comments + ! + # ... indented comments + ! + create {ifname test01, kind dummy, address 00:11:22:33:44:55} + commit + ! + test01 + # + show + ! + remove + commit diff --git a/examples/pyroute2-cli/create_bridge b/examples/pyroute2-cli/create_bridge new file mode 100644 index 000000000..ae23073bf --- /dev/null +++ b/examples/pyroute2-cli/create_bridge @@ -0,0 +1,70 @@ +#!/usr/bin/env pyroute2-cli +# +interfaces + # + # ports + create ifname br0p0, kind dummy, state up | commit + create ifname br0p1, kind dummy, state up | commit + # + # bridge + create ifname br0, kind bridge, state up + br_stp_state 1 + br_forward_delay 1500 + address 00:11:22:33:44:55 + add_port br0p0 + add_port br0p1 + add_ip 10.251.0.1/24 + add_ip 10.251.0.2/24 + commit + +# +# commas between properies are not required, if properties +# are in pairs like { name0 value0 name1 value1 } +routes + create dst 10.100.0.0/24 gateway 10.251.0.10 | commit + create dst 10.101.0.0/24 gateway 10.251.0.10 | commit + +# +# the pipe operator | connects calls on the same object, +# these two statements are equal: +# +# interfaces create { ifname test, kind dummy } | commit +# +# interfaces create { ifname test, kind dummy } +# commit +interfaces + br0 remove | commit + br0p0 remove | commit + br0p1 remove | commit + + +interfaces + # + # you can use more explicit syntax with properties + # specified within braces + # + # => is a synonym of | , use any variant you like more + # + create { ifname br0p0, kind dummy, state up } => commit + create { ifname br0p1, kind dummy, state up } => commit + create { ifname br0, kind bridge } + set { state up } + set { br_stp_state 1 } + set { br_forward_delay 1500 } + set { address 00:11:22:33:44:55 } + add_port { br0p0 } + add_port { br0p1 } + add_ip { address 10.251.0.1, prefixlen 24 } + add_ip { address 10.251.0.2, prefixlen 24 } + commit + +routes + create { dst 10.100.0.0/24, gateway 10.251.0.10 } => commit + create { dst 10.101.0.0/24, gateway 10.251.0.10 } => commit + +# +# run cleanup +interfaces + br0 remove => commit + br0p0 remove => commit + br0p1 remove => commit diff --git a/examples/pyroute2-cli/create_dummy b/examples/pyroute2-cli/create_dummy new file mode 100644 index 000000000..c54b5ddb7 --- /dev/null +++ b/examples/pyroute2-cli/create_dummy @@ -0,0 +1,31 @@ +#!/usr/bin/env pyroute2-cli +# +# +interfaces + # create a dummy interface + # + # there the very minimal spec consists of ifname, + # other properties may be set later + # + create {ifname test01} + # + # set properties + kind dummy + address 00:11:22:33:44:55 + commit + # + # create addresses ... + ipaddr + create {address 192.168.15.67, prefixlen 24} + commit + create {address 192.168.15.68, prefixlen 24} + commit + # + # and remove one of them + 192.168.15.68/24 + remove + commit + # + # remove the interface + remove + commit diff --git a/examples/pyroute2-cli/dump_lo b/examples/pyroute2-cli/dump_lo new file mode 100644 index 000000000..3a7285a81 --- /dev/null +++ b/examples/pyroute2-cli/dump_lo @@ -0,0 +1,5 @@ +#!/usr/bin/env pyroute2-cli +! +! Just dump the loopback interface. +! +interfaces lo show diff --git a/examples/wifi/nl80211_interface_type.py b/examples/wifi/nl80211_interface_type.py index 7f31269f3..2e640cea0 100644 --- a/examples/wifi/nl80211_interface_type.py +++ b/examples/wifi/nl80211_interface_type.py @@ -1,6 +1,6 @@ import errno - -from pyroute2 import IW, IPRoute +from pyroute2 import IW +from pyroute2 import IPRoute from pyroute2.netlink.exceptions import NetlinkError # interface name to check diff --git a/examples/wifi/nl80211_monitor.py b/examples/wifi/nl80211_monitor.py index 933bdc10e..e7935d2e2 100644 --- a/examples/wifi/nl80211_monitor.py +++ b/examples/wifi/nl80211_monitor.py @@ -1,5 +1,6 @@ from pyroute2 import IW + # register IW to get all the messages iw = IW(groups=0xFFF) print(iw.get()) diff --git a/examples/wifi/nl80211_scan_dump.py b/examples/wifi/nl80211_scan_dump.py index 8778978da..591b054e0 100644 --- a/examples/wifi/nl80211_scan_dump.py +++ b/examples/wifi/nl80211_scan_dump.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 -import logging import sys +import logging from pyroute2 import IPRoute + from pyroute2.iwutil import IW -from pyroute2.netlink import NLM_F_DUMP, NLM_F_REQUEST -from pyroute2.netlink.nl80211 import NL80211_NAMES, nl80211cmd +from pyroute2.netlink import NLM_F_REQUEST +from pyroute2.netlink import NLM_F_DUMP +from pyroute2.netlink.nl80211 import nl80211cmd +from pyroute2.netlink.nl80211 import NL80211_NAMES logging.basicConfig(level=logging.DEBUG) diff --git a/examples/wifi/nl80211_set_type.py b/examples/wifi/nl80211_set_type.py index 3fe44acb6..39d65455c 100644 --- a/examples/wifi/nl80211_set_type.py +++ b/examples/wifi/nl80211_set_type.py @@ -1,5 +1,8 @@ -from pyroute2 import IW, IPRoute +import errno +from pyroute2 import IW +from pyroute2 import IPRoute from pyroute2.netlink.exceptions import NetlinkError +from pyroute2.netlink.nl80211 import IFTYPE_NAMES # interface name to check ifname = 'wlx2' diff --git a/lab/Makefile b/lab/Makefile new file mode 100644 index 000000000..d4bb2cbb9 --- /dev/null +++ b/lab/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/pyroute2/decoder/__init__.py b/lab/_static/classic.css similarity index 100% rename from pyroute2/decoder/__init__.py rename to lab/_static/classic.css diff --git a/lab/_static/custom.css b/lab/_static/custom.css new file mode 100644 index 000000000..0acec5d37 --- /dev/null +++ b/lab/_static/custom.css @@ -0,0 +1,349 @@ +/* + * + * Sphinx stylesheet based on the default theme. + * + * :copyright: Copyright 2012 by Peter V. Saveliev + * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: sans-serif; + font-size: 100%; + color: #000; + margin: 0; + padding: 0; +} + +div.document { +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { +} + +div.body { + background-color: #ffffff; + color: #000000; + padding: 0 20px 30px 20px; + max-width: 45em; + min-width: 10em; + padding: 2em; +} + +div.footer { + color: #ffffff; + width: 100%; + padding: 9px 0 9px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: #ffffff; + text-decoration: underline; +} + +div.related { + background-color: #fafafa; + line-height: 30px; + border-top: 1px solid #c0c0c0; + border-bottom: 1px solid #c0c0c0; +} + +div.sphinxsidebar { + display: none; +} + +div.sphinxsidebar h3 { + color: #355f7c; + font-size: 1.2em; + font-weight: normal; + margin: 0; + padding: 0; +} + +div.sphinxsidebar h4 { + color: #355f7c; + font-size: 1.2em; + font-weight: normal; + margin: 5px 0 0 0; + padding: 0; +} + +div.sphinxsidebar p { + color: #ffffff; +} + +div.sphinxsidebar p.topless { + margin: 5px 10px 10px 10px; +} + +div.sphinxsidebar ul { + margin: 10px; + padding: 0; + color: #ffffff; +} + +div.sphinxsidebar a { +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + + + +/* -- hyperlink styles ------------------------------------------------------ */ + +a { + color: #355f7c; + text-decoration: none; +} + +a:visited { + color: #355f7c; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + + + +/* -- body styles ----------------------------------------------------------- */ + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-weight: normal; + color: #20435c; + border-bottom: 1px solid #ccc; + margin: 20px -20px 10px -20px; + padding: 3px 0 3px 10px; +} + +div.body h1 { margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 160%; } +div.body h3 { font-size: 140%; } +div.body h4 { font-size: 120%; } +div.body h5 { font-size: 110%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: #c60f0f; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: #c60f0f; + color: white; +} + +img.align-right { + position: absolute; + right: 50px; + top: 50px; +} + +div.body p, div.body dd, div.body li { + text-align: justify; + line-height: 130%; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.admonition p { + margin-bottom: 5px; +} + +div.admonition pre { + margin-bottom: 5px; +} + +div.admonition ul, div.admonition ol { + margin-bottom: 5px; +} + +p.first { + margin: 2px; +} + +dl.class, dl.exception { + border-bottom: 1px dashed #ccc; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; + border-radius: 10px; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +cite { + background-color: #ececec; + color: #333333; + font-family: monospace; + font-style: normal; +} + +pre { + padding: 5px; + color: #333333; + line-height: 120%; + border: 1px solid #ac9; + border-left: none; + border-right: none; + border-radius: 10px; +} + +tt { + background-color: #ecf0f3; + padding: 0 1px 0 1px; + font-size: 0.95em; +} + +th { + background-color: #ede; +} + +.warning tt { + background: #efc2c2; +} + +.note tt { + background: #d6d6d6; +} + +.viewcode-back { + font-family: sans-serif; +} + +div.viewcode-block:target { + background-color: #f4debf; + border-top: 1px solid #ac9; + border-bottom: 1px solid #ac9; +} + +.highlight { + background: #f9f9f9; +} + +dl.method > dt { + margin-bottom: 1em; + padding: 1em; +} + +dl.class > dt { + margin-bottom: 1em; + padding: 1em; +} + +div.aafig-caption { + width: 100%; +} + +div.aafig-caption p { + text-align: center; + color: #a0a0a0; +} + + +.exercise { + width: 100%; +} + +.loading { + background: #f0f0f0; + color: #909090; + width: 99%; +} + +.loaded { + background: #ffffff; + color: #000000; + width: 99%; +} + +.button, .button-right { + padding: 1em; + width: 10em; + border: solid 1px #000000; + cursor: pointer; +} + +.button-right { + float: right; +} + +section section { + display: none; +} + +.hidden { + display: none; +} + +#dmesg { + border: dashed 1px #c0c0c0; + display: block; +} + +.pyroute2-log-record { + display: block; + color: #909090; + margin: 0px; + padding: 2px; + padding-left: 1em; +} + +.pyroute2-example-description { + border: solid 1px #c0c0c0; + background-color: #ffffe0; + font-family: monospace; +} + +.pyroute2-example-description::before { + content: "Example:"; + font-weight: bold; +} diff --git a/lab/_static/fixup.js b/lab/_static/fixup.js new file mode 100644 index 000000000..37322986f --- /dev/null +++ b/lab/_static/fixup.js @@ -0,0 +1,10 @@ +window.addEventListener("load", function() { + Array.from( + document.getElementsByTagName("img") + ).map( + function(img) { + img.removeAttribute("width"); + img.removeAttribute("height"); + } + ) +}) diff --git a/lab/_static/lab.js b/lab/_static/lab.js new file mode 100644 index 000000000..eb98fb7c5 --- /dev/null +++ b/lab/_static/lab.js @@ -0,0 +1,139 @@ + +const pyroute2_lab_context = { + log_buffer: [], + log_size_max: 16, + pyodide: null, + python_namespace: null, + python_loaded: false, + bootstrap: ` +import io +import micropip +import sys +import pprint +import builtins + +await micropip.install("${pyroute2_base_url}/${pyroute2_distfile}") + +from pyroute2.netlink import nlmsg + + +def print(*argv, end='\\n'): + for data in argv: + if isinstance(data, nlmsg): + pprint.pprint(data.dump()) + elif isinstance(data, (str, int, float)): + builtins.print(data, end='') + else: + pprint.pprint(data) + builtins.print(' ', end='') + builtins.print(end=end) +`, + exercise_pre: "sys.stdout = io.StringIO()", + exercise_post: "result = sys.stdout.getvalue()", +}; + + +function pyroute2_log_record(argv) { + let ctime = new Date(); + pyroute2_lab_context.log_buffer.push([ctime, argv]); + if (pyroute2_lab_context.log_buffer.length > pyroute2_lab_context.log_size_max) { + pyroute2_lab_context.log_buffer.shift(); + }; + dmesg = document.getElementById("dmesg"); + if (dmesg) { + let log_output = ""; + pyroute2_lab_context.log_buffer.map(function (x) { + log_output += `${x[1]}`; + }); + dmesg.innerHTML = log_output; + }; +}; + +function pyroute2_escape_untrusted(data) { + return data.replace(/[<>&'"]/g, function (x) { + switch (x) { + case '<': return '<'; + case '>': return '>'; + case '&': return '&'; + case "'": return '''; + case '"': return '"'; + } + }); +} + +async function pyroute2_execute_example(name) { + let setup = document.getElementById(name + "-setup").value; + let task = document.getElementById(name + "-task").value; + let check = document.getElementById(name + "-check").value; + let data = ""; + let namespace = { globals: pyroute2_lab_context.python_namespace }; + let pyodide = pyroute2_lab_context.pyodide; + if (!pyroute2_lab_context.python_loaded) { + // if python is not loaded yet, wait a second... + await new Promise(resolve => setTimeout(resolve, 1000)); + // and try again + await pyroute2_execute_example(name); + return; + } else { + try { + pyodide.runPython(pyroute2_lab_context.exercise_pre, namespace); + pyodide.runPython(setup, namespace); + pyodide.runPython(task, namespace); + pyodide.runPython(check, namespace); + pyodide.runPython(pyroute2_lab_context.exercise_post, namespace); + data = pyroute2_lab_context.python_namespace.get("result"); + } catch(exception) { + data = `${exception}` + }; + }; + // recode untrusted output + data = pyroute2_escape_untrusted(data) + document.getElementById(name + "-data").innerHTML = `
${data}
`; +} + +function pyroute2_clear_example_output(name) { + document.getElementById(name + "-data").innerHTML = ""; +} + +async function pyroute2_lab_main() { + if (!document.getElementById("dmesg")) { + return; + }; + pyroute2_log_record("Booting the system, be patient"); + pyroute2_log_record("Starting python"); + let pyodide = null; + let namespace = null; + // try to load python + try { + pyodide = await loadPyodide(); + namespace = pyodide.globals.get("dict")(); + await pyodide.loadPackage("micropip"); + await pyodide.runPythonAsync(pyroute2_lab_context.bootstrap, { globals: namespace }); + } catch(exception) { + pyroute2_log_record(`
${exception}
`); + pyroute2_log_record("Please report this bug to the project bug tracker, and don't forget to specify your browser."); + return; + }; + // setup global context + pyroute2_lab_context.pyodide = pyodide; + pyroute2_lab_context.python_namespace = namespace; + // reset log + pyroute2_lab_context.log_buffer.length = 0; + pyroute2_lab_context.python_loaded = true; + // make exercises visible + Array.from( + document.getElementsByTagName("section") + ).map(function(x) { + x.style['display'] = 'block'; + }); + // unlock code blocks + Array.from( + document.getElementsByClassName("loading") + ).map(function(x) { + x.removeAttribute("readonly"); + x.className = "loaded"; + }); + pyroute2_log_record(`System loaded [ ${pyroute2_distfile} ]`); +}; + +window.addEventListener("load", pyroute2_lab_main); diff --git a/lab/_templates/conf.js b/lab/_templates/conf.js new file mode 100644 index 000000000..da6463a6b --- /dev/null +++ b/lab/_templates/conf.js @@ -0,0 +1,6 @@ +const pyroute2_distfile = "{{ distfile }}"; +let pyroute2_base_url = ""; + +if (window.hostname) { + pyroute2_base_url = `${window.protocol}//${window.hostname}`; +}; diff --git a/lab/_templates/form_template.html b/lab/_templates/form_template.html new file mode 100644 index 000000000..39db2c0cb --- /dev/null +++ b/lab/_templates/form_template.html @@ -0,0 +1,15 @@ +
{{ readme }}
+
+ + + + + +
+
diff --git a/lab/_templates/layout.html b/lab/_templates/layout.html new file mode 100644 index 000000000..bbdc2df8f --- /dev/null +++ b/lab/_templates/layout.html @@ -0,0 +1,30 @@ +{% extends "!layout.html" %} + +{% block extrahead %} + +{% endblock %} + +{%- macro c_relbar() %} + +{%- endmacro %} + +{% block relbar1 %} +{{ c_relbar() }} +{% endblock %} diff --git a/lab/conf.py b/lab/conf.py new file mode 100644 index 000000000..ae87244f8 --- /dev/null +++ b/lab/conf.py @@ -0,0 +1,23 @@ +import pyroute2 + +source_suffix = '.rst' +master_doc = 'index' + +project = 'lab.pyroute2' +copyright = '2022, Peter Saveliev' +author = 'Peter Saveliev' + +release = pyroute2.__version__ + + +extensions = [ +] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +html_theme = 'default' +html_css_files = ['custom.css'] +html_js_files = ['conf.js', 'lab.js', 'fixup.js'] +html_static_path = ['_static'] diff --git a/lab/index.rst b/lab/index.rst new file mode 100644 index 000000000..2a3443322 --- /dev/null +++ b/lab/index.rst @@ -0,0 +1,20 @@ +pyroute2 labs +============= + +Welcome to pyroute2 online labs, a collection of interactive tutorials. +The work is in progress, and the collection grows, so stay tuned. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + iproute + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/lab/iproute.rst b/lab/iproute.rst new file mode 100644 index 000000000..47e91ccaf --- /dev/null +++ b/lab/iproute.rst @@ -0,0 +1,74 @@ +IPRoute -- work with interfaces, addresses and routes +----------------------------------------------------- + +.. _dmesg: + +The lab requires JavaScript to be enabled, as it runs Python over JS. It +may be also incompatible with your browser, so consider using FireFox, +Chrome or like that. + + +Create IPRoute and get network objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`IPRoute` objects provide kernel RTNL API, just as Linux `iproute2` utility +does. The differences are that `IPRoute` provides a subset -- so your +contributions are more than welcome, -- and `IPRoute` returns parsed +netlink packets, only adding a few extra fields. + +.. note:: + Netlink protocol and packets structure: https://docs.pyroute2.org/netlink.html + + More about netlink packets structure and API see below in this lab. + +Let's start with a simple exercise: create an `IPRoute` object and get IP +addresses. Most of pyroute2 classes, that provide some netlink API, create a +netlink socket and allocate some other resources. Because of the netlink +protocol, there may be not more than 1024 netlink sockets opened at the same +time per process ID, and it may be important to close objects if they're not +in use. + +.. note:: + `IPRoute` methods always return iterables, as a response from the kernel always + is a series of `[1..x]` messages. + +To continue, run the exercise by pressing the `execute` button. + +.. raw:: html + :file: iproute_get_addr.html + +Access messages data +~~~~~~~~~~~~~~~~~~~~ + +All the messages returned by `IPRoute` methods provide standard `nlmsg/nla` +API. Every message is a recursive dict-like structure, with fields accessible +via `__getitem__()`, and an optional NLA list accessible via `msg['attrs']`, +as you can see in the exercise above. If a message or NLA has `value` field +defined, this field is being returned by `getvalue()` method, otherwise +`getvalue()` returns the message or NLA itself. This makes simple type +NLA retrieval a bit more convenient. + +These are methods to get fields and NLA values: + +* `__getitem__('field')` -- return a field of this name +* `.get('field')`, `get('NLA_TYPE')`, `.get(('NLA_TYPE', ..., 'field'))` -- + the universal get method, see below +* `.getvalue()` -- return the `value` field, if defined, otherwise the object + itself +* `.get_attr('NLA_TYPE')` -- get one NLA by the type; if there atr several + NLA of the same type, get the first one in the list +* `.get_attrs('NLA_NAME')` -- get a list of NLA of this type + +Some notes on `get()` method: + +* returns a field or NLA value +* case insensitive for NLA types +* if NLA has `prefix` defined, it allows type notation both with and + without the prefix, thus + `get('IFLA_IFNAME') == get('IFNAME') == get('ifname')` +* the method first looks up for an NLA, and only then for a field of this + name; if a message has both, like as `ndmsg` has a field `ifindex` and + NLA type `NDA_IFINDEX`, then you can use `__getitem__()` and `getattr()` + +.. raw:: html + :file: iproute_get_attr.html diff --git a/noxfile.py b/noxfile.py index b122c7bc6..a78610077 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,4 +1,3 @@ -import copy import getpass import json import os @@ -10,22 +9,18 @@ nox.options.envdir = f'./.nox-{getpass.getuser()}' nox.options.reuse_existing_virtualenvs = False nox.options.sessions = [ - 'linter-python3.9', - 'linter-python3.14', - 'ci-self-python3.9', - 'ci-self-python3.14', + 'linter', 'repo', 'unit', - 'limits', + 'lab', 'neutron', - 'process', 'integration', - 'core-python3.9', - 'core-python3.14', + 'linux-python3.8', 'linux-python3.9', - 'linux-python3.14', - 'minimal-python3.9', - 'minimal-python3.14', + 'linux-python3.10', + 'linux-python3.11', + 'linux-python3.12', + 'minimal', ] linux_kernel_modules = [ @@ -37,22 +32,9 @@ 'l2tp_ip', 'l2tp_eth', 'l2tp_netlink', - 'netdevsim', ] -def load_global_config(): - if sys.argv[-2] == '--' and len(sys.argv[-1]): - return json.loads(sys.argv[-1]) - return {} - - -global_config = load_global_config() -if global_config.get('fast'): - nox.options.reuse_venv = 'yes' - nox.options.no_install = True - - def add_session_config(func): '''Decorator to load the session config. @@ -71,7 +53,12 @@ def my_session_func(session, config): ''' def wrapper(session): - return func(session, global_config) + if session.posargs and len(session.posargs[0]) > 0: + config = json.loads(session.posargs[0]) + else: + config = {} + session.debug(f'session config: {config}') + return func(session, config) wrapper.__name__ = func.__name__ wrapper.__doc__ = func.__doc__ @@ -85,26 +72,20 @@ def options(module, config): 'python', '-m', 'pytest', - f'-r{config.get("summary", "x")}', - f'--timeout={config.get("timeout", 60)}', '--basetemp', './log', + '--exitfirst', + '--verbose', + '--junitxml=junit.xml', ] - if config.get('exitfirst', True): - ret.append('--exitfirst') - if config.get('verbose', True): - ret.append('--verbose') - if config.get('coverage', False): - ret.append('--cov=pyroute2') - ret.append('--cov-report=html') - if config.get('profile', False): - ret.append('--profile') - ret.append('--profile-svg') if config.get('fail_on_warnings'): ret.insert(1, 'error') ret.insert(1, '-W') if config.get('pdb'): ret.append('--pdb') + if config.get('coverage'): + ret.append('--cov-report=html') + ret.append('--cov=pyroute2') if config.get('tests_prefix'): module = f'{config["tests_prefix"]}/{module}' if config.get('sub'): @@ -135,51 +116,41 @@ def setup_linux(session): def setup_venv_minimal(session, config): if not config.get('reuse'): session.install('--upgrade', 'pip') - session.install('.[dev]') - session.install('.[docs]') + session.install('build') + session.install('twine') + session.install('-r', 'requirements.dev.txt') + session.install('-r', 'requirements.docs.txt') + session.run('mv', '-f', 'setup.cfg', '.setup.cfg.orig', external=True) session.run( - 'mv', '-f', 'pyproject.toml', '.pyproject.toml.full', external=True - ) - session.run( - 'mv', '-f', 'pyroute2/__init__.py', '.init.py.full', external=True - ) - session.run( - 'cp', 'pyproject.minimal.toml', 'pyproject.toml', external=True + 'mv', '-f', 'pyroute2/__init__.py', '.init.py.orig', external=True ) + session.run('cp', 'setup.minimal.cfg', 'setup.cfg', external=True) session.run( 'cp', 'pyroute2/minimal.py', 'pyroute2/__init__.py', external=True ) session.run('python', '-m', 'build') session.run('python', '-m', 'twine', 'check', 'dist/*') session.install('.') + session.run('mv', '-f', '.setup.cfg.orig', 'setup.cfg', external=True) session.run( - 'mv', '-f', '.pyproject.toml.full', 'pyproject.toml', external=True - ) - session.run( - 'mv', '-f', '.init.py.full', 'pyroute2/__init__.py', external=True + 'mv', '-f', '.init.py.orig', 'pyroute2/__init__.py', external=True ) session.run('rm', '-rf', 'build', external=True) tmpdir = os.path.abspath(session.create_tmp()) + session.run('cp', '-a', 'lab', tmpdir, external=True) session.run('cp', '-a', 'tests', tmpdir, external=True) session.run('cp', '-a', 'examples', tmpdir, external=True) return tmpdir -def setup_venv_common(session, flavour='dev', config=None): - if config is None: - config = {} - if not config.get('fast'): - session.install('--upgrade', 'pip') - session.install(f'.[{flavour}]') - session.install('.') +def setup_venv_common(session, flavour='dev'): + session.install('--upgrade', 'pip') + session.install('-r', f'requirements.{flavour}.txt') + session.install('.') return os.path.abspath(session.create_tmp()) -def setup_venv_dev(session, config=None): - if config is None: - config = {} - if config.get('fast'): - return os.getcwd() +def setup_venv_dev(session): tmpdir = setup_venv_common(session) session.run('cp', '-a', 'tests', tmpdir, external=True) session.run('cp', '-a', 'examples', tmpdir, external=True) @@ -197,7 +168,7 @@ def setup_venv_repo(session): ): session.run('cp', '-a', *item, external=True) git_ls_files = subprocess.run( - ['git', 'ls-files', 'pyproject*'], stdout=subprocess.PIPE + ['git', 'ls-files', 'requirements*'], stdout=subprocess.PIPE ) files = [x.decode('utf-8') for x in git_ls_files.stdout.split()] for fname in files: @@ -206,8 +177,8 @@ def setup_venv_repo(session): return tmpdir -def setup_venv_docs(session, config=None): - tmpdir = setup_venv_common(session, flavour='docs', config=config) +def setup_venv_docs(session): + tmpdir = setup_venv_common(session, 'docs') session.run('cp', '-a', 'docs', tmpdir, external=True) session.run('cp', '-a', 'examples', tmpdir, external=True) [ @@ -229,11 +200,10 @@ def test_platform(session): session.run('pyroute2-test-platform') -@nox.session(python='python3.10') -@add_session_config -def docs(session, config): +@nox.session +def docs(session): '''Generate project docs.''' - tmpdir = setup_venv_docs(session, config) + tmpdir = setup_venv_docs(session) cwd = os.path.abspath(os.getcwd()) # man pages session.chdir(f'{tmpdir}/docs/') @@ -253,31 +223,11 @@ def docs(session, config): session.log(f'man pages -> {cwd}/docs/man') -@nox.session( - python=[ - 'python3.9', - 'python3.10', - 'python3.11', - 'python3.12', - 'python3.13', - 'python3.14', - ] -) -@add_session_config -def linter(session, config): +@nox.session +def linter(session): '''Run code checks and linters.''' - if not config.get('fast'): - session.install('pre-commit') - session.install('mypy') + session.install('pre-commit') session.run('pre-commit', 'run', '-a') - with open('.mypy-check-paths', 'r') as f: - session.run( - 'python', - '-m', - 'mypy', - *f.read().split(), - env={'PYTHONPATH': os.getcwd()}, - ) @nox.session @@ -288,14 +238,6 @@ def unit(session, config): session.run(*options('test_unit', config)) -@nox.session -@add_session_config -def decoder(session, config): - '''Run decoder tests.''' - setup_venv_dev(session) - session.run(*options('test_decoder', config)) - - @nox.session @add_session_config def integration(session, config): @@ -304,94 +246,26 @@ def integration(session, config): session.run(*options('test_integration', config)) -def test_common(session, config, module): - setup_linux(session) - workspace = setup_venv_dev(session, config) - path = f'{workspace}/tests/mocklib' - if config.get('fast'): - path += f':{workspace}' - session.chdir('tests') - session.run( - *options(module, config), - env={'WORKSPACE': workspace, 'SKIPDB': 'postgres', 'PYTHONPATH': path}, - ) - - -@nox.session( - name='ci-self', - python=[ - 'python3.9', - 'python3.10', - 'python3.11', - 'python3.12', - 'python3.13', - 'python3.14', - ], -) -@add_session_config -def ci(session, config): - '''Run ci self-test. No root required.''' - test_common(session, config, 'test_ci') - - @nox.session( - python=[ - 'python3.9', - 'python3.10', - 'python3.11', - 'python3.12', - 'python3.13', - 'python3.14', - ] + python=['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12'] ) @add_session_config def linux(session, config): '''Run Linux functional tests. Requires root to run all the tests.''' - test_common(session, config, 'test_linux') - - -@nox.session( - python=[ - 'python3.9', - 'python3.10', - 'python3.11', - 'python3.12', - 'python3.13', - 'python3.14', - ] -) -@add_session_config -def core(session, config): - '''Run Linux tests in asyncio.''' - test_common(session, config, 'test_core') - - -@nox.session -@add_session_config -def limits(session, config): - '''Run limits & stress testing.''' - test_common(session, config, 'test_limits') + setup_linux(session) + workspace = setup_venv_dev(session) + session.run( + *options('test_linux', config), + env={ + 'WORKSPACE': workspace, + 'SKIPDB': 'postgres', + 'PYTHONPATH': f'{workspace}/tests/mocklib', + }, + ) @nox.session @add_session_config -def process(session, config): - '''Test child process module.''' - setup_venv_dev(session) - session.run(*options('test_process', config)) - - -@nox.session( - python=[ - 'python3.9', - 'python3.10', - 'python3.11', - 'python3.12', - 'python3.13', - 'python3.14', - ] -) -@add_session_config def minimal(session, config): '''Run tests on pyroute2.minimal package.''' tmpdir = setup_venv_minimal(session, config) @@ -399,6 +273,22 @@ def minimal(session, config): session.run(*options('test_minimal', config)) +@nox.session +@add_session_config +def lab(session, config): + '''Test lab code blocks.''' + workspace = setup_venv_minimal(session, config) + for fname in os.listdir('dist'): + if fname.startswith('pyroute2.minimal') and fname.endswith('whl'): + break + session.run('python', 'util/make_lab_templates.py', fname, external=True) + session.run('make', '-C', 'lab', 'html', external=True) + session.run('cp', f'dist/{fname}', 'lab/_build/html/', external=True) + # make tests + session.chdir(f'{workspace}/tests') + session.run(*options('test_lab', config), env={'WORKSPACE': workspace}) + + @nox.session @add_session_config def openbsd(session, config): @@ -420,7 +310,6 @@ def windows(session, config): def neutron(session, config): '''Run Neutron integration tests.''' setup_venv_dev(session) - session.install('eventlet') session.run(*options('test_neutron', config)) @@ -429,7 +318,6 @@ def neutron(session, config): def repo(session, config): '''Run repo tests.''' setup_venv_repo(session) - config = copy.copy(config) config['tests_prefix'] = 'tests' session.run(*options('test_repo', config)) diff --git a/pr2modules/__init__.py b/pr2modules/__init__.py new file mode 100644 index 000000000..3e5c6bd02 --- /dev/null +++ b/pr2modules/__init__.py @@ -0,0 +1,24 @@ +''' +This namespace is here only to provide compatibility with 0.6.x + +It will be removed in 0.8.x +''' + +import sys +import warnings + +# load pyroute2 entry points +import pyroute2 # noqa: F401 + +warnings.warn( + 'pr2modules namespace is deprecated, use pyroute2 instead', + DeprecationWarning, +) + +# alias every `pyroute2` entry, in addition to the block above +# +# Bug-Url: https://github.com/svinota/pyroute2/issues/913 +# +for key, value in list(sys.modules.items()): + if key.startswith("pyroute2."): + sys.modules[key.replace("pyroute2", "pr2modules")] = value diff --git a/pyproject.minimal.toml b/pyproject.minimal.toml deleted file mode 100644 index 03d6dc06e..000000000 --- a/pyproject.minimal.toml +++ /dev/null @@ -1,107 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "pyroute2-minimal" -dynamic = ["version"] -description = "Python Netlink library" -readme = "README.rst" -license = "GPL-2.0-or-later OR Apache-2.0" -license-files = ["LICENSE*"] -authors = [ - {name = "Peter Saveliev", email = "peter@svinota.eu"} -] -classifiers = [ - "Programming Language :: Python", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: System :: Networking", - "Topic :: System :: Systems Administration", - "Operating System :: POSIX :: Linux", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "Intended Audience :: Telecommunications Industry", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Development Status :: 4 - Beta" -] -urls = {Homepage = "https://github.com/svinota/pyroute2"} -dependencies = [ - "win_inet_pton ; platform_system == \"Windows\"" -] -requires-python = ">=3.9" - -[project.optional-dependencies] -dev = [ - "build", - "twine", - "flake8", - "netaddr", - "pytest", - "pytest-asyncio==0.26.0", - "pytest-timeout", - "pytest-cov", - "pytest-profiling", - "pre-commit", - "findimports", -] -docs = [ - "build", - "twine", - "aafigure", - "sphinx", - "sphinx-code-include", - "pre-commit", - "pytest", - "pytest-asyncio==0.26.0", - "docutils", -] -repo = [ - "nox", - "pytest", - "pytest-cov", - "pytest-html", - "pytest-timeout", - "setuptools", -] - -[tool.setuptools.packages.find] -where = ["."] -include = [ - "pyroute2", - "pyroute2.bsd", - "pyroute2.bsd.pf_route", - "pyroute2.bsd.rtmsocket", - "pyroute2.config", - "pyroute2.inotify", - "pyroute2.iproute", - "pyroute2.netlink", - "pyroute2.netlink.devlink", - "pyroute2.netlink.diag", - "pyroute2.netlink.event", - "pyroute2.netlink.generic", - "pyroute2.netlink.ipq", - "pyroute2.netlink.nfnetlink", - "pyroute2.netlink.nl80211", - "pyroute2.netlink.rtnl", - "pyroute2.netlink.rtnl.ifinfmsg", - "pyroute2.netlink.rtnl.ifinfmsg.plugins", - "pyroute2.netlink.rtnl.tcmsg", - "pyroute2.netlink.taskstats", - "pyroute2.netlink.uevent", - "pyroute2.netns", - "pyroute2.protocols", - "pyroute2.requests" -] - -[tool.setuptools.dynamic] -version = {file = "VERSION"} - -[[tool.mypy.overrides]] -module = ["pytest.*", "pytest_asyncio.*"] -ignore_missing_imports = true diff --git a/pyproject.toml b/pyproject.toml index d734da04a..9787c3bdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,89 +1,3 @@ [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" - -[project] -name = "pyroute2" -dynamic = ["version"] -description = "Python Netlink library" -readme = "README.rst" -license = "GPL-2.0-or-later OR Apache-2.0" -license-files = ["LICENSE*"] -authors = [ - {name = "Peter Saveliev", email = "peter@svinota.eu"} -] -classifiers = [ - "Programming Language :: Python", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: System :: Networking", - "Topic :: System :: Systems Administration", - "Operating System :: POSIX :: Linux", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "Intended Audience :: Telecommunications Industry", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Development Status :: 4 - Beta" -] -urls = {Homepage = "https://github.com/svinota/pyroute2"} -dependencies = [ - "win_inet_pton ; platform_system == \"Windows\"" -] -requires-python = ">=3.9" - -[project.optional-dependencies] -dev = [ - "build", - "twine", - "flake8", - "netaddr", - "pytest", - "pytest-asyncio==0.26.0", - "pytest-timeout", - "pytest-cov", - "pytest-profiling", - "pre-commit", - "findimports", -] -docs = [ - "build", - "twine", - "aafigure", - "sphinx", - "sphinx-code-include", - "pre-commit", - "pytest", - "pytest-asyncio==0.26.0", - "docutils", -] -repo = [ - "nox", - "pytest", - "pytest-cov", - "pytest-html", - "pytest-timeout", - "setuptools", -] - -[project.scripts] -ss2 = "pyroute2.netlink.diag.ss2:run [psutil]" -pyroute2-decoder = "pyroute2.decoder.main:run" -pyroute2-dhcp-client = "pyroute2.dhcp.cli:run" -pyroute2-test-platform = "pyroute2.config.test_platform:run" -dhcp-server-detector = "pyroute2.dhcp.server_detector:run" - -[tool.setuptools.packages.find] -where = ["."] -include = ["pyroute2*"] - -[tool.setuptools.dynamic] -version = {file = "VERSION"} - -[[tool.mypy.overrides]] -module = ["pytest.*", "pytest_asyncio.*"] -ignore_missing_imports = true diff --git a/pyroute2/__init__.py b/pyroute2/__init__.py index 88df5f32b..4979c64be 100644 --- a/pyroute2/__init__.py +++ b/pyroute2/__init__.py @@ -11,157 +11,103 @@ from pyroute2.config.version import __version__ except ImportError: __version__ = 'unknown' -import sys -## -# -# Logging setup -# -# See the history: -# * https://github.com/svinota/pyroute2/issues/246 -# * https://github.com/svinota/pyroute2/issues/255 -# * https://github.com/svinota/pyroute2/issues/270 -# * https://github.com/svinota/pyroute2/issues/573 -# * https://github.com/svinota/pyroute2/issues/601 -# -from pyroute2.config import log -from pyroute2.conntrack import AsyncConntrack, Conntrack, ConntrackEntry -from pyroute2.devlink import DL, AsyncDL +from pyroute2 import loader +from pyroute2.cli.console import Console +from pyroute2.cli.server import Server +from pyroute2.conntrack import Conntrack, ConntrackEntry +from pyroute2.devlink import DL from pyroute2.ethtool.ethtool import Ethtool -from pyroute2.ipdb import IPDB, CommitException, CreateException -from pyroute2.iproute import ( - AsyncIPRoute, - ChaoticIPRoute, - IPBatch, - IPRoute, - NetNS, - RawIPRoute, +from pyroute2.ipdb.exceptions import ( + CommitException, + CreateException, + DeprecationException, + PartialCommitException, ) +from pyroute2.ipdb.main import IPDB +from pyroute2.iproute import ChaoticIPRoute, IPBatch, IPRoute, RawIPRoute +from pyroute2.iproute.ipmock import IPRoute as IPMock from pyroute2.ipset import IPSet -from pyroute2.ipvs import IPVS, IPVSDest, IPVSService -from pyroute2.iwutil import IW, AsyncIW +from pyroute2.iwutil import IW from pyroute2.ndb.main import NDB +from pyroute2.ndb.noipdb import NoIPDB from pyroute2.netlink.connector.cn_proc import ProcEventSocket -from pyroute2.netlink.devlink import AsyncDevlinkSocket, DevlinkSocket +from pyroute2.netlink.devlink import DevlinkSocket from pyroute2.netlink.diag import DiagSocket, ss2 -from pyroute2.netlink.event import AsyncEventSocket, EventSocket -from pyroute2.netlink.event.acpi_event import ( - AcpiEventSocket, - AsyncAcpiEventSocket, -) -from pyroute2.netlink.event.dquot import AsyncDQuotSocket, DQuotSocket -from pyroute2.netlink.event.thermal import ( - AsyncThermalEventSocket, - ThermalEventSocket, -) +from pyroute2.netlink.event.acpi_event import AcpiEventSocket +from pyroute2.netlink.event.dquot import DQuotSocket from pyroute2.netlink.exceptions import ( ChaoticException, NetlinkDecodeError, NetlinkDumpInterrupted, NetlinkError, ) -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) -from pyroute2.netlink.generic.ethtool import AsyncNlEthtool, NlEthtool -from pyroute2.netlink.generic.ipvs import AsyncIPVSSocket, IPVSSocket -from pyroute2.netlink.generic.l2tp import AsyncL2tp, L2tp -from pyroute2.netlink.generic.mptcp import MPTCP, AsyncMPTCP -from pyroute2.netlink.generic.wireguard import AsyncWireGuard, WireGuard +from pyroute2.netlink.generic import GenericNetlinkSocket +from pyroute2.netlink.generic.l2tp import L2tp +from pyroute2.netlink.generic.mptcp import MPTCP +from pyroute2.netlink.generic.wireguard import WireGuard from pyroute2.netlink.ipq import IPQSocket -from pyroute2.netlink.nfnetlink.nfctsocket import AsyncNFCTSocket, NFCTSocket -from pyroute2.netlink.nfnetlink.nftsocket import AsyncNFTSocket, NFTSocket -from pyroute2.netlink.nl80211 import NL80211, AsyncNL80211 -from pyroute2.netlink.rtnl.iprsocket import AsyncIPRSocket, IPRSocket -from pyroute2.netlink.taskstats import AsyncTaskStats, TaskStats +from pyroute2.netlink.nfnetlink.nfctsocket import NFCTSocket +from pyroute2.netlink.nfnetlink.nftsocket import NFTSocket +from pyroute2.netlink.nl80211 import NL80211 +from pyroute2.netlink.rtnl.iprsocket import IPRSocket +from pyroute2.netlink.taskstats import TaskStats from pyroute2.netlink.uevent import UeventSocket +from pyroute2.nslink.nslink import NetNS from pyroute2.nslink.nspopen import NSPopen -from pyroute2.plan9.client import Plan9ClientSocket -from pyroute2.plan9.server import Plan9ServerSocket +from pyroute2.remote import RemoteIPRoute +from pyroute2.remote.transport import RemoteSocket from pyroute2.wiset import WiSet -## -# -# Windows platform specific: socket module monkey patching -# -# To use the library on Windows, run:: -# pip install win-inet-pton -# -if sys.platform.startswith('win'): # noqa: E402 - import win_inet_pton # noqa: F401 - - modules = [ AcpiEventSocket, - AsyncAcpiEventSocket, - AsyncConntrack, - AsyncDL, - AsyncDQuotSocket, - AsyncDevlinkSocket, - AsyncEventSocket, - AsyncGenericNetlinkSocket, - AsyncIPRSocket, - AsyncIPRoute, - AsyncIPVSSocket, - AsyncIW, - AsyncL2tp, - AsyncMPTCP, - AsyncNFCTSocket, - AsyncNFTSocket, - AsyncNL80211, - AsyncNlEthtool, - AsyncTaskStats, - AsyncThermalEventSocket, - AsyncWireGuard, ChaoticException, ChaoticIPRoute, CommitException, Conntrack, ConntrackEntry, + Console, CreateException, - DL, - DQuotSocket, + DeprecationException, DevlinkSocket, DiagSocket, + DL, + DQuotSocket, Ethtool, - EventSocket, - GenericNetlinkSocket, IPBatch, IPDB, + IPMock, IPQSocket, - IPRSocket, IPRoute, + IPRSocket, IPSet, - IPVS, - IPVSDest, - IPVSService, - IPVSSocket, IW, + GenericNetlinkSocket, L2tp, MPTCP, NDB, + NetlinkError, + NetlinkDecodeError, + NetlinkDumpInterrupted, + NetNS, NFCTSocket, NFTSocket, NL80211, + NoIPDB, NSPopen, - NetNS, - NetlinkDecodeError, - NetlinkDumpInterrupted, - NetlinkError, - NlEthtool, - Plan9ClientSocket, - Plan9ServerSocket, + PartialCommitException, ProcEventSocket, RawIPRoute, + RemoteIPRoute, + RemoteSocket, + Server, + ss2, TaskStats, - ThermalEventSocket, UeventSocket, - WiSet, WireGuard, - log, - ss2, + WiSet, ] +loader.init() __all__ = [] __all__.extend(modules) diff --git a/pyroute2/cli/__init__.py b/pyroute2/cli/__init__.py new file mode 100644 index 000000000..7b5d735cb --- /dev/null +++ b/pyroute2/cli/__init__.py @@ -0,0 +1,148 @@ +''' +CLI provides a simple syntax to manipulate NDB. The syntax is the +same for the console and the http versions. + +The first level of the hierarchy represents NDB views: + +* interfaces -- network interfaces +* addresses -- IP addresses +* routes -- IP and MPLS routes, one record per NH +* neighbours -- ARP cache +* rules -- RPDB rules +* netns -- network namespaces +* vlans -- bridge VLAN filters + +CLI supports indentation, though it is optional. A level up in the +indentation means a level up in the object hierarchy. Same effect +with `..` command. + +An example script to create a bridge and a port, no indentation:: + + ; comments start with ; + interfaces + create ifname br0, kind bridge + address 00:11:22:33:44:55 + state up + add_ip 192.168.123.21/24 + add_ip 192.168.123.22/24 + commit + .. ; level up to the interfaces view + create ifname br0p0, kind dummy + state up + master br0 + commit + +Same script with indentation, no `..` needed:: + + interfaces + create ifname br0, kind bridge + address 00:11:22:33:44:55 + state up + add_ip 192.168.123.21/24 + add_ip 192.168.123.22/24 + commit + create ifname br0p0, kind dummy + state up + master br0 + commit + +Select objects:: + + ; by name + interfaces br0 + ; ... + + ; by spec + interfaces + {target netns01, ifname eth0} + ; ... + {address 00:11:22:33:44:55} + ; ... + + + + +Manage interfaces +----------------- + +Create:: + + interfaces + create ifname br0.100, kind vlan, vlan_id 100, link br0 + commit + create ifname v0, kind veth, peer v0p + commit + +Change mac address:: + + interfaces br0 + address 00:11:22:33:44:55 + commit + +Change netns and rename:: + + sources + add netns test01 + interfaces + v0p + net_ns_fd test01 + commit + {target test01, ifname v0p} + ifname eth0 + commit + summary | filter kind veth | select target, ifname | format json + +Manage addresses +---------------- + +... + +Manage routes +------------- + +... + +Generate reports +---------------- + +It is possible to modify the output of the dump or summary commands:: + + interfaces + ; print index, ifname and MAC address for UP interfaces + dump | filter state up | select index, ifname, address + + routes + ; output in JSON format + summary | format json + +The `format` command can only be the last in the sentence. Available +output filters: + +filter { } -- filter out records +select { } -- output only selected record fields +format { } -- change the output format, possible values: csv, json + +Wait for events +--------------- + +... + +''' + +t_stmt = 1 +t_dict = 2 +t_comma = 3 +t_pipe = 4 +t_end_of_dict = 7 +t_end_of_sentence = 8 +t_end_of_stream = 9 + + +def change_pointer(f): + f.__cli_cptr__ = True + return f + + +def show_result(f): + f.__cli_publish__ = True + return f diff --git a/pyroute2/cli/auth/__init__.py b/pyroute2/cli/auth/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyroute2/cli/auth/auth_keystone.py b/pyroute2/cli/auth/auth_keystone.py new file mode 100644 index 000000000..0626d3c5b --- /dev/null +++ b/pyroute2/cli/auth/auth_keystone.py @@ -0,0 +1,33 @@ +import os +import time + +from dateutil.parser import parse as isodate +from keystoneauth1 import session +from keystoneauth1.identity import v3 +from keystoneclient.v3 import client as ksclient +from keystoneclient.v3.tokens import TokenManager + + +class OSAuthManager(object): + def __init__(self, headers): + # create a Keystone password object + auth = v3.Password( + auth_url=os.environ.get('OS_AUTH_URL'), + username=os.environ.get('OS_USERNAME'), + password=os.environ.get('OS_PASSWORD'), + user_domain_name=(os.environ.get('OS_USER_DOMAIN_NAME')), + project_id=os.environ.get('OS_PROJECT_ID'), + ) + # create a session object + sess = session.Session(auth=auth) + # create a token manager + tmanager = TokenManager(ksclient.Client(session=sess)) + # validate the token + keystone_response = tmanager.validate(headers['X-Auth-Token']) + # init attrs + self.expire = isodate(keystone_response['expires_at']).timestamp() + + def check(self, obj, tag): + if time.time() > self.expire: + raise PermissionError('keystone token has been expired') + return True diff --git a/pyroute2/cli/auth/auth_radius.py b/pyroute2/cli/auth/auth_radius.py new file mode 100644 index 000000000..f580d6c09 --- /dev/null +++ b/pyroute2/cli/auth/auth_radius.py @@ -0,0 +1,28 @@ +import os + +import pyrad.packet +from pyrad.client import Client +from pyrad.dictionary import Dictionary + + +class RadiusAuthManager(object): + def __init__(self, headers): + user = headers['X-Auth-User'] + password = headers['X-Auth-Password'] + + client = Client( + server=os.environ.get('RADIUS_SERVER'), + secret=os.environ.get('RADIUS_SECRET').encode('ascii'), + dict=Dictionary('dictionary'), + ) + + req = client.CreateAuthPacket( + code=pyrad.packet.AccessRequest, User_Name=user + ) + + req['User-Password'] = req.PwCrypt(password) + reply = client.SendPacket(req) + self.auth = reply.code + + def check(self, obj, tag): + return self.auth == pyrad.packet.AccessAccept diff --git a/pyroute2/cli/console.py b/pyroute2/cli/console.py new file mode 100644 index 000000000..7c44ff5fd --- /dev/null +++ b/pyroute2/cli/console.py @@ -0,0 +1,107 @@ +import code +import getpass +import socket +import sys + +from pyroute2.cli.session import Session +from pyroute2.ndb.main import NDB + + +class Console(code.InteractiveConsole): + def __init__(self, stdout=None, log=None, sources=None): + self.db = NDB(log=log, sources=sources) + self.db.config.update( + {'show_format': 'json', 'recordset_pipe': 'true'} + ) + self.stdout = stdout or sys.stdout + self.session = Session(self.db, self.stdout, self.set_prompt) + self.matches = [] + self.isatty = sys.stdin.isatty() + self.prompt = '' + self.set_prompt() + code.InteractiveConsole.__init__(self) + + def close(self): + self.db.close() + + def help(self): + self.session.lprint( + "Built-in commands: \n" + "exit\t-- exit cli\n" + "ls\t-- list current namespace\n" + ".\t-- print the current object\n" + ".. or Ctrl-D\t-- one level up\n" + ) + + def set_prompt(self, prompt=None): + if self.isatty: + if prompt is not None: + self.prompt = '%s > ' % (prompt) + else: + self.prompt = '%s > ' % (self.session.ptr.__class__.__name__) + self.prompt = '%s@%s : %s' % ( + getpass.getuser(), + (socket.gethostname().split('.')[0]), + self.prompt, + ) + + def loadrc(self, fname): + with open(fname, 'r') as f: + self.session.handle(f.read()) + + def interact(self, readfunc=None): + if self.isatty and readfunc is None: + self.session.lprint("pyroute2 cli prototype") + + if readfunc is None: + readfunc = self.raw_input + + indent = 0 + while True: + try: + text = readfunc(self.prompt) + except EOFError: + if self.session.stack: + self.session.stack_pop() + continue + else: + self.close() + break + except Exception: + self.close() + break + + try: + indent = self.session.handle(text, indent) + except SystemExit: + self.close() + return + except: + self.showtraceback() + continue + + def set_completer(self, readline): + readline.parse_and_bind('tab: complete') + readline.set_completer(self.completer) + readline.set_completion_display_matches_hook(self.display) + + def completer(self, text, state): + if state == 0: + d = [x for x in dir(self.session.ptr) if x.startswith(text)] + if isinstance(self.session.ptr, dict): + keys = [str(y) for y in self.session.ptr.keys()] + d.extend([x for x in keys if x.startswith(text)]) + self.matches = d + try: + return self.matches[state] + except: + pass + + def display(self, line, matches, length): + self.session.lprint() + self.session.lprint(matches) + self.session.lprint('%s%s' % (self.prompt, line), end='') + + +if __name__ == '__main__': + Console().interact() diff --git a/pyroute2/cli/parser.py b/pyroute2/cli/parser.py new file mode 100644 index 000000000..93444ab42 --- /dev/null +++ b/pyroute2/cli/parser.py @@ -0,0 +1,204 @@ +import re +import shlex + +from pyroute2.cli import ( + t_comma, + t_dict, + t_end_of_dict, + t_end_of_sentence, + t_end_of_stream, + t_pipe, + t_stmt, +) +from pyroute2.common import basestring + + +class Token(object): + def __init__(self, lex, expect=(), prohibit=(), leaf=False): + self.lex = lex + self.leaf = leaf + self.kind = 0 + self.name = None + self.argv = [] + self.kwarg = {} + self.parse() + if expect and self.kind not in expect: + raise SyntaxError('expected %s, got %s' % (expect, self.kind)) + if prohibit and self.kind in prohibit: + raise SyntaxError('unexpected %s' % (self.name,)) + + def convert(self, arg): + if re.match('^[0-9]+$', arg): + return int(arg) + else: + return arg + + def parse(self): + # triage + first = self.lex.get_token() + self.name = first + + ## + # no token + # + if first == '': + self.kind = t_end_of_stream + + ## + # dict, e.g. + # + # resource spec, function arguments:: + # {arg1, arg2} + # {key1 value1, key2 value2} + # {key {skey1 value}} + # + elif first == '{': + arg_name = None + while True: + nt = Token( + self.lex, expect=(t_stmt, t_dict, t_comma, t_end_of_dict) + ) + if arg_name is None: + if nt.kind == t_dict: + self.argv.append(nt.kwarg) + elif nt.kind == t_comma: + continue + elif nt.kind == t_stmt: + arg_name = nt.name + else: + if nt.kind in (t_end_of_dict, t_comma): + self.argv.append(arg_name) + elif nt.kind == t_stmt: + self.kwarg[arg_name] = nt.name + elif nt.kind == t_dict: + self.kwarg[arg_name] = nt.kwarg + arg_name = None + + if nt.kind == t_end_of_dict: + self.kind = t_dict + self.name = '%s %s' % (self.argv, self.kwarg) + return + + ## + # end of dict + # + elif first == '}': + self.kind = t_end_of_dict + + ## + # end of sentence + # + elif first == ';': + self.kind = t_end_of_sentence + + ## + # end of dict entry + # + elif first == ',': + self.kind = t_comma + + ## + # pipe + # + elif first == '|': + self.kind = t_pipe + + elif first == '=': + lookahead = self.lex.get_token() + if lookahead == '>': + self.name = '=>' + self.kind = t_pipe + else: + self.lex.push_token(lookahead) + self.kind = t_stmt + + ## + # simple statement + # + # object name:: + # name + # + # function call:: + # func + # func {arg1, arg2} + # func {key1 value1, key2 value2} + # + else: + self.name = self.convert(first) + self.kind = t_stmt + + +class Sentence(object): + def __init__(self, text, indent=0, master=None): + self.offset = 0 + self.statements = [] + self.text = text + self.lex = shlex.shlex(text) + self.lex.wordchars += '.:/' + self.lex.commenters = '#!' + self.lex.debug = False + self.indent = indent + if master: + self.chain = master.chain + else: + self.chain = [] + self.parse() + + def __iter__(self): + for stmt in self.statements: + yield stmt + + def parse(self): + sentence = self + while True: + nt = Token(self.lex) + if nt.kind == t_end_of_sentence: + sentence = Sentence(None, self.indent, master=self) + elif nt.kind == t_end_of_stream: + return + else: + sentence.statements.append(nt) + if sentence not in self.chain: + self.chain.append(sentence) + + def __repr__(self): + ret = '----\n' + for s in self.statements: + ret += '%i [%s] %s\n' % (self.indent, s.kind, s.name) + ret += '\targv: %s\n' % (s.argv) + ret += '\tkwarg: %s\n' % (s.kwarg) + return ret + + +class Parser(object): + def __init__(self, stream): + self.stream = stream + self.indent = None + self.sentences = [] + self.parse() + + def parse(self): + if hasattr(self.stream, 'readlines'): + for text in self.stream.readlines(): + self.parse_string(text) + elif isinstance(self.stream, basestring): + self.parse_string(self.stream) + else: + raise ValueError('unsupported stream') + self.parsed = True + + def parse_string(self, text): + # 1. get indentation + indent = re.match(r'^([ \t]*)', text).groups(0)[0] + spaces = [] + # 2. sort it + if indent: + spaces = list(set(indent)) + if len(spaces) > 1: + raise SyntaxError('mixed indentation') + if self.indent is None: + self.indent = spaces[0] + if self.indent != spaces[0]: + raise SyntaxError('mixed indentation') + sentence = Sentence(text, len(indent)) + self.sentences.extend(sentence.chain) diff --git a/pyroute2/cli/server.py b/pyroute2/cli/server.py new file mode 100644 index 000000000..ea5b8ad0b --- /dev/null +++ b/pyroute2/cli/server.py @@ -0,0 +1,108 @@ +import json + +from pyroute2.cli.session import Session +from pyroute2.ndb.main import NDB + +try: + from BaseHTTPServer import BaseHTTPRequestHandler + from BaseHTTPServer import HTTPServer as HTTPServer +except ImportError: + from http.server import BaseHTTPRequestHandler + from http.server import HTTPServer as HTTPServer + + +class ProxyEncoder(object): + def __init__(self, wfile): + self.wfile = wfile + + def write(self, data): + self.wfile.write(data.encode('utf-8')) + + def flush(self): + self.wfile.flush() + + +class Handler(BaseHTTPRequestHandler): + def do_error(self, code, reason): + self.send_error(code, reason) + self.end_headers() + + def do_POST(self): + # + # sanity checks: + # + # * path + if self.path != '/v1/': + return self.do_error(404, 'url not found') + # * content length + if 'Content-Length' not in self.headers: + return self.do_error(411, 'Content-Length') + # * content type + if 'Content-Type' not in self.headers: + return self.do_error(400, 'Content-Type') + # + + content_length = int(self.headers['Content-Length']) + content_type = self.headers['Content-Type'] + data = self.rfile.read(content_length) + + if content_type == 'application/json': + try: + request = json.loads(data) + except ValueError: + return self.do_error(400, 'Incorrect JSON input') + elif content_type == 'text/plain': + request = {'commands': data.decode('utf-8').split(';')} + else: + self.do_error(400, 'Incorrect content type') + + # auth plugins + if 'X-Auth-Mech' in self.headers: + auth_plugin = self.server.auth_plugins.get( + self.headers['X-Auth-Mech'] + ) + if auth_plugin is None: + return self.do_error(501, 'Authentication mechanism not found') + try: + am = auth_plugin(self.headers) + except Exception: + return self.do_error(401, 'Authentication failed') + ndb = self.server.ndb.auth_proxy(am) + elif self.server.auth_strict: + return self.do_error(401, 'Authentication required') + else: + ndb = self.server.ndb + + session = Session( + ndb=ndb, + stdout=ProxyEncoder(self.wfile), + builtins=('ls', '.', '..', 'version'), + ) + self.send_response(200) + self.end_headers() + for cmd in request['commands']: + session.handle(cmd) + + +class Server(HTTPServer): + def __init__( + self, + address='localhost', + port=8080, + sources=None, + ndb=None, + log=None, + auth_strict=False, + auth_plugins=None, + ): + self.sessions = {} + self.auth_strict = auth_strict + self.auth_plugins = auth_plugins or {} + if ndb is not None: + self.ndb = ndb + else: + self.ndb = NDB(sources=sources, log=log) + self.ndb.config.update( + {'show_format': 'json', 'recordset_pipe': 'true'} + ) + HTTPServer.__init__(self, (address, port), Handler) diff --git a/pyroute2/cli/session.py b/pyroute2/cli/session.py new file mode 100644 index 000000000..546e3960b --- /dev/null +++ b/pyroute2/cli/session.py @@ -0,0 +1,256 @@ +from __future__ import print_function + +import sys +import traceback +from collections import namedtuple + +from pyroute2 import config +from pyroute2.cli import t_comma, t_dict, t_pipe, t_stmt +from pyroute2.cli.parser import Parser +from pyroute2.common import basestring + + +class Session(object): + def __init__(self, ndb, stdout=None, ptrname_callback=None, builtins=None): + self.db = ndb + self.ptr = self.db + self._ptrname = None + self._ptrname_callback = ptrname_callback + self.stack = [] + self.errors = 0 + self.indent_stack = set() + self.prompt = '' + self.stdout = stdout or sys.stdout + self.builtins = builtins or ( + 'ls', + '.', + '..', + 'version', + 'exit', + ':stack', + ) + + @property + def ptrname(self): + return self._ptrname + + @ptrname.setter + def ptrname(self, name): + self._ptrname = name + if self._ptrname_callback is not None: + self._ptrname_callback(name) + + def stack_pop(self): + self.ptr, self.ptrname = self.stack.pop() + return (self.ptr, self.ptrname) + + def lprint(self, text='', end='\n'): + if not isinstance(text, basestring): + text = str(text) + self.stdout.write(text) + if end: + self.stdout.write(end) + self.stdout.flush() + + def handle_statement(self, stmt, token): + obj = None + if stmt.kind == t_dict: + obj = self.ptr[stmt.kwarg] + elif stmt.kind == t_stmt: + obj = getattr(self.ptr, stmt.name, None) + if obj is None and isinstance(self.ptr, dict): + try: + obj = self.ptr.get(stmt.name, None) + except KeyError: + pass + + if hasattr(obj, '__call__'): + try: + nt = next(token) + except StopIteration: + nt = namedtuple('Token', ('kind', 'argv', 'kwarg'))( + t_dict, [], {} + ) + + if nt.kind == t_dict: + args = nt + try: + pipe = next(token) + if pipe.kind != t_pipe: + raise TypeError('pipe expected') + except StopIteration: + pipe = None + elif nt.kind == t_stmt: + argv = [] + kwarg = {} + arg_name = nt.name + pipe = None + for nt in token: + if arg_name is None: + if nt.kind == t_stmt: + arg_name = nt.name + elif nt.kind == t_comma: + continue + elif nt.kind == t_pipe: + pipe = nt + break + else: + raise TypeError('stmt expected') + else: + if nt.kind == t_comma: + argv.append(arg_name) + elif nt.kind == t_stmt: + kwarg[arg_name] = nt.name + elif nt.kind == t_pipe: + pipe = nt + break + else: + raise TypeError('stmt or comma expected') + arg_name = None + if arg_name is not None: + argv.append(arg_name) + args = namedtuple('Token', ('kind', 'argv', 'kwarg'))( + t_dict, argv, kwarg + ) + elif nt.kind == t_pipe: + args = namedtuple('Token', ('kind', 'argv', 'kwarg'))( + t_dict, [], {} + ) + pipe = nt + else: + raise TypeError('dict, stmt or comma expected') + + # at this step we have + # args -- arguments + # pipe -- pipe or None + + try: + ret = obj(*args.argv, **args.kwarg) + # + if pipe is not None: + ptr = self.ptr + self.ptr = ret + try: + stmt = next(token) + except StopIteration: + raise TypeError('statement expected') + if stmt.kind != t_stmt: + raise TypeError('statement expected') + try: + self.handle_statement(stmt, token) + except Exception: + pass + self.ptr = ptr + return + if hasattr(obj, '__cli_cptr__'): + obj = ret + elif hasattr(obj, '__cli_publish__'): + if hasattr(ret, 'generator') or hasattr(ret, 'next'): + for line in ret: + if isinstance(line, basestring): + self.lprint(line) + else: + self.lprint(repr(line)) + else: + self.lprint(ret) + return + elif isinstance(ret, (bool, basestring, int, float)): + self.lprint(ret) + return + else: + return + except Exception: + self.errors += 1 + traceback.print_exc() + return + else: + if isinstance(self.ptr, dict) and not isinstance(obj, dict): + try: + nt = next(token) + if nt.kind == t_stmt: + self.ptr[stmt.name] = nt.name + elif nt.kind == t_dict and nt.argv: + self.ptr[stmt.name] = nt.argv + elif nt.kind == t_dict and nt.kwarg: + self.ptr[stmt.name] = nt.kwarg + else: + raise TypeError('failed setting a key/value pair') + return + except NotImplementedError: + raise KeyError() + except StopIteration: + pass + + if obj is None: + raise KeyError() + elif isinstance(obj, (basestring, int, float)): + self.lprint(obj) + else: + return obj + + def handle_sentence(self, sentence, indent): + if sentence.indent < indent: + while max(self.indent_stack) > sentence.indent: + self.indent_stack.remove(max(self.indent_stack)) + if self.stack: + self.ptr, self.ptrname = self.stack.pop() + else: + self.indent_stack.add(sentence.indent) + indent = sentence.indent + iterator = iter(sentence) + obj = None + save_ptr = self.ptr + save_ptrname = self.ptrname + try: + for stmt in iterator: + if stmt.name in self.builtins: + if stmt.name == 'exit': + raise SystemExit() + elif stmt.name == 'ls': + self.lprint(dir(self.ptr)) + elif stmt.name == ':stack': + self.lprint('stack:') + for item in self.stack: + self.lprint(item) + self.lprint('end') + elif stmt.name == '.': + self.lprint(repr(self.ptr)) + elif stmt.name == '..': + if self.stack: + save_ptr, save_ptrname = self.stack.pop() + elif stmt.name == 'version': + try: + self.lprint(config.version.__version__) + except: + self.lprint('unknown') + break + else: + try: + obj = self.handle_statement(stmt, iterator) + if obj is not None: + self.ptr = obj + if hasattr(obj, 'key_repr'): + self.ptrname = obj.key_repr() + else: + self.ptrname = stmt.name + except KeyError: + self.lprint('object not found') + self.errors += 1 + return indent + except: + self.errors += 1 + traceback.print_exc() + except SystemExit: + raise + finally: + if obj is not None: + self.stack.append((save_ptr, save_ptrname)) + else: + self.ptr, self.ptrname = save_ptr, save_ptrname + return indent + + def handle(self, text, indent=0): + parser = Parser(text) + for sentence in parser.sentences: + indent = self.handle_sentence(sentence, indent) + return indent diff --git a/pyroute2/common.py b/pyroute2/common.py index 21ad8d4d5..caaf0f54b 100644 --- a/pyroute2/common.py +++ b/pyroute2/common.py @@ -4,22 +4,41 @@ ''' import errno import io +import logging import os +import re import socket import struct +import sys import threading import time import types -from functools import partial -from typing import Any, Callable, Literal, Optional, TypeVar, Union -basestring = (str, bytes) -file = io.BytesIO +log = logging.getLogger(__name__) + +try: + # + # Python2 section + # + basestring = basestring + reduce = reduce + file = file + +except NameError: + # + # Python3 section + # + basestring = (str, bytes) + from functools import reduce + + reduce = reduce + file = io.BytesIO AF_MPLS = 28 +AF_PIPE = 255 # Right now AF_MAX == 40 +DEFAULT_RCVBUF = 65536 _uuid32 = 0 # (singleton) the last uuid32 value saved to avoid collisions _uuid32_lock = threading.Lock() -USE_DEFAULT_TIMEOUT = -1 size_suffixes = { 'b': 1, @@ -72,24 +91,150 @@ ## # General purpose # -class Namespace: - def __init__(self, parent, override=None): +class View(object): + ''' + A read-only view of a dictionary object. + ''' + + def __init__(self, src=None, path=None, constraint=lambda k, v: True): + self.src = src if src is not None else {} + if path is not None: + path = path.split('/') + for step in path: + self.src = getattr(self.src, step) + self.constraint = constraint + + def __getitem__(self, key): + if key in self.keys(): + return self.src[key] + raise KeyError() + + def __setitem__(self, key, value): raise NotImplementedError() + def __delitem__(self, key): + raise NotImplementedError() + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def _filter(self): + ret = [] + for key, value in tuple(self.src.items()): + try: + if self.constraint(key, value): + ret.append((key, value)) + except Exception as e: + log.error("view filter error: %s", e) + return ret + + def keys(self): + return [x[0] for x in self._filter()] + + def values(self): + return [x[1] for x in self._filter()] + + def items(self): + return self._filter() + + def __iter__(self): + for key in self.keys(): + yield key -def _no_change(s: str) -> str: - return s + def __repr__(self): + return repr(dict(self._filter())) -def _default_normalize(s: str, prefix: str) -> str: - return s[len(prefix) :].lower() +class Namespace(object): + def __init__(self, parent, override=None): + self.parent = parent + self.override = override or {} + + def __getattr__(self, key): + if key in ('parent', 'override'): + return object.__getattr__(self, key) + elif key in self.override: + return self.override[key] + else: + ret = getattr(self.parent, key) + # ACHTUNG + # + # if the attribute we got with `getattr` + # is a method, rebind it to the Namespace + # object, so all subsequent getattrs will + # go through the Namespace also. + # + if isinstance(ret, types.MethodType): + ret = type(ret)(ret.__func__, self) + return ret + + def __setattr__(self, key, value): + if key in ('parent', 'override'): + object.__setattr__(self, key, value) + elif key in self.override: + self.override[key] = value + else: + setattr(self.parent, key, value) + + +class Dotkeys(dict): + ''' + This is a sick-minded hack of dict, intended to be an eye-candy. + It allows to get dict's items by dot reference: + + ipdb["lo"] == ipdb.lo + ipdb["eth0"] == ipdb.eth0 + + Obviously, it will not work for some cases, like unicode names + of interfaces and so on. Beside of that, it introduces some + complexity. + + But it simplifies live for old-school admins, who works with good + old "lo", "eth0", and like that naming schemes. + ''' + + __var_name = re.compile('^[a-zA-Z_]+[a-zA-Z_0-9]*$') + + def __dir__(self): + return [ + i for i in self if isinstance(i, str) and self.__var_name.match(i) + ] + + def __getattribute__(self, key, *argv): + try: + return dict.__getattribute__(self, key) + except AttributeError as e: + if key == '__deepcopy__': + raise e + elif key[:4] == 'set_': + + def set_value(value): + self[key[4:]] = value + return self + + return set_value + elif key in self: + return self[key] + else: + raise e + + def __setattr__(self, key, value): + if key in self: + self[key] = value + else: + dict.__setattr__(self, key, value) + + def __delattr__(self, key): + if key in self: + del self[key] + else: + dict.__delattr__(self, key) -def map_namespace( - prefix: str, - ns: dict[str, Any], - normalize: Union[None, Literal[True], Callable[[str], str]] = None, -) -> tuple[dict[str, int], dict[int, str]]: +def map_namespace(prefix, ns, normalize=None): ''' Take the namespace prefix, list all constants and build two dictionaries -- straight and reverse mappings. E.g.: @@ -117,34 +262,30 @@ def map_namespace( - True — cut the prefix and `lower()` the rest - lambda x: … — apply the function to every name ''' - transform: Callable[[str], str] - - if normalize is None: - transform = _no_change - elif normalize is True: - transform = partial(_default_normalize, prefix=prefix) - elif isinstance(normalize, types.FunctionType): - transform = normalize - else: - raise ValueError("Invalid value for `normalize` parameter") + nmap = {None: lambda x: x, True: lambda x: x[len(prefix) :].lower()} + + if not isinstance(normalize, types.FunctionType): + normalize = nmap[normalize] - by_name = {transform(i): ns[i] for i in ns.keys() if i.startswith(prefix)} - by_value = {ns[i]: transform(i) for i in ns.keys() if i.startswith(prefix)} + by_name = dict( + [(normalize(i), ns[i]) for i in ns.keys() if i.startswith(prefix)] + ) + by_value = dict( + [(ns[i], normalize(i)) for i in ns.keys() if i.startswith(prefix)] + ) return (by_name, by_value) -def getbroadcast( - addr: str, mask: int, family: socket.AddressFamily = socket.AF_INET -) -> str: +def getbroadcast(addr, mask, family=socket.AF_INET): # 1. convert addr to int i = socket.inet_pton(family, addr) if family == socket.AF_INET: - i_unpacked = struct.unpack('>I', i)[0] + i = struct.unpack('>I', i)[0] a = 0xFFFFFFFF length = 32 elif family == socket.AF_INET6: - i_unpacked = struct.unpack('>QQ', i) - i_unpacked = i_unpacked[0] << 64 | i_unpacked[1] + i = struct.unpack('>QQ', i) + i = i[0] << 64 | i[1] a = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF length = 128 else: @@ -152,7 +293,7 @@ def getbroadcast( # 2. calculate mask m = (a << length - mask) & a # 3. calculate default broadcast - n = (i_unpacked & m) | a >> mask + n = (i & m) | a >> mask # 4. convert it back to the normal address form if family == socket.AF_INET: n = struct.pack('>I', n) @@ -161,7 +302,7 @@ def getbroadcast( return socket.inet_ntop(family, n) -def dqn2int(mask: str, family: socket.AddressFamily = socket.AF_INET) -> int: +def dqn2int(mask, family=socket.AF_INET): ''' IPv4 dotted quad notation to int mask conversion ''' @@ -174,23 +315,25 @@ def dqn2int(mask: str, family: socket.AddressFamily = socket.AF_INET) -> int: return ret -def get_address_family(address: str) -> socket.AddressFamily: +def get_address_family(address): if address.find(':') > -1: return socket.AF_INET6 else: return socket.AF_INET -def hexdump(payload: bytes, length: int = 0) -> str: +def hexdump(payload, length=0): ''' Represent byte string as hex -- for debug purposes ''' return ':'.join('{0:02x}'.format(c) for c in payload[:length] or payload) -def load_dump( - f: Union[str, io.StringIO], meta: Optional[dict[str, str]] = None -) -> Union[bytes, str]: +def hexload(data): + return bytes(bytearray((int(x, 16) for x in data.split(':')))) + + +def load_dump(f, meta=None): ''' Load a packet dump from an open file-like object or a string. @@ -211,7 +354,6 @@ def load_dump( code = None meta_data = None meta_label = None - io_obj: Union[io.StringIO, str] if isinstance(f, str): io_obj = io.StringIO() io_obj.write(f) @@ -256,10 +398,13 @@ def load_dump( if isinstance(meta, dict): if code is not None: meta['code'] = code - if meta_data is not None and meta_label is not None: + if meta_data is not None: meta[meta_label] = meta_data - return bytes(data, 'iso8859-1') + if sys.version[0] == '3': + return bytes(data, 'iso8859-1') + else: + return data class AddrPool(object): @@ -270,17 +415,16 @@ class AddrPool(object): cell = 0xFFFFFFFFFFFFFFFF def __init__( - self, - minaddr: int = 0xF, - maxaddr: int = 0xFFFFFF, - reverse: bool = False, - release: bool = False, + self, minaddr=0xF, maxaddr=0xFFFFFF, reverse=False, release=False ): self.cell_size = 0 # in bits mx = self.cell self.reverse = reverse self.release = release - self.ban: list[dict[str, int]] = [] + self.allocated = 0 + if self.release and not isinstance(self.release, int): + raise TypeError() + self.ban = [] while mx: mx >>= 8 self.cell_size += 1 @@ -293,7 +437,7 @@ def __init__( self.maxaddr = maxaddr self.lock = threading.RLock() - def alloc(self) -> int: + def alloc(self): with self.lock: # gc self.ban: for item in tuple(self.ban): @@ -324,6 +468,7 @@ def alloc(self) -> int: if self.minaddr <= ret <= self.maxaddr: if self.release: self.free(ret, ban=self.release) + self.allocated += 1 return ret else: self.free(ret) @@ -338,7 +483,29 @@ def alloc(self) -> int: else: raise KeyError('no free address available') - def locate(self, addr: int) -> tuple[int, int, bool]: + def alloc_multi(self, count): + with self.lock: + addresses = [] + raised = False + try: + for _ in range(count): + addr = self.alloc() + try: + addresses.append(addr) + except: + # In case of a MemoryError during appending, + # the finally block would not free the address. + self.free(addr) + return addresses + except: + raised = True + raise + finally: + if raised: + for addr in addresses: + self.free(addr) + + def locate(self, addr): if self.reverse: addr = self.maxaddr - addr else: @@ -351,24 +518,60 @@ def locate(self, addr: int) -> tuple[int, int, bool]: is_allocated = False return (base, bit, is_allocated) - def free(self, addr: int, ban: int = 0): + def setaddr(self, addr, value): + if value not in ('free', 'allocated'): + raise TypeError() + with self.lock: + base, bit, is_allocated = self.locate(addr) + if value == 'free' and is_allocated: + self.allocated -= 1 + self.addr_map[base] |= 1 << bit + elif value == 'allocated' and not is_allocated: + self.allocated += 1 + self.addr_map[base] &= ~(1 << bit) + + def free(self, addr, ban=0): with self.lock: if ban != 0: self.ban.append({'addr': addr, 'counter': ban}) else: - base, bit, _ = self.locate(addr) + base, bit, is_allocated = self.locate(addr) if len(self.addr_map) <= base: raise KeyError('address is not allocated') if self.addr_map[base] & (1 << bit): raise KeyError('address is not allocated') + self.allocated -= 1 self.addr_map[base] ^= 1 << bit -def fnv1(data: bytes) -> int: +def _fnv1_python2(data): + ''' + FNV1 -- 32bit hash, python2 version + + @param data: input + @type data: bytes + + @return: 32bit int hash + @rtype: int + + See: http://www.isthe.com/chongo/tech/comp/fnv/index.html + ''' + hval = 0x811C9DC5 + for i in range(len(data)): + hval *= 0x01000193 + hval ^= struct.unpack('B', data[i])[0] + return hval & 0xFFFFFFFF + + +def _fnv1_python3(data): ''' FNV1 -- 32bit hash, python3 version - Returns: 32bit int hash + @param data: input + @type data: bytes + + @return: 32bit int hash + @rtype: int See: http://www.isthe.com/chongo/tech/comp/fnv/index.html ''' @@ -379,10 +582,19 @@ def fnv1(data: bytes) -> int: return hval & 0xFFFFFFFF -def uuid32() -> int: +if sys.version[0] == '3': + fnv1 = _fnv1_python3 +else: + fnv1 = _fnv1_python2 + + +def uuid32(): ''' Return 32bit UUID, based on the current time and pid. + @return: 32bit int uuid + @rtype: int + The uuid is guaranteed to be unique within one process. ''' global _uuid32 @@ -398,25 +610,23 @@ def uuid32() -> int: return candidate -def uifname() -> str: +def uifname(): ''' Return a unique interface name based on a prime function + + @return: interface name + @rtype: str ''' return 'pr%x' % uuid32() -F = TypeVar("F", bound=Callable[..., Any]) - - -def map_exception( - match: Callable[[Exception], bool], subst: Callable[[Exception], Exception] -) -> Callable[[F], F]: +def map_exception(match, subst): ''' Decorator to map exception types ''' - def wrapper(f: F) -> F: - def decorated(*argv: Any, **kwarg: Any) -> Any: + def wrapper(f): + def decorated(*argv, **kwarg): try: f(*argv, **kwarg) except Exception as e: @@ -424,12 +634,12 @@ def decorated(*argv: Any, **kwarg: Any) -> Any: raise subst(e) raise - return decorated # type: ignore + return decorated return wrapper -def map_enoent(f: F) -> F: +def map_enoent(f): ''' Shortcut to map OSError(2) -> OSError(95) ''' @@ -439,11 +649,8 @@ def map_enoent(f: F) -> F: )(f) -T = TypeVar('T', bound=type) - - -def metaclass(mc: T) -> Callable[[T], T]: - def wrapped(cls: T) -> T: +def metaclass(mc): + def wrapped(cls): nvars = {} skip = ['__dict__', '__weakref__'] slots = cls.__dict__.get('__slots__') @@ -459,21 +666,11 @@ def wrapped(cls: T) -> T: return wrapped -def msg_done(msg) -> bytes: - newmsg = struct.pack('IHH', 40, 2, 0) - newmsg += msg.data[8:16] - newmsg += struct.pack('I', 0) - # nlmsgerr struct alignment - newmsg += b'\0' * 20 - return newmsg +def failed_class(message): + class FailedClass(object): + def __init__(self, *argv, **kwarg): + ret = RuntimeError(message) + ret.feature_supported = False + raise ret - -def get_time() -> float: - ''' - Return seconds since arbitrary start point. - - It cannot go backward. It includes any time that the system is suspended. - You should use this instead of time.time() to measure time between two - execution points, like a timeout. - ''' - return time.monotonic() + return FailedClass diff --git a/pyroute2/compat.py b/pyroute2/compat.py deleted file mode 100644 index 5efa6ebe1..000000000 --- a/pyroute2/compat.py +++ /dev/null @@ -1,19 +0,0 @@ -'''Compatibility with older but supported Python versions''' - -try: - from enum import StrEnum # noqa: F401 -except ImportError: - from enum import Enum - - class StrEnum(str, Enum): - '''Same as enum, but members are also strings.''' - - -try: - from socket import ETHERTYPE_IP -except ImportError: - # ETHERTYPE_* are new in python 3.12 - ETHERTYPE_IP = 0x800 - - -__all__ = ('StrEnum', 'ETHERTYPE_IP') diff --git a/pyroute2/config/__init__.py b/pyroute2/config/__init__.py index 27d03a150..79515f3a4 100644 --- a/pyroute2/config/__init__.py +++ b/pyroute2/config/__init__.py @@ -3,57 +3,6 @@ import re import signal import socket -from typing import Literal, Union - -metric_type = Literal['c', 'g', 'ms'] - - -class LocalMock: - - def __init__(self): - self.call_args_list = [] - - def __call__(self, *argv, **kwarg): - self.call_args_list.append((argv, kwarg)) - return self - - def close(self) -> None: - pass - - def setsockopt(self, level: int, optname: int, value: int) -> None: - pass - - def fileno(self) -> None: - pass - - def put( - self, name: str, value: Union[int, str], kind: metric_type - ) -> None: - pass - - def commit(self) -> None: - pass - - def incr(self, name: str, value: int = 1) -> None: - pass - - -def mock_if(name): - def decorator(func): - local_mock = LocalMock() - - def wrapper(*argv, **kwarg): - if globals().get(name, False): - local_mock(*argv, **kwarg) - return local_mock - return func(*argv, **kwarg) - - wrapper.__doc__ = func.__doc__ - wrapper.__name__ = func.__name__ - return wrapper - - return decorator - kernel_version_re = re.compile('^[0-9.]+') @@ -77,15 +26,12 @@ def parse_kernel_version(kernel_name): gc_timeout = 60 db_transaction_limit = 1 cache_expire = 60 -telemetry = None -child_process_mode = 'fork' -force_gc = False + signal_stop_remote = None if hasattr(signal, 'SIGUSR1'): signal_stop_remote = signal.SIGUSR1 -mock_netlink = False -mock_netns = False +mock_iproute = False nlsocket_thread_safe = True # save uname() on startup time: it is not so @@ -97,9 +43,6 @@ def parse_kernel_version(kernel_name): arch = platform.architecture()[0] kernel = parse_kernel_version(uname[2]) -default_create_socket_timeout = 10 -default_communicate_timeout = 2 - AF_BRIDGE = getattr(socket, 'AF_BRIDGE', 7) AF_NETLINK = getattr(socket, 'AF_NETLINK', 16) diff --git a/pyroute2/config/asyncio.py b/pyroute2/config/asyncio.py index efc055f9d..ff2eeb965 100644 --- a/pyroute2/config/asyncio.py +++ b/pyroute2/config/asyncio.py @@ -64,10 +64,6 @@ def fileno(self): def dup(self): return self.__class__(_sock=self._sock.dup()) - @property - def type(self): - return self._sock.type - class _MpConnection(object): """Highly limited multiprocessing.Connection alternative""" diff --git a/pyroute2/conntrack.py b/pyroute2/conntrack.py index 94e26a005..62cf4e260 100644 --- a/pyroute2/conntrack.py +++ b/pyroute2/conntrack.py @@ -4,7 +4,6 @@ IP_CT_TCP_FLAG_TO_NAME, IPSBIT_TO_NAME, TCP_CONNTRACK_TO_NAME, - AsyncNFCTSocket, NFCTAttrTuple, NFCTSocket, ) @@ -119,19 +118,22 @@ def __repr__(self): return s -class AsyncConntrack(AsyncNFCTSocket): +class Conntrack(NFCTSocket): """ High level conntrack functions """ - async def stat(self): + def __init__(self, nlm_generator=True, **kwargs): + super(Conntrack, self).__init__(nlm_generator=nlm_generator, **kwargs) + + def stat(self): """Return current statistics per CPU Same result than conntrack -S command but a list of dictionaries """ stats = [] - for msg in await super().stat(): + for msg in super(Conntrack, self).stat(): stats.append({'cpu': msg['res_id']}) stats[-1].update( (k[10:].lower(), v) @@ -141,45 +143,65 @@ async def stat(self): return stats - async def count(self): + def count(self): """Return current number of conntrack entries Same result than /proc/sys/net/netfilter/nf_conntrack_count file or conntrack -C command """ - for ndmsg in await super().count(): + for ndmsg in super(Conntrack, self).count(): return ndmsg.get_attr('CTA_STATS_GLOBAL_ENTRIES') - async def conntrack_max_size(self): + def conntrack_max_size(self): """ Return the max size of connection tracking table /proc/sys/net/netfilter/nf_conntrack_max """ - for ndmsg in await super().conntrack_max_size(): + for ndmsg in super(Conntrack, self).conntrack_max_size(): return ndmsg.get_attr('CTA_STATS_GLOBAL_MAX_ENTRIES') - async def delete(self, entry): + def delete(self, entry): if isinstance(entry, ConntrackEntry): tuple_orig = entry.tuple_orig elif isinstance(entry, NFCTAttrTuple): tuple_orig = entry else: raise NotImplementedError() - for ndmsg in await self.entry('del', tuple_orig=tuple_orig): + for ndmsg in self.entry('del', tuple_orig=tuple_orig): return ndmsg - async def entry(self, cmd, **kwarg): - for res in await super().entry(cmd, **kwarg): + def entry(self, cmd, **kwargs): + for res in super(Conntrack, self).entry(cmd, **kwargs): return res - async def _dump_entries_task( + def dump_entries( self, mark=None, mark_mask=0xFFFFFFFF, tuple_orig=None, tuple_reply=None, ): - async for ndmsg in await self.dump( + """ + Dump all entries from conntrack table with filters + + Filters can be only part of a conntrack tuple + + :param NFCTAttrTuple tuple_orig: filter on original tuple + :param NFCTAttrTuple tuple_reply: filter on reply tuple + + Examples:: + # Filter only on tcp connections + for entry in ct.dump_entries(tuple_orig=NFCTAttrTuple( + proto=socket.IPPROTO_TCP)): + print("This entry is tcp: {}".format(entry)) + + # Filter only on icmp message to 8.8.8.8 + for entry in ct.dump_entries(tuple_orig=NFCTAttrTuple( + proto=socket.IPPROTO_ICMP, + daddr='8.8.8.8')): + print("This entry is icmp to 8.8.8.8: {}".format(entry)) + """ + for ndmsg in self.dump( mark=mark, mark_mask=mark_mask, tuple_orig=tuple_orig, @@ -206,70 +228,3 @@ async def _dump_entries_task( ndmsg.get_attr('CTA_ID'), ndmsg.get_attr('CTA_USE'), ) - - async def dump_entries( - self, - mark=None, - mark_mask=0xFFFFFFFF, - tuple_orig=None, - tuple_reply=None, - ): - """ - Dump all entries from conntrack table with filters - - Filters can be only part of a conntrack tuple - - :param NFCTAttrTuple tuple_orig: filter on original tuple - :param NFCTAttrTuple tuple_reply: filter on reply tuple - - Examples:: - # Filter only on tcp connections - for entry in ct.dump_entries(tuple_orig=NFCTAttrTuple( - proto=socket.IPPROTO_TCP)): - print("This entry is tcp: {}".format(entry)) - - # Filter only on icmp message to 8.8.8.8 - for entry in ct.dump_entries(tuple_orig=NFCTAttrTuple( - proto=socket.IPPROTO_ICMP, - daddr='8.8.8.8')): - print("This entry is icmp to 8.8.8.8: {}".format(entry)) - """ - return self._dump_entries_task( - mark, mark_mask, tuple_orig, tuple_reply - ) - - -class Conntrack(NFCTSocket): - - def __init__(self, nlm_generator=True, **kwarg): - self.asyncore = AsyncConntrack(**kwarg) - - def stat(self): - return self._run_with_cleanup(self.asyncore.stat) - - def count(self): - return self._run_with_cleanup(self.asyncore.count) - - def conntrack_max_size(self): - return self._run_with_cleanup(self.asyncore.conntrack_max_size) - - def delete(self, entry): - return self._run_with_cleanup(self.asyncore.delete, entry) - - def entry(self, cmd, **kwarg): - return self._run_with_cleanup(self.asyncore.entry, cmd, **kwarg) - - def dump_entries( - self, - mark=None, - mark_mask=0xFFFFFFFF, - tuple_orig=None, - tuple_reply=None, - ): - return self._generate_with_cleanup( - self.asyncore.dump_entries, - mark, - mark_mask, - tuple_orig, - tuple_reply, - ) diff --git a/pyroute2/decoder/args.py b/pyroute2/decoder/args.py deleted file mode 100644 index 32426837f..000000000 --- a/pyroute2/decoder/args.py +++ /dev/null @@ -1,26 +0,0 @@ -import argparse - - -def parse_args(): - argument_parser = argparse.ArgumentParser() - argument_parser.add_argument( - '-c', '--cls', help='message class to use for decoding the data' - ) - argument_parser.add_argument('-d', '--data', help='data dump file') - argument_parser.add_argument( - '-f', '--format', default='hex', help='data file format: hex, pcap' - ) - argument_parser.add_argument( - '-m', '--match', help='match protocol family (only for pcap data)' - ) - argument_parser.add_argument( - '-o', - '--offset', - help='message offset in the data', - default=0, - type=int, - ) - argument_parser.add_argument( - '-k', '--key', help='key format (see struct)', default='H' - ) - return argument_parser.parse_args() diff --git a/pyroute2/decoder/loader.py b/pyroute2/decoder/loader.py deleted file mode 100644 index 59cdea78c..000000000 --- a/pyroute2/decoder/loader.py +++ /dev/null @@ -1,316 +0,0 @@ -import io -import json -import shlex -import struct -from collections import namedtuple -from importlib import import_module - -from pyroute2.common import hexdump, load_dump - -PcapMetaData = namedtuple( - "pCAPMetaData", - ( - "magic_number", - "version_major", - "version_minor", - "thiszone", - "sigfigs", - "snaplen", - "network", - ), -) -PcapPacketHeader = namedtuple( - "PcapPacketHeader", - ("ts_sec", "ts_usec", "incl_len", "orig_len", "header_len"), -) -PcapLLHeader = namedtuple( - "PcapLLHeader", - ("pad0", "addr_type", "pad1", "pad2", "pad3", "family", "header_len"), -) - - -class Message: - - def __init__(self, packet_header, ll_header, met, key, data): - self.packet_header = packet_header - self.ll_header = ll_header - self.cls = None - self.met = met - self.key = key - self.kl = struct.calcsize(self.key) - self.data = data - self.exception = None - self.msg = None - - def get_message_class(self): - if hasattr(self.met, 'msg_map'): - (msg_type,) = struct.unpack(self.key, self.data[4 : 4 + self.kl]) - return self.met.msg_map[msg_type] - return self.met - - def decode(self): - try: - self.cls = self.get_message_class() - self.msg = self.cls(self.data) - self.msg.decode() - self.msg = self.msg.dump() - except Exception as e: - self.exception = repr(e) - self.msg = hexdump(self.data) - - def dump(self): - return { - "pcap header": repr(self.packet_header), - "link layer header": repr(self.ll_header), - "message class": repr(self.cls), - "exception": self.exception, - "data": self.msg, - } - - def __repr__(self): - return json.dumps(self.dump(), indent=4) - - -class MatchOps: - ''' - Functions to match netlink messages. - - The matcher object maintains a stack, where every function - leaves True or False. A message matches only when the stack - contains True. - - Some functions take arguments from the command line, other - like `AND` and `OR` work with the stack. - ''' - - @staticmethod - def AND(): - ''' - Consumes values left on the stack by functions to the - left and to the right in the expression, and leaves - the result of AND operation:: - - func_a{...} AND func_b{...} - ''' - - def f(packet_header, ll_header, raw, data_offset, stack): - v1 = stack.pop() - v2 = stack.pop() - return v1 and v2 - - return f - - @staticmethod - def OR(): - ''' - Consumes values left on the stack by functions to the - left and to the right in the expression, and leaves - the result of OR operation:: - - func_a{...} OR func_b{...} - ''' - - def f(packet_header, ll_header, raw, data_offset, stack): - v1 = stack.pop() - v2 = stack.pop() - return v1 or v2 - - return f - - @staticmethod - def ll_header(family): - ''' - Match link layer header fields. As for now only netlink - family is supported, see `pyroute2.netlink` for netlink - families (`NETLINK_.*`) constants:: - - # match generic netlink messages - ll_header{family=16} - - # match RTNL messages - ll_header{family=0} - ''' - if not isinstance(family, int) or family < 0 or family > 0xFFFF: - raise TypeError('family must be unsigned short integer') - - def f(packet_header, ll_header, raw, data_offset, stack): - if ll_header is None: - return False - return ll_header.family == family - - return f - - @staticmethod - def data(fmt, offset, value): - ''' - Match a voluntary data in the message. Use `struct` notation - for the format, integers for offset and value:: - - # match four bytes with offset 4 bytes and value 16, - # or 10:00:00:00 in hex: - - data{fmt='I', offset=4, value=16} - - # match one byte with offset 16 and value 1, or 01 in hex - - data{fmt='B', offset=16, value=1} - - More examples:: - - # match: - # * generic netlink protocol, 16 - # * message type 37 -- IPVS protocol for this session - # * message command 1 -- IPVS_CMD_NEW_SERVICE - ll_header{family=16} - AND data{fmt='H', offset=4, value=37} - AND data{fmt='B', offset=16, value=1} - ''' - if not isinstance(fmt, str): - raise TypeError('format must be string') - if not isinstance(offset, int) or not isinstance(value, int): - raise TypeError('offset and value must be integers') - - def f(packet_header, ll_header, raw, data_offset, stack): - o = data_offset + offset - s = struct.calcsize(fmt) - return struct.unpack(fmt, raw[o : o + s])[0] == value - - return f - - -class Matcher: - def __init__(self, script): - self.parsed = [] - self.filters = [] - self.script = script - self.shlex = shlex.shlex(instream=io.StringIO(script)) - self.shlex.wordchars += '-~' - postpone = None - while True: - token = self.get_token(ignore=',') - if token == '': - break - method = getattr(MatchOps, token) - if token in ('AND', 'OR'): - postpone = method - continue - kwarg = {} - token = self.get_token(expect='{') - while True: - token = self.get_token(ignore=',') - if token in ('}', ''): - break - self.get_token(expect='=') - value = self.get_token() - if value[0] in ['"', "'"]: - # string - value = value[1:-1] - else: - # int - value = int(value) - kwarg[token] = value - self.filters.append(method(**kwarg)) - if postpone is not None: - self.filters.append(postpone()) - postpone = None - - def get_token(self, expect=None, ignore=None): - token = self.shlex.get_token() - self.parsed.append(token) - if expect is not None and token != expect: - raise SyntaxError(f"expected {expect}: {' '.join(self.parsed)} <-") - if ignore is not None and token in ignore: - token = self.shlex.get_token() - self.parsed.append(token) - return token - - def match(self, packet_header, ll_header, data, offset): - stack = [] - for method in self.filters: - stack.append(method(packet_header, ll_header, data, offset, stack)) - return all(stack) - - -class LoaderHex: - - def __init__(self, data, cls, key, data_offset, script): - with open(data, 'r') as f: - self.raw = load_dump(f) - self.cls = cls - self.key = key - self.offset = 0 - self.matcher = Matcher(script) - - @property - def data(self): - while self.offset < len(self.raw): - msg = Message( - None, None, self.cls, self.key, self.raw[self.offset :] - ) - msg.decode() - if self.matcher.match(None, None, self.raw, self.offset): - yield msg - self.offset += msg.msg['header']['length'] - - -class LoaderPcap: - - def __init__(self, data, cls, key, data_offset, script): - with open(data, 'rb') as f: - self.raw = f.read() - self.metadata = PcapMetaData(*struct.unpack("IHHiIII", self.raw[:24])) - self.offset = 24 - self.key = key - self.data_offset = data_offset - self.cls = cls - self.matcher = Matcher(script) - - def decode_packet_header(self, data, offset): - return PcapPacketHeader( - *struct.unpack("IIII", data[offset : offset + 16]) + (16,) - ) - - def decode_ll_header(self, data, offset): - return PcapLLHeader( - *struct.unpack(">HHIIHH", data[offset : offset + 16]) + (16,) - ) - - @property - def data(self): - while self.offset < len(self.raw): - packet_header = self.decode_packet_header(self.raw, self.offset) - self.offset += packet_header.header_len - ll_header = self.decode_ll_header(self.raw, self.offset) - self.offset += ll_header.header_len - length = packet_header.incl_len - ll_header.header_len - off_length = length - self.data_offset - if self.matcher.match( - packet_header, ll_header, self.raw, self.offset - ): - offset = self.offset + self.data_offset - msg = Message( - packet_header, - ll_header, - self.cls, - self.key, - self.raw[offset : offset + off_length], - ) - msg.decode() - yield msg - self.offset += length - - -def get_loader(args): - if args.cls: - cls = args.cls.replace('/', '.').split('.') - module_name = '.'.join(cls[:-1]) - cls_name = cls[-1] - module = import_module(module_name) - cls = getattr(module, cls_name) - - if args.format == 'pcap': - return LoaderPcap(args.data, cls, args.key, args.offset, args.match) - elif args.format == 'hex': - return LoaderHex(args.data, cls, args.key, args.offset, args.match) - else: - raise ValueError('data format not supported') diff --git a/pyroute2/decoder/main.py b/pyroute2/decoder/main.py deleted file mode 100644 index 3d8f8d29b..000000000 --- a/pyroute2/decoder/main.py +++ /dev/null @@ -1,189 +0,0 @@ -''' -This tool is intended to decode existing data dumps produced with -other tools like tcpdump or strace, and print the data out in -JSON format. - -The strace tool is not as convenient since version 4.13, as it -started to parse some of netlink messages at least partly, -rendering them useless for third party decoders. So if you plan -to use strace to obtain messages, be sure it is older than 4.13. -The strace related manual can be found in archive documentation -for older pyroute2 versions. - -This manual is focused on pcap dumps. - -An example session: - -.. code-block:: console - - # set up netlink monitoring interface - sudo ip link add dev nlmon0 type nlmon - sudo ip link set dev nlmon0 up - - # dump the traffic into a pcap file - # run netlink communication to be captured at the same time - sudo tcpdump -i nlmon0 -w nl.pcap - ^C - - # decode RTNL messages from the dump - pyroute2-decoder \\ - -c pyroute2.netlink.rtnl.marshal.MarshalRtnl \\ - -d nl.pcap \\ - -m "ll_header{family=0}" - -The result will be printed out in JSON format, so you can load -it directly from stdout, or use jq tool to navigate: - -.. code-block:: console - - # print only pcap headers information - pyroute2-decoder ... | jq '.[]."pcap header"' - -pcap data dumps -~~~~~~~~~~~~~~~ - -This format is the default for `pyroute2-decoder`. To explicitly instruct -the decoder to use the pcap format, use `-f pcap` or `--format pcap`. - -An ordinary everyday normal pcap dumps produced by tcpdump. The format -is described here shortly and only to the extent that is important for -the decoder. Please see other resources for detailed pcap format -descriptions. Pyroute2 decoder expect these headers in the pcap dump: - -* Pcap file header. This header is being decoded, but not used by the - tools as for now. -* Packet header. From this header the decoder uses only `incl_len` to - properly read the stored data. -* Link layer header. From this header only the family field is used as - for now, it can be matched with `ll_header{family=...}` expression. - -hex data dumps -~~~~~~~~~~~~~~ - -Use `-f hex` or `--format hex`. - -Just a raw data flow with no service headers added. The decoder uses -message headers to calculate the buffer lengths to read. This dump -can be obtained using strace or the IPBatch compiler. - -Data should use hex bytes representation either in escaped or in -colon separated format. Equivalent variants: - -* `\\\\x49\\\\x61\\\\x03\\\\x55` -* `49:61:03:55` - -Comment strings start with `#`, comments and whitespaces are ignored. -A message example: - -.. code-block:: - - # ifinfmsg headers - # - # nlmsg header - \\x84\\x00\\x00\\x00 # length - \\x10\\x00 # type - \\x05\\x06 # flags - \\x49\\x61\\x03\\x55 # sequence number - \\x00\\x00\\x00\\x00 # pid - # RTNL header - \\x00\\x00 # ifi_family - \\x00\\x00 # ifi_type - \\x00\\x00\\x00\\x00 # ifi_index - \\x00\\x00\\x00\\x00 # ifi_flags - \\x00\\x00\\x00\\x00 # ifi_change - # ... - - -message classes -~~~~~~~~~~~~~~~ - -In order to properly debug the stream, one should specify either -a message class, or a marshal class: - -.. code-block:: console - - # use a message class - pyroute2-decoder \\ - -c pyroute2.netlink.generic.ipvs.ipvsmsg \\ - ... - - # use a marshal class - pyroute2-decoder \\ - -c pyroute2.netlink.rtnl.marshal.MarshalRtnl \\ - ... - -The decoder will try to use the specified class to decode every -matching message. That work well for generic protocols, but for other -protocols like RTNL it's more convenient to use marshal classes -that return corresponding message classes for different message -types. - -generic protocols ids -~~~~~~~~~~~~~~~~~~~~~ - -Generic netlink protocols have dynamic IDs, so the first operation is to -get the ID. The message class used for that is `pyroute2.netlink.ctrlmsg`, -the request is `CTRL_CMD_GETFAMILY == 3`, and the response is -`CTRL_CMD_NEWFAMILY == 1`. The command is one byte right after the netlink -header, so the filters are: - -* `ll_header{family=16}` match family 16, NETLINK_GENERIC -* `data{fmt='B', offset=16, value=1}` match one byte with - value 1 by offset 16 - -Here is the code to get the family ID: - -.. code-block:: console - - pyroute2-decoder \\ - -c pyroute2.netlink.ctrlmsg \\ - -d nl.pcap \\ - -m "ll_header{family=16} AND data{fmt='B', offset=16, value=1}" | \\ - jq \\ - '.[0].data.attrs[] | select(.[0] | contains("FAMILY"))' - - [ - "CTRL_ATTR_FAMILY_NAME", - "IPVS" - ] - [ - "CTRL_ATTR_FAMILY_ID", - 37 - ] - -Having the family ID you can filter out relevant messages. The filters: - -* `ll_header{family=16}` match family 16, NETLINK_GENERIC -* `data{fmt='H', offset=4, value=37}` match IPVS family ID in - the message header -* `data{fmt='B', offset=16, value=1}` match IPVS_CMD_NEW_SERVICE - -.. code-block:: console - - pyroute2-decoder \\ - -c pyroute2.netlink.generic.ipvs.ipvsmsg \\ - -d nl0.pcap \\ - -m "ll_header{family=16} \\ - AND data{fmt='H', offset=4, value=37} \\ - AND data{fmt='B', offset=16, value=1}" - - -''' - -import json - -from pyroute2.common import hexdump -from pyroute2.decoder.args import parse_args -from pyroute2.decoder.loader import get_loader - - -def run(): - loader = get_loader(parse_args()) - ret = [] - for message in loader.data: - ret.append(message.dump()) - print(json.dumps(ret, indent=4, default=lambda x: hexdump(x))) - - -if __name__ == "__main__": - run() diff --git a/pyroute2/devlink.py b/pyroute2/devlink.py index 23f5cb88f..f91cd9834 100644 --- a/pyroute2/devlink.py +++ b/pyroute2/devlink.py @@ -1,56 +1,73 @@ import logging from pyroute2.netlink import NLM_F_DUMP, NLM_F_REQUEST -from pyroute2.netlink.devlink import ( - DEVLINK_NAMES, - AsyncDevlinkSocket, - DevlinkSocket, - devlinkcmd, -) +from pyroute2.netlink.devlink import DEVLINK_NAMES, DevlinkSocket, devlinkcmd log = logging.getLogger(__name__) -class AsyncDL(AsyncDevlinkSocket): - - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - await super().setup_endpoint() - await self.bind() - - async def list(self): - return await self.get_dump() +class DL(DevlinkSocket): + def __init__(self, *argv, **kwarg): + # get specific groups kwarg + if 'groups' in kwarg: + groups = kwarg['groups'] + del kwarg['groups'] + else: + groups = None - async def get_dump(self): - msg = devlinkcmd() - msg['cmd'] = DEVLINK_NAMES['DEVLINK_CMD_GET'] - return await self.nlm_request( - msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP - ) + # get specific async kwarg + if 'async' in kwarg: + # FIXME + # raise deprecation error after 0.5.3 + # + log.warning( + 'use "async_cache" instead of "async", ' + '"async" is a keyword from Python 3.7' + ) + kwarg['async_cache'] = kwarg.pop('async') - async def port_list(self): - return await self.get_port_dump() + if 'async_cache' in kwarg: + async_cache = kwarg.pop('async_cache') + else: + async_cache = False - async def get_port_dump(self): - msg = devlinkcmd() - msg['cmd'] = DEVLINK_NAMES['DEVLINK_CMD_PORT_GET'] - return await self.nlm_request( - msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP - ) + # align groups with async_cache + if groups is None: + groups = ~0 if async_cache else 0 + # continue with init + super(DL, self).__init__(*argv, **kwarg) -class DL(DevlinkSocket): - async_class = AsyncDL + # do automatic bind + # FIXME: unfortunately we can not omit it here + try: + self.bind(groups, async_cache=async_cache) + except: + # thanks to jtluka at redhat.com and the LNST + # team for the fixed fd leak + super(DL, self).close() + raise def list(self): - return self._run_sync_cleanup(self.asyncore.list) + return self.get_dump() def get_dump(self): - return self._run_sync_cleanup(self.asyncore.get_dump) + msg = devlinkcmd() + msg['cmd'] = DEVLINK_NAMES['DEVLINK_CMD_GET'] + return tuple( + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) + ) def port_list(self): - return self._run_sync_cleanup(self.asyncore.port_list) + return self.get_port_dump() def get_port_dump(self): - return self._run_sync_cleanup(self.asyncore.get_port_dump) + msg = devlinkcmd() + msg['cmd'] = DEVLINK_NAMES['DEVLINK_CMD_PORT_GET'] + return tuple( + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) + ) diff --git a/pyroute2/dhcp/__init__.py b/pyroute2/dhcp/__init__.py index c37463eef..a8c72e555 100644 --- a/pyroute2/dhcp/__init__.py +++ b/pyroute2/dhcp/__init__.py @@ -98,63 +98,38 @@ class array8(option): the code 255 is the end of options code. ''' -import logging import struct +import sys from array import array -from typing import ClassVar, NamedTuple, Optional, TypedDict, TypeVar, Union -from pyroute2.protocols import Policy, _decode_mac, _encode_mac, msg +from pyroute2.common import basestring +from pyroute2.protocols import msg -from .enums.dhcp import MessageType, Option +BOOTREQUEST = 1 +BOOTREPLY = 2 -LOG = logging.getLogger(__name__) +DHCPDISCOVER = 1 +DHCPOFFER = 2 +DHCPREQUEST = 3 +DHCPDECLINE = 4 +DHCPACK = 5 +DHCPNAK = 6 +DHCPRELEASE = 7 +DHCPINFORM = 8 -# hack because Self is not supported in py39 -_dhcpmsgSelf = TypeVar('_dhcpmsgSelf', bound='dhcpmsg') -_optionSelf = TypeVar('_optionSelf', bound='option') - - -class ClientId(TypedDict): - '''A dict with 'type' and 'key' keys. - - The types stores the client id type, and the key stores the value. - See RFC for their meaning. - ''' - - type: int - key: Union[bytes, str] - - -def _decode_client_id(value: bytes) -> ClientId: - '''Decode a raw client id option into a dict with type and key. - - If the type is 1, the key is decoded as a mac address, - otherwise it's just the raw bytes. - ''' - type_ = value[0] - key: Union[bytes, str] = value[1:] - - if type_ == 1: - # ethernet - assert isinstance(key, bytes) - key = _decode_mac(value=key) - return ClientId(type=type_, key=key) - - -def _encode_client_id(value: ClientId) -> bytes: - '''Encode a client_id dict into bytes.''' - type_ = value['type'] - key = value['key'] - if type_ == 1: - assert isinstance(key, str), 'client_id must be a mac str when type=1' - key = bytes(_encode_mac(key)) - assert isinstance(key, bytes), 'client_id must be bytes' - return struct.pack('B', type_) + key +if not hasattr(array, 'tobytes'): + # Python2 and Python3 versions of array differ, + # but we need here a consistent API w/o warnings + class array(array): + tobytes = array.tostring class option(msg): - policy: ClassVar[Optional[Policy]] = None + code = 0 + data_length = 0 + policy = None + value = None def __init__(self, content=None, buf=b'', offset=0, value=None, code=0): msg.__init__( @@ -163,7 +138,7 @@ def __init__(self, content=None, buf=b'', offset=0, value=None, code=0): self.code = code @property - def length(self) -> Optional[int]: + def length(self): if self.data_length is None: return None if self.data_length == 0: @@ -171,7 +146,7 @@ def length(self) -> Optional[int]: else: return self.data_length + 2 - def encode(self: _optionSelf) -> _optionSelf: + def encode(self): # pack code self.buf += struct.pack('B', self.code) if self.code in (0, 255): @@ -181,15 +156,12 @@ def encode(self: _optionSelf) -> _optionSelf: self.buf = b'' # pack data into the new buf if self.policy is not None: - value = self.policy.encode(self.value) - if self.policy.format == 'string': + value = self.policy.get('encode', lambda x: x)(self.value) + if self.policy['format'] == 'string': fmt = '%is' % len(value) - if isinstance(value, list): - # Byte strings can be provided as a list of bytes - value = bytes(value) else: - fmt = self.policy.format - if isinstance(value, str): + fmt = self.policy['format'] + if sys.version_info[0] == 3 and isinstance(value, str): value = value.encode('utf-8') self.buf = struct.pack(fmt, value) else: @@ -202,48 +174,27 @@ def encode(self: _optionSelf) -> _optionSelf: self.buf += data return self - def decode(self: _optionSelf) -> _optionSelf: - try: - self.data_length = struct.unpack( - 'B', self.buf[self.offset + 1 : self.offset + 2] - )[0] - except struct.error as err: - raise ValueError( - f'Cannot decode length for DHCP option {self.code}: {err}' - ) + def decode(self): + self.data_length = struct.unpack( + 'B', self.buf[self.offset + 1 : self.offset + 2] + )[0] if self.policy is not None: - if self.policy.format == 'string': + if self.policy['format'] == 'string': fmt = '%is' % self.data_length else: - fmt = self.policy.format - try: - value = struct.unpack( - fmt, - self.buf[ - self.offset + 2 : self.offset + 2 + self.data_length - ], - ) - except struct.error as err: - raise ValueError( - f'Cannot decode option {self.code} ' - f'as {self.policy.format}: {err}' - ) + fmt = self.policy['format'] + value = struct.unpack( + fmt, + self.buf[self.offset + 2 : self.offset + 2 + self.data_length], + ) if len(value) == 1: value = value[0] - value = self.policy.decode(value) - if self.policy.format == 'string': - if isinstance(value, bytes): - try: - # Try to decode as a string - value = value.decode() - except ValueError: - pass - if isinstance(value, str): - # Strip trailing zeroes for strings - value = value.rstrip("\x00") - if isinstance(value, bytes): - # Turn bytes to lists of bytes so they're JSON-encodable - value = list(value) + value = self.policy.get('decode', lambda x: x)(value) + if ( + isinstance(value, basestring) + and self.policy['format'] == 'string' + ): + value = value[: value.find(b'\x00')] self.value = value else: # remember current offset as msg.decode() will advance it @@ -258,139 +209,115 @@ def decode(self: _optionSelf) -> _optionSelf: return self -class CodeMapping(NamedTuple): - name: str - code: int - format: str - - class dhcpmsg(msg): - options: ClassVar[tuple[tuple[Option, str], ...]] = () - - def __init__(self, content=None, buf=b'', offset=0, value=None) -> None: - super().__init__(content, buf, offset, value) - self._encode_map: dict[str, CodeMapping] = {} - self._decode_map: dict[int, CodeMapping] = {} - self._register_options() - - def _register_options(self) -> None: - for code, fmt in self.options: - name = code.name.lower() - self._decode_map[code] = self._encode_map[name] = CodeMapping( - name=name, code=code, format=fmt - ) - - def decode(self: _dhcpmsgSelf) -> _dhcpmsgSelf: + options = () + l2addr = None + _encode_map = {} + _decode_map = {} + + def _register_options(self): + for option in self.options: + code, name, fmt = option[:3] + self._decode_map[code] = self._encode_map[name] = { + 'name': name, + 'code': code, + 'format': fmt, + } + + def decode(self): msg.decode(self) + self._register_options() self['options'] = {} while self.offset < len(self.buf): - code: int = struct.unpack( - 'B', self.buf[self.offset : self.offset + 1] - )[0] - if code == Option.PAD: + code = struct.unpack('B', self.buf[self.offset : self.offset + 1])[ + 0 + ] + if code == 0: self.offset += 1 continue - if code == Option.END: + if code == 255: return self - if code in self._decode_map: - # use the known decoded & name - option_class = getattr(self, self._decode_map[code].format) - optname = self._decode_map[code].name - else: - # code is unknown - # if we know this option number, get the value as a list - # even if we don't know how to parse it - option_class = self.array8 - try: - # if we know this option name, use it - code = Option(code) - optname = code.name.lower() - except ValueError: - optname = f"option{code}" - - option = option_class(buf=self.buf, offset=self.offset, code=code) - try: - option.decode() - except ValueError as err: - # FIXME: maybe we would like the raw option data - # if we can't decode it ? but that would complicate typing - LOG.error("%s", err) - break + # code is unknown -- bypass it + if code not in self._decode_map: + length = struct.unpack( + 'B', self.buf[self.offset + 1 : self.offset + 2] + )[0] + self.offset += length + 2 + continue + + # code is known, work on it + option_class = getattr(self, self._decode_map[code]['format']) + option = option_class(buf=self.buf, offset=self.offset) + option.decode() self.offset += option.length if option.value is not None: value = option.value else: value = option - self['options'][optname] = value + self['options'][self._decode_map[code]['name']] = value return self - def encode(self: _dhcpmsgSelf) -> _dhcpmsgSelf: + def encode(self): msg.encode(self) + self._register_options() # put message type - options = self.get('options') or {'message_type': MessageType.DISCOVER} + options = self.get('options') or { + 'message_type': DHCPDISCOVER, + 'parameter_list': [1, 3, 6, 12, 15, 28], + } self.buf += ( - self.uint8(code=Option.MESSAGE_TYPE, value=options['message_type']) + self.uint8(code=53, value=options['message_type']).encode().buf + ) + self.buf += ( + self.client_id({'type': 1, 'key': self['chaddr']}, code=61) .encode() .buf ) + self.buf += self.string(code=60, value='pyroute2').encode().buf + for name, value in options.items(): - if name == 'message_type': + if name in ('message_type', 'client_id', 'vendor_id'): continue - if (code_mapping := self._encode_map.get(name)) is None: + fmt = self._encode_map.get(name, {'format': None})['format'] + if fmt is None: continue - # name is known, ok - option_class = getattr(self, code_mapping.format) + option_class = getattr(self, fmt) if isinstance(value, dict): - option = option_class(value, code=code_mapping.code) + option = option_class( + value, code=self._encode_map[name]['code'] + ) else: - option = option_class(code=code_mapping.code, value=value) + option = option_class( + code=self._encode_map[name]['code'], value=value + ) self.buf += option.encode().buf - self.buf += self.none(code=Option.END).encode().buf + self.buf += self.none(code=255).encode().buf return self class none(option): pass class be16(option): - policy = Policy(format='>H') + policy = {'format': '>H'} class be32(option): - policy = Policy(format='>I') - - class sbe32(option): - policy = Policy(format='>i') + policy = {'format': '>I'} class uint8(option): - policy = Policy(format='B') + policy = {'format': 'B'} class string(option): - policy = Policy(format='string') + policy = {'format': 'string'} class array8(option): - policy = Policy( - format='string', - encode=lambda x: array('B', x).tobytes(), - decode=lambda x: array('B', x).tolist(), - ) + policy = { + 'format': 'string', + 'encode': lambda x: array('B', x).tobytes(), + 'decode': lambda x: array('B', x).tolist(), + } class client_id(option): - policy = Policy( - format='string', encode=_encode_client_id, decode=_decode_client_id - ) - - def __init__( - self, content=None, buf=b'', offset=0, value=None, code=0 - ): - # FIXME: we have to override value w/ content here, because - # when trying to encode an option, if it's a dict, its value is - # set to None. - super().__init__(content, buf, offset, value=content, code=code) - - class message_type(option): - policy = Policy(format='B', decode=MessageType) - - class bool(option): - policy = Policy(format='B', encode=int, decode=bool) + fields = (('type', 'uint8'), ('key', 'l2addr')) diff --git a/pyroute2/dhcp/cli.py b/pyroute2/dhcp/cli.py deleted file mode 100644 index 5f7936a45..000000000 --- a/pyroute2/dhcp/cli.py +++ /dev/null @@ -1,185 +0,0 @@ -import asyncio -import logging -from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser -from importlib import import_module -from typing import Any, Optional - -from pyroute2.dhcp.client import AsyncDHCPClient, ClientConfig -from pyroute2.dhcp.fsm import State -from pyroute2.dhcp.hooks import Hook -from pyroute2.dhcp.iface_status import InterfaceNotFound, InterfaceStateWatcher -from pyroute2.dhcp.leases import Lease - -LOG = logging.getLogger(__name__) - - -def import_dotted_name(name: str) -> Any: - '''Import anything by name. Return None if the import wasn't successful.''' - try: - module_name, obj_name = name.rsplit('.', 1) - module = import_module(module_name) - return getattr(module, obj_name) - except (ValueError, ImportError, AttributeError): - return None - - -def get_psr() -> ArgumentParser: - psr = ArgumentParser( - description='A DHCP client based on pyroute2. ' - 'Tries to obtain & keep a lease on an interface, running ' - 'configurable hooks to assign the obtained IP address and gw to it.', - epilog='Send a SIGUSR1 to renew the current lease, SIGUSR2 to rebind ' - 'it and SIGHUP to reset & get a new lease.', - formatter_class=ArgumentDefaultsHelpFormatter, - ) - psr.add_argument( - 'interface', help='The interface to request an address for.' - ) - psr.add_argument( - '--lease-type', - help='Class to use for leases. ' - 'Must be a subclass of `pyroute2.dhcp.leases.Lease`.', - type=str, - default='pyroute2.dhcp.leases.JSONFileLease', - metavar='dotted.name', - ) - psr.add_argument( - '--hook', - help='Hooks to load. ' - 'These are used to run async python code when, ' - 'for example, renewing or expiring a lease. ' - 'Defaults to adding & removing ip & gateway.', - action='append', - type=str, - metavar='dotted.name', - ) - psr.add_argument( - '--disable-hooks', - help='Disable all hooks.', - default=False, - action='store_true', - ) - psr.add_argument( - '-x', - '--exit-on-timeout', - metavar='N', - help='Wait for max N seconds for a lease, ' - 'exit if none could be obtained.', - type=float, - ) - psr.add_argument( - '--log-level', - help='Logging level to use.', - choices=('DEBUG', 'INFO', 'WARNING', 'ERROR'), - default='INFO', - ) - psr.add_argument( - '-p', - '--write-pidfile', - default=False, - action='store_true', - help='Write a pid file in the working directory. ', - ) - psr.add_argument( - '-R', - '--no-release', - default=False, - action='store_true', - help='Do not send a DHCPRELEASE on exit.', - ) - # TODO: add options for parameters, retransmission, timeouts... - return psr - - -async def run_client( - cfg: ClientConfig, exit_timeout: Optional[float] = None -) -> None: - '''Run the client until interrupted, or a timeout occurs. - - The optional `exit_timeout` controls 2 things when provided: - - How long to wait for the interface to be up - - How long to wait for the client to be bound when starting up - ''' - - acli = AsyncDHCPClient(cfg) - - async with InterfaceStateWatcher(cfg.interface) as iface_watcher: - while True: - # Open the socket, read existing lease, etc - if iface_watcher.state != 'up': - LOG.info('Waiting for %s to go up...', cfg.interface) - await asyncio.wait_for( - iface_watcher.up.wait(), timeout=exit_timeout - ) - async with acli: - # Bootstrap the client by sending a DISCOVER or a REQUEST - await acli.bootstrap() - if exit_timeout: - # Wait a bit for a lease, and raise if we have none - await acli.wait_for_state( - State.BOUND, timeout=exit_timeout - ) - break - await iface_watcher.down.wait() - LOG.warning('%s went down', cfg.interface) - - -async def main() -> None: - psr = get_psr() - args = psr.parse_args() - logging.basicConfig( - format='%(asctime)s %(levelname)s [%(name)s:%(funcName)s] %(message)s' - ) - logging.getLogger('pyroute2.dhcp').setLevel(args.log_level) - - LOG.setLevel(args.log_level) - - # parse lease type - lease_type = import_dotted_name(args.lease_type) - if not (isinstance(lease_type, type) and issubclass(lease_type, Lease)): - psr.error(f'{args.lease_type!r} must point to a Lease subclass.') - - # parse hooks - hooks: list[Hook] = [] - if not args.disable_hooks: - if not args.hook: - args.hook = [ - 'pyroute2.dhcp.hooks.configure_ip', - 'pyroute2.dhcp.hooks.add_default_gw', - 'pyroute2.dhcp.hooks.remove_default_gw', - 'pyroute2.dhcp.hooks.remove_ip', - ] - LOG.debug('Configured hooks:') - for dotted_hook_name in args.hook: - hook = import_dotted_name(dotted_hook_name) - if not isinstance(hook, Hook): - psr.error(f'{dotted_hook_name!r} must point to a valid hook.') - hooks.append(hook) - LOG.debug("- %s", hook.name) - - # Create configuration - cfg = ClientConfig( - interface=args.interface, - lease_type=lease_type, - hooks=hooks, - write_pidfile=args.write_pidfile, - release=not args.no_release, - handle_signals=True, - ) - - try: - await run_client(cfg, exit_timeout=args.exit_on_timeout) - except InterfaceNotFound as err: - psr.error(f"Interface not found: {err}") - - -def run(): - # for the setup.cfg entrypoint - try: - asyncio.run(main()) - except KeyboardInterrupt: - pass - - -if __name__ == '__main__': # pragma: no cover - run() diff --git a/pyroute2/dhcp/client.py b/pyroute2/dhcp/client.py index 82a0f4344..7b62bd813 100644 --- a/pyroute2/dhcp/client.py +++ b/pyroute2/dhcp/client.py @@ -1,680 +1,82 @@ -import asyncio -import errno -import os -import random -from dataclasses import dataclass, field -from logging import getLogger -from math import floor -from pathlib import Path -from signal import Signals -from socket import gethostname -from time import time -from typing import Callable, DefaultDict, Iterable, Iterator, Optional, Union +import json +import select +import sys -from pyroute2.dhcp import fsm, messages -from pyroute2.dhcp.dhcp4socket import AsyncDHCP4Socket -from pyroute2.dhcp.enums import dhcp -from pyroute2.dhcp.hooks import Hook, Trigger, run_hooks -from pyroute2.dhcp.leases import JSONFileLease, Lease -from pyroute2.dhcp.timers import LeaseTimers -from pyroute2.dhcp.xids import Xid +from pyroute2.dhcp import ( + BOOTREQUEST, + DHCPACK, + DHCPDISCOVER, + DHCPOFFER, + DHCPREQUEST, +) +from pyroute2.dhcp.dhcp4msg import dhcp4msg +from pyroute2.dhcp.dhcp4socket import DHCP4Socket -LOG = getLogger(__name__) +def req(s, poll, msg, expect): + do_req = True + xid = None -# TODO: maybe move retransmission stuff to its own file -Retransmission = Callable[[], Union[Iterator[int], Iterator[float]]] - - -def randomized_increasing_backoff( - wait_first: float = 4.0, wait_max: float = 32.0, factor: float = 2.0 -) -> Iterator[float]: - '''Yields seconds to wait until the next retry, forever.''' - delay = wait_first while True: - yield delay - if delay <= wait_max: - delay = min(random.uniform(delay, delay * factor), wait_max) - - -@dataclass -class ClientConfig: - '''Stores configuration option for the DHCP client.''' - - # FIXME: this is probably not enough if we want the client to work with - # WLAN interfaces, as we would have to handle the SSID somehow ? - # The interface to bind to and obtain a lease for. - interface: str - # The lease type to use, allows flexibility on where and how to store it. - lease_type: type[Lease] = JSONFileLease - # A list of hooks that will be called when the required triggers are met. - hooks: Iterable[Hook] = () - # The DHCP parameters requested by the client. - requested_parameters: Iterable[dhcp.Option] = ( - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.DOMAIN_NAME, - dhcp.Option.LEASE_TIME, - dhcp.Option.RENEWAL_TIME, - dhcp.Option.REBINDING_TIME, - ) - # Timeouts for various client states. - # If the client reaches the specified amount of seconds in one of the - # configured states, it will reset the lease process and start anew. - timeouts: dict[fsm.State, int] = field( - default_factory=lambda: { - # No use staying too long in REBOOTING state if nobody is answering - fsm.State.REBOOTING: 10, - # When we get an OFFER, how long should we wait for an ACK ? - fsm.State.REQUESTING: 30, + # get transaction id + if do_req: + xid = s.put(msg)['xid'] + # wait for response + events = poll.poll(2) + for fd, event in events: + response = s.get() + if response['xid'] != xid: + do_req = False + continue + if response['options']['message_type'] != expect: + raise Exception("DHCP protocol error") + return response + do_req = True + + +def action(ifname): + s = DHCP4Socket(ifname) + poll = select.poll() + poll.register(s, select.POLLIN | select.POLLPRI) + + # DISCOVER + discover = dhcp4msg( + { + 'op': BOOTREQUEST, + 'chaddr': s.l2addr, + 'options': { + 'message_type': DHCPDISCOVER, + 'parameter_list': [1, 3, 6, 12, 15, 28], + }, } ) - # FIXME: we send too many retries according to the RFC - # In both RENEWING and REBINDING states, if the client receives no - # response to its DHCPREQUEST message, the client SHOULD wait one-half - # of the remaining time until T2 (in RENEWING state) and one-half of - # the remaining lease time (in REBINDING state), down to a minimum of - # 60 seconds, before retransmitting the DHCPREQUEST message. - retransmission: Retransmission = randomized_increasing_backoff - # Whether to write a pidfile in the working directory - write_pidfile: bool = False - # Send a DHCPRELEASE on client exit - release: bool = True - # Maximum execution duration for a single hook - hook_timeout: Optional[float] = 2.0 - # Custom client id. The mac is used as the client id if not provided. - client_id: Optional[bytes] = None - # Whether to handle USR1 (renew), USR2 (rebind) and HUP (reset) - handle_signals: bool = True - # optional vendor_id & hostname options included in requests - vendor_id: Optional[str] = 'pyroute2' - host_name: Optional[str] = field(default_factory=gethostname) - - @property - def pidfile_path(self) -> Path: - '''Where to write the pid file. It's named after the interface.''' - return ( - Path.cwd() - .joinpath(self.interface) - .with_name(f"{self.interface}.pid") - ) - - -class AsyncDHCPClient: - '''A DHCP client based on pyroute2. - - Implemented as an async context manager running a finite state machine. - - The client will try to acquire and keep a lease as long as it's running. - - It can run hooks specified in its configuration to perform various - actions such as adding an IP address to an interface, configuring routing, - when the specified events are triggered (see the `hook()` decorator.) - - Exiting the context manager causes the lease to be released and relevant - hooks to be run. - - Example usage:: - - from pyroute2.dhcp.client import AsyncDHCPClient, ClientConfig - from pyroute2.dhcp.fsm import State - - cfg = ClientConfig(interface='eth3') - async with AsyncDHCPClient(cfg) as client: - # Bootstrap the client by sending a DISCOVER or a REQUEST - await client.bootstrap() - # Wait 20s until the client gets a lease - await client.wait_for_state(State.BOUND, timeout=20.0) - - ''' - - def __init__(self, config: ClientConfig) -> None: - self.config = config - # The raw socket used to send and receive packets - self._sock: AsyncDHCP4Socket = AsyncDHCP4Socket(self.config.interface) - # Current client state - self._state: fsm.State = fsm.State.OFF - # Current lease, read from persistent storage or sent by a server - self._lease: Optional[Lease] = None - # dhcp messages put in this queue are sent by _send_forever - self._sendq: asyncio.Queue[Optional[messages.SentDHCPMessage]] = ( - asyncio.Queue() - ) - # Handle to run _send_forever for the context manager's lifetime - self._sender_task: Optional[asyncio.Task] = None - # Handle to run _recv_forever for the context manager's lifetime - self._receiver_task: Optional[asyncio.Task] = None - # Timers to run callbacks on lease timeouts expiration - self.lease_timers = LeaseTimers() - # Calls reset() after a timeout in some states to avoid getting stuck - self._state_watchdog: asyncio.Task | None = None - # Allows to easily track the state when running the client from python - self._states: DefaultDict[fsm.State, asyncio.Event] = DefaultDict( - asyncio.Event - ) - # Used to compute lease times, taking into account the request time - self.last_state_change: float = time() - self._xid: Optional[Xid] = None - - # 'public api' - - async def wait_for_state( - self, state: fsm.State, timeout: Optional[float] = None - ) -> None: - '''Waits until the client is in the target state.''' - try: - await asyncio.wait_for(self._states[state].wait(), timeout=timeout) - except asyncio.exceptions.TimeoutError as err: - raise asyncio.exceptions.TimeoutError( - f'Timed out waiting for the {state.name} state. ' - f'Current state: {self.state.name}' - ) from err - - @fsm.state_guard(fsm.State.INIT, fsm.State.INIT_REBOOT) - async def bootstrap(self) -> None: - '''Send a `DISCOVER` or a `REQUEST`, - - depending on whether we're initializing or rebooting. - - Use this to get a lease when running the client from Python code. - ''' - if self.state is fsm.State.INIT: - # send discover - await self.transition( - to=fsm.State.SELECTING, - send=messages.discover( - parameter_list=self.config.requested_parameters - ), - ) - elif self.state is fsm.State.INIT_REBOOT: - assert self.lease, 'cannot init_reboot without a lease' - # send request for lease - await self.transition( - to=fsm.State.REBOOTING, - send=messages.request_for_lease( - parameter_list=self.config.requested_parameters, - lease=self.lease, - state=fsm.State.REBOOTING, - ), - ) - # the decorator prevents the needs for an else - - # properties - - @property - def lease(self) -> Optional[Lease]: - '''The current lease, if we have one.''' - return self._lease - - @lease.setter - def lease(self, value: Lease) -> None: - '''Set a fresh lease; only call this when a server grants one.''' - - self._lease = value - self.lease_timers.arm( - lease=self._lease, - renewal=self._renew, - rebinding=self._rebind, - expiration=self._expire_lease, - ) - # whatever error might happen when writing a lease, - # it should never make the client crash - try: - self._lease.dump() - except Exception as exc: - LOG.error('Could not dump lease: %s', exc) - - @property - def state(self) -> fsm.State: - '''The current client state.''' - return self._state - - @state.setter - def state(self, value: fsm.State) -> None: - '''Check the client can transition to the state, and set it. - - Only supposed to be called by the client internally. - ''' - old_state = self.state - if value and value not in fsm.TRANSITIONS[old_state]: - raise ValueError( - f'Cannot transition from {self._state.name} to {value.name}' - ) - LOG.info('%s -> %s', old_state.name, value.name) - if self._state_watchdog: - self._state_watchdog.cancel() - self._state_watchdog = None - if old_state in self._states: - self._states[old_state].clear() - self._state = value - self.last_state_change = time() - self._states[value].set() - if state_timeout := self.config.timeouts.get(value): - self._state_watchdog = asyncio.Task( - self.reset(delay=state_timeout) - ) - - @property - def xid(self) -> Xid: - '''The xid currently in use by the client. - - Only available when the context manager is running. - ''' - if self._xid is None: - raise AttributeError('xid is not defined') - return self._xid - - # Timer callbacks - - @fsm.state_guard(fsm.State.BOUND) - async def _renew(self, signal: Optional[Signals] = None) -> None: - '''Called when the renewal time defined in the lease expires.''' - assert self.lease, 'cannot renew without an existing lease' - LOG.info( - '%s, renewing lease', - f'{signal.name} received' if signal else 'T1 expired', - ) - self.lease_timers._reset_timer('renewal') # FIXME should be automatic - await self.transition( - to=fsm.State.RENEWING, - send=messages.request_for_lease( - parameter_list=self.config.requested_parameters, - lease=self.lease, - state=fsm.State.RENEWING, - ), - ) - - @fsm.state_guard(fsm.State.RENEWING, fsm.State.BOUND) - async def _rebind(self, signal: Optional[Signals] = None) -> None: - '''Called when the rebinding time defined in the lease expires.''' - assert self.lease, 'cannot rebind without an existing lease' - LOG.info( - '%s, rebinding lease', - f'{signal.name} received' if signal else 'T2 expired', - ) - # if the user asks for a rebind before the renewal timer expires, - # better cancel the renewal timer too - self.lease_timers._reset_timer('renewal') - self.lease_timers._reset_timer('rebinding') # FIXME - await self.transition( - to=fsm.State.REBINDING, - send=messages.request_for_lease( - parameter_list=self.config.requested_parameters, - lease=self.lease, - state=fsm.State.REBINDING, - ), - ) - - async def _expire_lease(self) -> None: - '''Called when the expiration time defined in the lease expires.''' - LOG.info('Lease expired') - self.lease_timers._reset_timer('expiration') - await self._run_hooks(Trigger.EXPIRED) - await self.reset() - - # DHCP packet sending & receving coroutines - - async def _send_message(self, msg: messages.SentDHCPMessage) -> None: - '''Set secs, xid & client id on the message, and send it.''' - - # Set secs to the time elapsed since the last state change - # (max 16 bits) - msg.dhcp['secs'] = min(floor(time() - self.last_state_change), 0xFFFF) - msg.dhcp['xid'] = self.xid.for_state(self.state) - - # set the client id, rfc says: - # A hardware type of 0 (zero) should be used when the value field - # contains an identifier other than a hardware address (e.g. a fully - # qualified domain name). - if self.config.client_id: - client_id = {'type': 0, 'key': self.config.client_id} - else: - # default behavior, use hw type & hw addr as client id - client_id = {'type': 1, 'key': self._sock.l2addr} - msg.dhcp['options']['client_id'] = client_id - # set hostname & vendor id if configured to do so - if self.config.host_name: - msg.dhcp['options']['host_name'] = self.config.host_name - if self.config.vendor_id: - msg.dhcp['options']['vendor_id'] = self.config.vendor_id - LOG.info('Sending %s', msg) - await self._sock.put(msg) - - async def _send_forever(self) -> None: - '''Send packets from `_sendq` until the client is in `State.OFF`.''' - msg_to_send: Optional[messages.SentDHCPMessage] = None - # Called to get the interval value below - interval_factory: Optional[Union[Iterator[int], Iterator[float]]] = ( - None - ) - # How long to sleep before retrying - interval: Union[int, float] = 1 - - # this is triggered by __aexit__ - wait_til_off = asyncio.Task(self.wait_for_state(fsm.State.OFF)) - while not (wait_til_off.done() and self._sendq.empty()): - - # FIXME: interval handling is pretty awkward and convoluted - if interval_factory: - interval = next(interval_factory) - else: - interval = 9999999 - - wait_for_msg_to_send = asyncio.Task( - self._sendq.get(), name='wait for packet to send' - ) - if msg_to_send: - LOG.debug('%.1f seconds until retransmission', interval) - done, pending = await asyncio.wait( - (wait_til_off, wait_for_msg_to_send), - return_when=asyncio.FIRST_COMPLETED, - timeout=interval, - ) - if wait_for_msg_to_send in done: - if msg_to_send := wait_for_msg_to_send.result(): - # There is a new message to send, reset the interval - interval_factory = self.config.retransmission() - else: - # No need to retry anything - interval_factory = None - elif wait_for_msg_to_send in pending: - wait_for_msg_to_send.cancel() - if msg_to_send: - if ( - msg_to_send.message_type != dhcp.MessageType.RELEASE - and wait_til_off.done() - ): - LOG.debug( - 'Not sending %s, client is shutting down', - msg_to_send.message_type.name, - ) - continue - try: - await self._send_message(msg_to_send) - except OSError as err: - # That happens when the interface goes down. - # In theses cases, the client is supposed to be restarted - if err.errno == errno.ENETDOWN: - LOG.error('Could not send, network is down') - return - raise - - async def _recv_forever(self) -> None: - '''Receive & process DHCP messages until the client stops.''' - - # TODO: is there a better way to wait for the client to stop ? - wait_til_off = asyncio.Task(self.wait_for_state(fsm.State.OFF)) - - while not wait_til_off.done(): - wait_for_received_msg = asyncio.Task( - coro=self._sock.get(), - name=f'wait for DHCP messages on {self.config.interface}', - ) - # sleep until a new message is received, or the client is OFF - done, pending = await asyncio.wait( - (wait_til_off, wait_for_received_msg), - return_when=asyncio.FIRST_COMPLETED, - ) - if wait_for_received_msg in done: - try: - received_msg = wait_for_received_msg.result() - except OSError as err: - # That happens when the interface goes down. - # In theses cases, the client is supposed to be restarted - if err.errno == errno.ENETDOWN: - LOG.error('Could not recv, network is down') - return - raise - await self._process_msg(received_msg) - - elif wait_for_received_msg in pending: - wait_for_received_msg.cancel() - - # Callbacks for received DHCP messages - - @fsm.state_guard( - fsm.State.REQUESTING, - fsm.State.REBOOTING, - fsm.State.REBINDING, - fsm.State.RENEWING, - ) - async def ack_received(self, msg: messages.ReceivedDHCPMessage) -> None: - '''Called when an ACK is received. - - Stores the lease and puts the client in the BOUND state. - ''' - # FIXME: according to the RFC: - # When the client receives a DHCPACK from the server, the client - # computes the lease expiration time as the sum of the time at which - # the client sent the DHCPREQUEST message and the duration of the lease - # in the DHCPACK message. - - # The state the client was in when sending the message - request_state = msg.xid.request_state - - if request_state in (fsm.State.REQUESTING, fsm.State.REBOOTING): - trigger = Trigger.BOUND - elif request_state == fsm.State.RENEWING: - trigger = Trigger.RENEWED - elif request_state == fsm.State.REBINDING: - trigger = Trigger.REBOUND - else: - LOG.warning('Invalid request state for xid %s', msg.xid) - return - - if 'lease_time' not in msg.dhcp['options']: - # that would not make sense at all, but still... - LOG.warning('Server did not define a lease time, ignoring ACK.') - return - - self.lease = self.config.lease_type( - ack=msg.dhcp, - interface=self.config.interface, - server_mac=msg.eth_src, - ) - LOG.info( - 'Got lease for %s from %s (%s)', - self.lease.ip, - self.lease.server_id, - msg.eth_src, - ) - await self.transition(to=fsm.State.BOUND) - await self._run_hooks(trigger) - - @fsm.state_guard( - fsm.State.REQUESTING, - fsm.State.REBOOTING, - fsm.State.RENEWING, - fsm.State.REBINDING, + reply = req(s, poll, discover, expect=DHCPOFFER) + + # REQUEST + request = dhcp4msg( + { + 'op': BOOTREQUEST, + 'chaddr': s.l2addr, + 'options': { + 'message_type': DHCPREQUEST, + 'requested_ip': reply['yiaddr'], + 'server_id': reply['options']['server_id'], + 'parameter_list': [1, 3, 6, 12, 15, 28], + }, + } ) - async def nak_received(self, msg: messages.ReceivedDHCPMessage) -> None: - '''Called when a NAK is received. - - Resets the client and starts looking for a new IP. - ''' - await self.reset() - - @fsm.state_guard(fsm.State.SELECTING) - async def offer_received(self, msg: messages.ReceivedDHCPMessage) -> None: - '''Called when an OFFER is received. - - Sends a REQUEST for the offered IP address. - ''' - # FIXME: we should probably validate the offer some more, - # like checking if the offered options are satisfying - await self.transition( - to=fsm.State.REQUESTING, - send=messages.request_for_offer( - parameter_list=self.config.requested_parameters, offer=msg - ), - ) - - # Async context manager methods - - async def __aenter__(self) -> 'AsyncDHCPClient': - '''Set up the client so it's ready to obtain an IP. - - Tries to load a lease for the client's interface, - opens the socket, starts the sender & receiver tasks - and allocates a request ID. - ''' - if self.config.handle_signals: - self._register_signal_handlers() - self._xid = Xid() - if self.config.write_pidfile: - self.config.pidfile_path.write_text(str(os.getpid())) - LOG.debug('Wrote pidfile to %s', self.config.pidfile_path) - if loaded_lease := self.config.lease_type.load(self.config.interface): - self._lease = loaded_lease - self.state = fsm.State.INIT_REBOOT - else: - LOG.debug('No current lease') - self.state = fsm.State.INIT - await self._sock.__aenter__() - - self._receiver_task = asyncio.Task( - self._recv_forever(), - name=f'Listen for DHCP packets on {self.config.interface}', - ) - self._sender_task = asyncio.Task( - self._send_forever(), - name=f'Send outgoing DHCP packets on {self.config.interface}', - ) - return self - - async def __aexit__(self, *_) -> None: - '''Shut down the client. - - If there's an active lease, send a RELEASE for it first. - ''' - if self.config.handle_signals: - self._remove_signal_handlers() - self.lease_timers.cancel() - if self.lease and self.config.release: - await self._run_hooks(Trigger.UNBOUND) - if not self.lease.expired: - await self._sendq.put(messages.release(lease=self.lease)) - self.state = fsm.State.OFF - if self._sender_task: - await self._sender_task - if self._receiver_task: - await self._receiver_task - await self._sock.__aexit__() - self._xid = None - if self.config.write_pidfile: - self.config.pidfile_path.unlink(missing_ok=True) - LOG.debug('Removed pidfile at %s', self.config.pidfile_path) - - # internal methods - - async def transition( - self, to: fsm.State, send: Optional[messages.SentDHCPMessage] = None - ) -> None: - '''Change the client's state, and start sending a message repeatedly. - - If the message is None, any current message will stop being sent. - ''' - self.state = to - await self._sendq.put(send) - - async def reset( - self, delay: float = 0.0, signal: Optional[Signals] = None - ) -> None: - '''Called internally to restart the lease acquisition process. - - - When the client receives a NAK; - - When it spends too much time in a configured state - (see `ClientConfig.timeouts`) - - When the current lease expires. - - Upon receiving a SIGHUP - - Erases the lease, cancel timers, resets the xid, and sends - DISCOVERs to get a new lease. - ''' - if delay: - await asyncio.sleep(delay=delay) - LOG.warning('Resetting after %.1f seconds', delay) - if signal: - LOG.info('%s received, resetting', signal.name) - await self.transition(to=fsm.State.INIT) - # Reset lease & timers and start again - self._lease = None - self.lease_timers.cancel() - self._xid = Xid() - await self.bootstrap() - - async def _run_hooks(self, trigger: Trigger) -> None: - '''Run hooks for the given trigger. - - Hooks are awaited and not run in the background; that means they could - cause issues if they run for too long. Hence, each hook is limited to - the `hook_timeout` config setting. - ''' - assert self.lease, 'tried to run hooks without a lease' - await run_hooks( - hooks=self.config.hooks, - lease=self.lease, - trigger=trigger, - timeout=self.config.hook_timeout, - ) - - async def _process_msg(self, msg: messages.ReceivedDHCPMessage) -> None: - '''Processes a received DHCP message. - - The messages's xid is checked against the client's xid, and the - appropriate async handler (`self._received`) - is then called if it exists. - ''' - msg_type = dhcp.MessageType(msg.dhcp['options']['message_type']) - LOG.info('Received %s', msg) - if not self.xid.matches(msg.xid): - LOG.error( - 'Incorrect xid %s (expected %sX), discarding', - msg.xid, - hex(self.xid.random_part)[:-1], - ) - return - handler_name = f'{msg_type.name.lower()}_received' - handler = getattr(self, handler_name, None) - if not handler: - LOG.debug('DHCP %s messages are not handled', msg_type.name) - else: - await handler(msg) - - def _add_signal_handler( - self, - loop: asyncio.AbstractEventLoop, - signal: Signals, - callback: fsm.AsyncCallback, - ): - '''Arranges for `callback` to be called when receiving `signal`. - - `callback` must take a `signal` keyword argument. - ''' - loop.add_signal_handler( - sig=signal, - callback=lambda: asyncio.create_task(callback(signal=signal)), - ) + reply = req(s, poll, request, expect=DHCPACK) + s.close() + return reply - def _register_signal_handlers(self): - '''Add signal handlers to catch USR1, USR2 and HUP. - Called by the context manager when `handle_signals` is True. +def run(): + if len(sys.argv) > 1: + ifname = sys.argv[1] + else: + ifname = 'eth0' + print(json.dumps(action(ifname), indent=4)) - - SIGUSR1 causes a renew, - - SIGUSR2 causes a rebind, - - SIGHUP causes a reset. - ''' - LOG.info('Registering signal handlers for USR1, USR2 and HUP') - loop = asyncio.get_running_loop() - self._add_signal_handler(loop, Signals.SIGUSR1, self._renew) - self._add_signal_handler(loop, Signals.SIGUSR2, self._rebind) - self._add_signal_handler(loop, Signals.SIGHUP, self.reset) - def _remove_signal_handlers(self): - remove_handler = asyncio.get_running_loop().remove_signal_handler - LOG.info('Removing signal handlers for USR1, USR2 and HUP') - remove_handler(Signals.SIGUSR1) - remove_handler(Signals.SIGUSR2) - remove_handler(Signals.SIGHUP) +if __name__ == '__main__': + run() diff --git a/pyroute2/dhcp/dhcp4msg.py b/pyroute2/dhcp/dhcp4msg.py index b17eefec6..ec5f93c6a 100644 --- a/pyroute2/dhcp/dhcp4msg.py +++ b/pyroute2/dhcp/dhcp4msg.py @@ -1,7 +1,6 @@ from socket import AF_INET, inet_ntop, inet_pton -from pyroute2.dhcp import Policy, dhcpmsg, option -from pyroute2.dhcp.enums.dhcp import Option +from pyroute2.dhcp import dhcpmsg, option class dhcp4msg(dhcpmsg): @@ -13,9 +12,9 @@ class dhcp4msg(dhcpmsg): ('htype', 'uint8', 1), # ethernet ('hlen', 'uint8', 6), # ethernet addr len ('hops', 'uint8'), - ('xid', 'be32'), - ('secs', 'be16'), - ('flags', 'be16'), + ('xid', 'uint32'), + ('secs', 'uint16'), + ('flags', 'uint16'), ('ciaddr', 'ip4addr'), ('yiaddr', 'ip4addr'), ('siaddr', 'ip4addr'), @@ -29,95 +28,42 @@ class dhcp4msg(dhcpmsg): # https://www.ietf.org/rfc/rfc2132.txt # options = ( - # TODO: add & test more options - (Option.PAD, 'none'), - (Option.SUBNET_MASK, 'ip4addr'), - (Option.TIME_OFFSET, 'be32'), - (Option.ROUTER, 'ip4list'), - (Option.TIME_SERVER, 'ip4list'), - (Option.IEN_NAME_SERVER, 'ip4list'), - (Option.NAME_SERVER, 'ip4list'), - (Option.LOG_SERVER, 'ip4list'), - (Option.COOKIE_SERVER, 'ip4list'), - (Option.LPR_SERVER, 'ip4list'), - (Option.IMPRESS_SERVER, 'ip4list'), - (Option.RESOURCE_LOCATION_SERVER, 'ip4list'), - (Option.HOST_NAME, 'string'), - # in multiples of 512 bytes - (Option.BOOT_FILE_SIZE, 'be16'), - (Option.MERIT_DUMP_FILE, 'string'), - (Option.DOMAIN_NAME, 'string'), - (Option.SWAP_SERVER, 'ip4addr'), - (Option.ROOT_PATH, 'string'), - (Option.EXTENSIONS_PATH, 'string'), - (Option.IP_FORWARDING, 'bool'), - (Option.NON_LOCAL_SOURCE_ROUTING, 'bool'), - # TODO: dict of ip/mask pairs - # (Option.POLICY_FILTER, '?'), - # minimum legal value is 567 according to rfc - (Option.MAX_DATAGRAM_REASSEMBLY, 'be16'), - (Option.DEFAULT_TTL, 'uint8'), - (Option.PATH_MTU_AGING_TIMEOUT, 'be32'), - # TODO: list of be16 - # (Option.PATH_MTU_PLATEAU_TABLE, '?'), - (Option.INTERFACE_MTU, 'be16'), - (Option.ALL_SUBNETS_LOCAL, 'bool'), - (Option.BROADCAST_ADDRESS, 'ip4addr'), - (Option.PERFORM_MASK_DISCOVERY, 'bool'), - (Option.MASK_SUPPLIER, 'bool'), - (Option.PERFORM_ROUTER_DISCOVERY, 'bool'), - (Option.ROUTER_SOLICITATION_ADDRESS, 'ip4addr'), - # TODO: dict of ip addrs - # (Option.STATIC_ROUTE, '?') - (Option.TRAILER_ENCAPSULATION, 'bool'), - (Option.ARP_CACHE_TIMEOUT, 'be32'), - (Option.ETHERNET_ENCAPSULATION, 'bool'), - (Option.TCP_DEFAULT_TTL, 'uint8'), - (Option.TCP_KEEPALIVE_INTERVAL, 'be32'), - (Option.TCP_KEEPALIVE_GARBAGE, 'bool'), - (Option.NIS_DOMAIN, 'string'), - (Option.NDS_SERVERS, 'ip4list'), - (Option.NTP_SERVERS, 'ip4list'), - (Option.VENDOR_SPECIFIC_INFORMATION, 'string'), - (Option.NETBIOS_NAME_SERVER, 'ip4list'), - (Option.NETBIOS_DDG_SERVER, 'ip4list'), - # 1=B, 2=P, 4=M, 8=H, probably obsolete & unused - (Option.NETBIOS_NODE_TYPE, 'uint8'), - (Option.NETBIOS_SCOPE, 'string'), - (Option.X_WINDOW_FONT_SERVER, 'ip4list'), - (Option.X_WINDOW_DISPLAY_MANAGER, 'ip4list'), - (Option.REQUESTED_IP, 'ip4addr'), - (Option.LEASE_TIME, 'sbe32'), - # 1: options in file, 2: options in sname, 4, both - (Option.OPTION_OVERLOAD, 'uint8'), - (Option.TFTP_SERVER_NAME, 'string'), - (Option.BOOTFILE_NAME, 'string'), - (Option.MESSAGE_TYPE, 'message_type'), - (Option.SERVER_ID, 'ip4addr'), - (Option.PARAMETER_LIST, 'array8'), - (Option.MESSAGE, 'string'), - # minimum value: 576 bytes - (Option.MAX_MSG_SIZE, 'be16'), - (Option.RENEWAL_TIME, 'sbe32'), - (Option.REBINDING_TIME, 'sbe32'), - (Option.VENDOR_ID, 'string'), - (Option.CLIENT_ID, 'client_id'), - (Option.END, 'none'), + (0, 'pad', 'none'), + (1, 'subnet_mask', 'ip4addr'), + (2, 'time_offset', 'be32'), + (3, 'router', 'ip4list'), + (4, 'time_server', 'ip4list'), + (5, 'ien_name_server', 'ip4list'), + (6, 'name_server', 'ip4list'), + (7, 'log_server', 'ip4list'), + (8, 'cookie_server', 'ip4list'), + (9, 'lpr_server', 'ip4list'), + (50, 'requested_ip', 'ip4addr'), + (51, 'lease_time', 'be32'), + (53, 'message_type', 'uint8'), + (54, 'server_id', 'ip4addr'), + (55, 'parameter_list', 'array8'), + (57, 'messagi_size', 'be16'), + (58, 'renewal_time', 'be32'), + (59, 'rebinding_time', 'be32'), + (60, 'vendor_id', 'string'), + (61, 'client_id', 'client_id'), + (255, 'end', 'none'), ) class ip4addr(option): - policy = Policy( - format='4s', - encode=lambda x: inet_pton(AF_INET, x), - decode=lambda x: inet_ntop(AF_INET, x), - ) + policy = { + 'format': '4s', + 'encode': lambda x: inet_pton(AF_INET, x), + 'decode': lambda x: inet_ntop(AF_INET, x), + } class ip4list(option): - policy = Policy( - format='string', - encode=lambda x: b''.join([inet_pton(AF_INET, i) for i in x]), - decode=lambda x: [ + policy = { + 'format': 'string', + 'encode': lambda x: ''.join([inet_pton(AF_INET, i) for i in x]), + 'decode': lambda x: [ inet_ntop(AF_INET, x[i * 4 : i * 4 + 4]) for i in range(len(x) // 4) ], - ) + } diff --git a/pyroute2/dhcp/dhcp4socket.py b/pyroute2/dhcp/dhcp4socket.py index 3088677b8..9b2286b08 100644 --- a/pyroute2/dhcp/dhcp4socket.py +++ b/pyroute2/dhcp/dhcp4socket.py @@ -4,225 +4,129 @@ ''' -import asyncio -import logging -import socket -from typing import Optional - -from pyroute2.compat import ETHERTYPE_IP +from pyroute2.common import AddrPool from pyroute2.dhcp.dhcp4msg import dhcp4msg -from pyroute2.dhcp.messages import ReceivedDHCPMessage, SentDHCPMessage -from pyroute2.ext.bpf import BPF -from pyroute2.ext.rawsocket import AsyncRawSocket +from pyroute2.ext.rawsocket import RawSocket from pyroute2.protocols import ethmsg, ip4msg, udp4_pseudo_header, udpmsg -LOG = logging.getLogger(__name__) - - -UDP_HEADER_SIZE = 8 -IPV4_HEADER_SIZE = 20 -SKF_AD_OFF = -0x1000 -SKF_AD_VLAN_TAG_PRESENT = 48 - -def listen_udp_port(port: int = 68) -> list[list[int]]: - '''BPF filter that matches Ethernet + IPv4 + UDP on the given port. - - Packets tagged on a vlan are also dropped, see - https://lore.kernel.org/netdev/51FB6A9D.2050002@redhat.com/T/ - ''' +def listen_udp_port(port=68): + # pre-scripted BPF code that matches UDP port bpf_code = [ - # Load vlan presence indicator - [BPF.LD + BPF.B + BPF.ABS, 0, 0, SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT], - # bail out immediately if there is one and we don't want it - [BPF.JMP + BPF.JEQ + BPF.K, 0, 10, 0], - # Load eth type at offset 12 and check it's IPv4 - [BPF.LD + BPF.H + BPF.ABS, 0, 0, 12], - [BPF.JMP + BPF.JEQ + BPF.K, 0, 8, 0x0800], - # Load IP proto at offset 23 and check it's UDP - [BPF.LD + BPF.B + BPF.ABS, 0, 0, 23], - [BPF.JMP + BPF.JEQ + BPF.K, 0, 6, socket.IPPROTO_UDP], - # load frag offset at offset 20 - [BPF.LD + BPF.H + BPF.ABS, 0, 0, 20], - # Check mask & drop fragmented packets - [BPF.JMP + BPF.JSET + BPF.K, 4, 0, 8191], - # load ip header length at offset 14 - [BPF.LDX + BPF.B + BPF.MSH, 0, 0, 14], - # load udp dport from that offset + 16 and check it - [BPF.LD + BPF.H + BPF.IND, 0, 0, 16], - [BPF.JMP + BPF.JEQ + BPF.K, 0, 1, port], - # allow packet - [BPF.RET + BPF.K, 0, 0, 65535], - # drop packet - [BPF.RET + BPF.K, 0, 0, 0], + [40, 0, 0, 12], + [21, 0, 8, 2048], + [48, 0, 0, 23], + [21, 0, 6, 17], + [40, 0, 0, 20], + [69, 4, 0, 8191], + [177, 0, 0, 14], + [72, 0, 0, 16], + [21, 0, 1, port], + [6, 0, 0, 65535], + [6, 0, 0, 0], ] return bpf_code -class AsyncDHCP4Socket(AsyncRawSocket): +class DHCP4Socket(RawSocket): ''' Parameters: * ifname -- interface name to work on - * port -- UDP port to listen on - This raw socket binds to an interface and installs a BPF filter - to receive (non-VLAN) messages only on the specified UDP port. - It can be used in poll/select and implements the async context manager - protocol, so can be used in `aysnc with` statements. + This raw socket binds to an interface and installs BPF filter + to get only its UDP port. It can be used in poll/select and + provides also the context manager protocol, so can be used in + `with` statements. + + It does not provide any DHCP state machine, and does not inspect + DHCP packets, it is totally up to you. No default values are + provided here, except `xid` -- DHCP transaction ID. If `xid` is + not provided, DHCP4Socket generates it for outgoing messages. ''' - def __init__(self, ifname: str, port: int = 68): - AsyncRawSocket.__init__(self, ifname, listen_udp_port(port)) + def __init__(self, ifname, port=68): + RawSocket.__init__(self, ifname, listen_udp_port(port)) self.port = port - self._loop = asyncio.get_running_loop() - - @property - def loop(self) -> asyncio.AbstractEventLoop: - # We define this as a property because it's easier to patch in tests - return self._loop - - async def put(self, msg: SentDHCPMessage) -> SentDHCPMessage: - '''Send a DHCP message. - - This encapsulates the `SentDHCPMessage` into Ethernet, IPv4, and UDP. - - If not provided, both the Ethernet source and the DHCP chaddr are set - to the MAC address of the underlying socket's interface. - - Example:: - - msg = SentDHCPMessage( - dhcp=dhcp4msg({ - 'op': bootp.MessageType.BOOTREQUEST, - 'flags': bootp.Flag.BROADCAST, - 'options': { - 'message_type': dhcp.MessageType.DISCOVER, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.LEASE_TIME, - ], - }, - }), - ) - await sock.put(msg) + # Create xid pool + # + # Every allocated xid will be released automatically after 1024 + # alloc() calls, there is no need to call free(). Minimal xid == 16 + self.xid_pool = AddrPool(minaddr=16, release=1024) + + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def put(self, msg=None, dport=67): ''' + Put DHCP message. Parameters: + + * msg -- dhcp4msg instance + * dport -- DHCP server port - if msg.sport != self.port: - raise ValueError( - f"Client source port is set to {self.port}, " - f"cannot send message from port {msg.sport}." - ) + If `msg` is not provided, it is constructed as default + BOOTREQUEST + DHCPDISCOVER. - if not msg.eth_src: - msg.eth_src = self.l2addr + Examples:: + sock.put(dhcp4msg({'op': BOOTREQUEST, + 'chaddr': 'ff:11:22:33:44:55', + 'options': {'message_type': DHCPREQUEST, + 'parameter_list': [1, 3, 6, 12, 15], + 'requested_ip': '172.16.101.2', + 'server_id': '172.16.101.1'}})) + + The method returns dhcp4msg that was sent, so one can get from + there `xid` (transaction id) and other details. + ''' # DHCP layer - dhcp = msg.dhcp + dhcp = msg or dhcp4msg({'chaddr': self.l2addr}) - # auto add src addr - if dhcp['chaddr'] is None: - dhcp['chaddr'] = msg.eth_src + # dhcp transaction id + if dhcp['xid'] is None: + dhcp['xid'] = self.xid_pool.alloc() data = dhcp.encode().buf - dhcp_payload_size = len(data) # UDP layer udp = udpmsg( - { - 'sport': self.port, - 'dport': msg.dport, - 'len': UDP_HEADER_SIZE + dhcp_payload_size, - } + {'sport': self.port, 'dport': dport, 'len': 8 + len(data)} ) - # Pseudo UDP header, only for checksum purposes udph = udp4_pseudo_header( - { - 'src': msg.ip_src, - 'dst': msg.ip_dst, - 'len': UDP_HEADER_SIZE + dhcp_payload_size, - } + {'dst': '255.255.255.255', 'len': 8 + len(data)} ) udp['csum'] = self.csum(udph.encode().buf + udp.encode().buf + data) udp.reset() # IPv4 layer ip4 = ip4msg( - { - 'len': IPV4_HEADER_SIZE + UDP_HEADER_SIZE + dhcp_payload_size, - 'proto': socket.IPPROTO_UDP, - 'dst': msg.ip_dst, - 'src': msg.ip_src, - } + {'len': 20 + 8 + len(data), 'proto': 17, 'dst': '255.255.255.255'} ) ip4['csum'] = self.csum(ip4.encode().buf) ip4.reset() # MAC layer eth = ethmsg( - {'dst': msg.eth_dst, 'src': msg.eth_src, 'type': ETHERTYPE_IP} + {'dst': 'ff:ff:ff:ff:ff:ff', 'src': self.l2addr, 'type': 0x800} ) data = eth.encode().buf + ip4.encode().buf + udp.encode().buf + data - await self.loop.sock_sendall(self, data) + self.send(data) dhcp.reset() - return msg + return dhcp - async def get(self) -> ReceivedDHCPMessage: + def get(self): ''' Get the next incoming packet from the socket and try - to decode it as IPv4 DHCP. - - Packets that cannot be decoded are logged & discarded. - Invalid/truncated packets raise `ValueError`. - - Example:: - - # Send a DISCOVER and read an OFFER - disco = messages.discover(parameter_list=[ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.LEASE_TIME, - ]) - await sock.put(disco) - offer = await sock.get() - print('received', offer.message_type, - 'from', offer.eth_src, - 'lease time', offer.dhcp['options']['lease_time'], - ) - - ''' - msg: Optional[ReceivedDHCPMessage] = None - while not msg: - raw = await self.loop.sock_recv(self, 4096) - try: - msg = self._decode_msg(raw) - except ValueError as err: - LOG.error('%s', err) - return msg - - @classmethod - def _decode_msg(cls, data: bytes) -> ReceivedDHCPMessage: - '''Decode an IPv4 DHCP packet from bytes. - - No analysis is done here. The MAC/IPv4/UDP headers are stripped out, - the relevant values are stored in the `ReceivedDHCPMessage` metadata, - and the rest is interpreted as DHCP. + to decode it as IPv4 DHCP. No analysis is done here, + only MAC/IPv4/UDP headers are stripped out, and the + rest is interpreted as DHCP. ''' + (data, addr) = self.recvfrom(4096) eth = ethmsg(buf=data).decode() ip4 = ip4msg(buf=data, offset=eth.offset).decode() udp = udpmsg(buf=data, offset=ip4.offset).decode() - dhcp = dhcp4msg(buf=data, offset=udp.offset).decode() - return ReceivedDHCPMessage( - dhcp=dhcp, - eth_src=eth['src'], - eth_dst=eth['dst'], - ip_src=ip4['src'], - ip_dst=ip4['dst'], - sport=udp['sport'], - dport=udp['dport'], - ) + return dhcp4msg(buf=data, offset=udp.offset).decode() diff --git a/pyroute2/dhcp/enums/__init__.py b/pyroute2/dhcp/enums/__init__.py deleted file mode 100644 index 9cb0220d5..000000000 --- a/pyroute2/dhcp/enums/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import bootp, dhcp # noqa: F401 diff --git a/pyroute2/dhcp/enums/bootp.py b/pyroute2/dhcp/enums/bootp.py deleted file mode 100644 index 20403441c..000000000 --- a/pyroute2/dhcp/enums/bootp.py +++ /dev/null @@ -1,21 +0,0 @@ -from enum import IntEnum, IntFlag - - -class MessageType(IntEnum): - BOOTREQUEST = 1 # Client to server - BOOTREPLY = 2 # Server to client - - -class HardwareType(IntEnum): - ETHERNET = 1 # Ethernet (10Mb) - EXPERIMENTAL_ETHERNET = 2 - AMATEUR_RADIO = 3 - TOKEN_RING = 4 - FDDI = 8 - ATM = 19 - WIRELESS_IEEE_802_11 = 20 - - -class Flag(IntFlag): - UNICAST = 0x0000 # Unicast response requested - BROADCAST = 0x8000 # Broadcast response requested diff --git a/pyroute2/dhcp/enums/dhcp.py b/pyroute2/dhcp/enums/dhcp.py deleted file mode 100644 index 2c176885c..000000000 --- a/pyroute2/dhcp/enums/dhcp.py +++ /dev/null @@ -1,201 +0,0 @@ -from enum import IntEnum - - -class MessageType(IntEnum): - '''DHCP message types, see RFC 2131 table 2.''' - - DISCOVER = 1 - OFFER = 2 - REQUEST = 3 - DECLINE = 4 - ACK = 5 - NAK = 6 - RELEASE = 7 - INFORM = 8 - - -class Option(IntEnum): - '''DHCP options and parameters. - - These constants are used for two purposes: - - - Requesting configuration values from the DHCP server. - In this case, they're passed in the `PARAMETER_LIST` request option. - Not all values are valid for parameter requests: - - `PAD` is only used as padding when decoding server-sent options, - - `PARAMETER_LIST` is only sent from client and cannot be requested, - - `END` is used as a marker when decoding server-sent options. - - - Reading responses. - `Option` numbers are used to parse DHCP options sent by the server. - They are translated to their name by the client and stored in leases. - ''' - - PAD = 0 - SUBNET_MASK = 1 - TIME_OFFSET = 2 - ROUTER = 3 - TIME_SERVER = 4 - IEN_NAME_SERVER = 5 # prehistoric DNS - # this should be DOMAIN_NAME_SERVER but it's often used & shorted this way - NAME_SERVER = 6 # plain old DNS - LOG_SERVER = 7 - COOKIE_SERVER = 8 - LPR_SERVER = 9 - IMPRESS_SERVER = 10 - RESOURCE_LOCATION_SERVER = 11 - HOST_NAME = 12 - BOOT_FILE_SIZE = 13 - MERIT_DUMP_FILE = 14 - DOMAIN_NAME = 15 - SWAP_SERVER = 16 - ROOT_PATH = 17 - EXTENSIONS_PATH = 18 - IP_FORWARDING = 19 - NON_LOCAL_SOURCE_ROUTING = 20 - POLICY_FILTER = 21 - MAX_DATAGRAM_REASSEMBLY = 22 - DEFAULT_TTL = 23 - PATH_MTU_AGING_TIMEOUT = 24 - PATH_MTU_PLATEAU_TABLE = 25 - INTERFACE_MTU = 26 - ALL_SUBNETS_LOCAL = 27 - BROADCAST_ADDRESS = 28 - PERFORM_MASK_DISCOVERY = 29 - MASK_SUPPLIER = 30 - PERFORM_ROUTER_DISCOVERY = 31 - ROUTER_SOLICITATION_ADDRESS = 32 - STATIC_ROUTE = 33 - TRAILER_ENCAPSULATION = 34 - ARP_CACHE_TIMEOUT = 35 - ETHERNET_ENCAPSULATION = 36 - TCP_DEFAULT_TTL = 37 - TCP_KEEPALIVE_INTERVAL = 38 - TCP_KEEPALIVE_GARBAGE = 39 - NIS_DOMAIN = 40 - NIS_SERVERS = 41 - NTP_SERVERS = 42 - VENDOR_SPECIFIC_INFORMATION = 43 - NETBIOS_NAME_SERVER = 44 - NETBIOS_DDG_SERVER = 45 - NETBIOS_NODE_TYPE = 46 - NETBIOS_SCOPE = 47 - X_WINDOW_FONT_SERVER = 48 - X_WINDOW_DISPLAY_MANAGER = 49 - REQUESTED_IP = 50 - LEASE_TIME = 51 - OPTION_OVERLOAD = 52 - MESSAGE_TYPE = 53 - SERVER_ID = 54 - PARAMETER_LIST = 55 - MESSAGE = 56 - MAX_MSG_SIZE = 57 - RENEWAL_TIME = 58 - REBINDING_TIME = 59 - VENDOR_ID = 60 - CLIENT_ID = 61 - NETWARE_IP_DOMAIN = 62 - NETWARE_IP_OPTION = 63 - NIS_PLUS_DOMAIN = 64 - NIS_PLUS_SERVERS = 65 - TFTP_SERVER_NAME = 66 - BOOTFILE_NAME = 67 - MOBILE_IP_HOME_AGENT = 68 - SMTP_SERVER = 69 - POP3_SERVER = 70 - NNTP_SERVER = 71 - DEFAULT_WWW_SERVER = 72 - DEFAULT_FINGER_SERVER = 73 - DEFAULT_IRC_SERVER = 74 - STREETTALK_SERVER = 75 - STDA_SERVER = 76 - USER_CLASS_INFORMATION = 77 - SLP_DIRECTORY_AGENT = 78 - SLP_SERVICE_SCOPE = 79 - RAPID_COMMIT = 80 - CLIENT_FQDN = 81 - RELAY_AGENT_INFORMATION = 82 - INTERNET_STORAGE_NAME_SERVICE = 83 - # 84 is unused - NDS_SERVERS = 85 - NDS_TREE_NAME = 86 - NDS_CONTEXT = 87 - BCMCS_CONTROLLER_DOMAIN = 88 - BCMS_CONTROLLER_IP = 89 - AUTHENTICATION = 90 - LQ_CLIENT_LAST_TRANSACTION = 91 - LQ_ASSOCIATED_IP = 92 - CLIENT_SYSTEM_ARCHITECTURE_TYPE = 93 - CLIENT_NETWORK_INTERFACE_IDENTIFIER = 94 - LDAP_SERVERS = 95 - # 96 is unused - PXE_CLIENT_ID = 97 - OPEN_GROUP_USER_AUTH = 98 - GEOCONF_CIVIC = 99 - POSIX_TIMEZONE = 100 - TZDB_TIMEZONE = 101 - # 102 to 107 are unused - IPV6_ONLY_PREFERRED = 108 - DHCP4TO6_S46_SADDR = 109 - # 110 & 111 are unused - NETINFO_PARENT_SERVER_ADDR = 112 - NETINFO_PARENT_SERVER_TAG = 113 - DHCP_CAPTIVE_PORTAL = 114 - # 115 is unused - AUTO_CONFIGURE = 116 - NAME_SERVICE_SEARCH = 117 - SUBNET_SELECTION = 118 - DOMAIN_SEARCH = 119 - SIP_SERVERS = 120 - CLASSLESS_STATIC_ROUTE = 121 - CABLELABS_CLIENT_CONFIG = 122 - GEOCONF = 123 - VENDOR_IDENTIFYING_CLASS = 124 - VENDOR_IDENTIFYING_OPTS = 125 - # 126 & 127 are unused - MCNS_SECURITY_SERVER = 128 - # 129 to 135 are unused - PANA_AGENT = 136 - LOST_SERVER = 137 - CAPWAP_AC_V4 = 138 - MOS_ADDR = 139 - MOS_FQDN = 140 - SIP_UA_CS_DOMAINS = 141 - ANDSF_V4 = 142 - SZTP_REDIRECT = 143 - GEOLOC = 144 - FORCERENEW_NONCE_CAPABLE = 145 - RDNSS_SELECTION = 146 - DOTS_REFERENCE_IDENTIFIER = 147 - DOTS_ADDR = 148 - # 149 is unused - TFTP_SERVER_ADDR = 150 - STATUS_CODE = 151 - BASE_TIME = 152 - STATE_START_TIME = 153 - QUERY_START_TIME = 154 - QUERY_END_TIME = 155 - DHCP_STATE = 156 - DATA_SOURCE = 157 - PCP_SERVER = 158 - PORT_PARAMS = 159 - OLD_CAPTIVE_PORTAL = 160 - MUD_URL = 161 - CISCO_REQUESTED_HOST_NAME = 162 - CISCO_LAST_TRANSACTION_TIME = 163 - # 164 to 184 are unused - VPN_ID = 185 - # 186 to 207 are unused - PXELINUX_MAGIC = 208 - PXELINUX_CONFIG_FILE = 209 - PXELINUX_PATH_PREFIX = 210 - PXELINUX_REBOOT_TIME = 211 - DHCP_6RD = 212 - ACCESS_DOMAIN_NAME = 213 - # 214 to 219 seem unused - SUBNET_ALLOC = 220 - VIRTUAL_SUBNET_SELECTION = 221 - PRIVATE_CLASSIC_ROUTE_MS = 249 - CISCO_AUTO_CONFIGURE = 251 - PRIVATE_PROXY_AUTODISCOVERY = 252 - END = 255 diff --git a/pyroute2/dhcp/fsm.py b/pyroute2/dhcp/fsm.py deleted file mode 100644 index b6263bd57..000000000 --- a/pyroute2/dhcp/fsm.py +++ /dev/null @@ -1,69 +0,0 @@ -'''DHCP client state machine helpers.''' - -import functools -from enum import IntEnum, auto -from logging import getLogger -from typing import Any, Callable, Final, Protocol - -LOG = getLogger(__name__) - - -class State(IntEnum): - '''DHCP client states. - - see - http://www.tcpipguide.com/free/t_DHCPGeneralOperationandClientFiniteStateMachine.htm - ''' - - OFF = 0 - INIT = auto() - INIT_REBOOT = auto() - REBOOTING = auto() - REQUESTING = auto() - SELECTING = auto() - BOUND = auto() - RENEWING = auto() - REBINDING = auto() - - -# allowed transitions between states -TRANSITIONS: Final[dict[State, set[State]]] = { - State.OFF: {State.INIT, State.INIT_REBOOT}, - State.INIT_REBOOT: {State.REBOOTING}, - State.REBOOTING: {State.INIT, State.BOUND}, - State.INIT: {State.SELECTING}, - State.SELECTING: {State.REQUESTING, State.INIT}, - State.REQUESTING: {State.BOUND, State.INIT}, - State.BOUND: {State.INIT, State.RENEWING, State.REBINDING}, - State.RENEWING: {State.BOUND, State.INIT, State.REBINDING}, - State.REBINDING: {State.BOUND, State.INIT}, -} - - -class AsyncCallback(Protocol): - __name__: str - - async def __call__(self, *a, **kwds: Any) -> Any: - '''Protocol for async methods that makes mypy happy.''' - - -def state_guard(*states: State) -> Callable[[AsyncCallback], AsyncCallback]: - '''Decorator that prevents a method from running - - if the associated instance is not in one of the given States.''' - - def decorator(meth: AsyncCallback) -> AsyncCallback: - @functools.wraps(meth) - async def wrapper(self, *args: Any, **kwargs: Any) -> None: - if self.state not in states: - LOG.debug( - 'Ignoring call to %r in %s state', - meth.__name__, - self.state.name, - ) - return - await meth(self, *args, **kwargs) - - return wrapper - - return decorator diff --git a/pyroute2/dhcp/hooks.py b/pyroute2/dhcp/hooks.py deleted file mode 100644 index b5545dc62..000000000 --- a/pyroute2/dhcp/hooks.py +++ /dev/null @@ -1,201 +0,0 @@ -'''Hooks called by the DHCP client when bound, a leases expires, etc.''' - -import asyncio -import errno -from enum import auto -from logging import getLogger -from typing import Callable, Iterable, NamedTuple, Optional, Protocol - -from pyroute2.compat import StrEnum -from pyroute2.dhcp.leases import Lease, MissingOptionError -from pyroute2.iproute.linux import AsyncIPRoute -from pyroute2.netlink.exceptions import NetlinkError - -LOG = getLogger(__name__) - - -class Trigger(StrEnum): - '''Events that can trigger hooks in the client.''' - - # The client has obtained a new lease - BOUND = auto() - # The client has voluntarily relinquished its lease - UNBOUND = auto() - # The client has renewed its lease after the renewal timer expired - RENEWED = auto() - # The client has rebound its leas after the rebinding timer expired - REBOUND = auto() - # The lease has expired (the client will restart the lease process) - EXPIRED = auto() - - -class HookFunc(Protocol): - '''Signature for functions that can be passed to the hook decorator.''' - - __name__: str - - async def __call__(self, lease: Lease) -> None: - pass # pragma: no cover - - -class Hook(NamedTuple): - '''Stores a hook function and its triggers. - - Returned by the `hook()` decorator; no need to subclass or instantiate. - ''' - - func: HookFunc - triggers: set[Trigger] - - async def __call__(self, lease: Lease) -> None: - '''Call the hook function.''' - await self.func(lease=lease) - - @property - def name(self) -> str: - '''Shortcut for the function name.''' - return self.func.__name__ - - -async def run_hooks( - hooks: Iterable[Hook], - lease: Lease, - trigger: Trigger, - timeout: Optional[float] = None, -): - '''Called by the client to run the hooks registered for the given trigger. - - The optional `timeout` arguments causes individual hooks to timeout - if they exceed it. - - Exceptions are handled and printed, but don't prevent the other hooks from - running. - - .. warning:: - The timeout is async, which means that hooks that block on non-async - code can ignore it and freeze the whole DHCP client ! - - ''' - if hooks := list(hooks): - LOG.debug('Running %s hooks', trigger) - for i in filter(lambda y: trigger in y.triggers, hooks): - try: - await asyncio.wait_for(i(lease), timeout=timeout) - except asyncio.exceptions.TimeoutError: - LOG.error('Hook %r timed out', i.name) - except Exception as exc: - LOG.error('Hook %s failed: %r', i.name, exc) - - -def hook(*triggers: Trigger) -> Callable[[HookFunc], Hook]: - '''Decorator for dhcp client hooks. - - A hook is an async function that takes a lease. - Hooks set in `ClientConfig.hooks` will be called in order by the client - when one of the triggers passed to the decorator happens. - - For example:: - - @hook(Trigger.RENEWED) - async def lease_was_renewed(lease: Lease): - print(lease.server_mac, 'renewed our lease !') - - .. warning:: - - blocking non-async code in hooks might freeze the client - - long-running async hooks might be canceled after a timeout - - The decorator returns a `Hook` instance, a utility class storing the hook - function and its triggers. - - .. warning:: - The hooks API might still change. - - ''' - - def decorator(hook_func: HookFunc) -> Hook: - return Hook(func=hook_func, triggers=set(triggers)) - - return decorator - - -@hook(Trigger.BOUND) -async def configure_ip(lease: Lease): - '''Add the IP allocated in the lease to its interface. - - Use the `remove_ip` hook in addition to this one for cleanup. - The DHCP server must have set the subnet mask and broadcast address. - ''' - LOG.info( - 'Adding %s/%s to %s', lease.ip, lease.subnet_mask, lease.interface - ) - try: - bcast: Optional[str] = lease.broadcast_address - except MissingOptionError as err: - LOG.debug("%s", err) - bcast = None - async with AsyncIPRoute(ext_ack=True, strict_check=True) as ipr: - await ipr.addr( - 'replace', - index=await ipr.link_lookup(ifname=lease.interface), - address=lease.ip, - prefixlen=lease.subnet_mask, - broadcast=bcast, - ) - - -@hook(Trigger.UNBOUND, Trigger.EXPIRED) -async def remove_ip(lease: Lease): - '''Remove the IP in the lease from its interface.''' - LOG.info( - 'Removing %s/%s from %s', lease.ip, lease.subnet_mask, lease.interface - ) - async with AsyncIPRoute(ext_ack=True, strict_check=True) as ipr: - await ipr.addr( - 'del', - index=await ipr.link_lookup(ifname=lease.interface), - address=lease.ip, - prefixlen=lease.subnet_mask, - ) - - -@hook(Trigger.BOUND) -async def add_default_gw(lease: Lease): - '''Configures the default gateway set in the lease. - - Use in addition to `remove_default_gw` for cleanup. - ''' - LOG.info( - 'Adding %s as default route through %s', - lease.default_gateway, - lease.interface, - ) - async with AsyncIPRoute(ext_ack=True, strict_check=True) as ipr: - ifindex = (await ipr.link_lookup(ifname=lease.interface),) - await ipr.route( - 'replace', - dst='0.0.0.0/0', - gateway=lease.default_gateway, - oif=ifindex, - ) - - -@hook(Trigger.UNBOUND, Trigger.EXPIRED) -async def remove_default_gw(lease: Lease): - '''Removes the default gateway set in the lease.''' - LOG.info('Removing %s as default route', lease.default_gateway) - async with AsyncIPRoute(ext_ack=True, strict_check=True) as ipr: - ifindex = await ipr.link_lookup(ifname=lease.interface) - try: - await ipr.route( - 'del', - dst='0.0.0.0/0', - gateway=lease.default_gateway, - oif=ifindex, - ) - except NetlinkError as err: - if err.code == errno.ESRCH: - LOG.info( - 'Default route was already removed by another process' - ) - else: - LOG.error('Got a netlink error: %s', err) diff --git a/pyroute2/dhcp/iface_status.py b/pyroute2/dhcp/iface_status.py deleted file mode 100644 index ae4a3d4ec..000000000 --- a/pyroute2/dhcp/iface_status.py +++ /dev/null @@ -1,104 +0,0 @@ -import asyncio -import sys -from typing import Literal, Optional - -from pyroute2.iproute.linux import AsyncIPRoute -from pyroute2.netlink.exceptions import NetlinkError -from pyroute2.netlink.rtnl import RTMGRP_LINK - -IfaceState = Literal['up', 'down'] - - -class InterfaceNotFound(LookupError): - '''Raised when an interface is not found.''' - - -class InterfaceStateWatcher: - '''Async context manager, fires events when an interface changes state. - - Used by the dhcp client to restart itself in such cases. - ''' - - def __init__(self, interface: str) -> None: - self.interface = interface - self.up = asyncio.Event() - self.down = asyncio.Event() - self._state: Optional[IfaceState] = None - self._watcher: Optional[asyncio.Task] = None - self._ipr: Optional[AsyncIPRoute] = None - - @property - def ipr(self) -> AsyncIPRoute: - '''The async iproute context. Only useable in the context manager.''' - assert self._ipr, 'need to use as a context manager' - return self._ipr - - @property - def state(self) -> IfaceState: - '''The current state of the watched interface.''' - assert self._state, "not yet started" - return self._state - - @state.setter - def state(self, value: IfaceState) -> None: - '''Set the state & trigger the relevant event.''' - self._state = value - if value == 'up': - self.up.set() - self.down.clear() - elif value == 'down': - self.down.set() - self.up.clear() - - async def _fetch_current_state(self) -> IfaceState: - '''Get the initial state before we're notified of changes.''' - lookup_results = await self.ipr.link_lookup(ifname=self.interface) - if not lookup_results: - raise InterfaceNotFound(self.interface) - get_results = await self.ipr.link("get", index=lookup_results[0]) - return get_results[0].get('state') - - async def _watch_changes(self) -> None: - '''Updates `state` in real time forever.''' - while True: - # TODO: svinota says we should call .link('dump') here - # to empty the buffer before starting - async for msg in self.ipr.get(): - if msg.get('IFLA_IFNAME') == self.interface: - self.state = msg.get('state') - - async def __aenter__(self) -> 'InterfaceStateWatcher': - self._ipr = await AsyncIPRoute().__aenter__() - await self.ipr.bind(RTMGRP_LINK) - self.state = await self._fetch_current_state() - self._watcher = asyncio.create_task(self._watch_changes()) - return self - - async def __aexit__(self, *_) -> None: - self.ipr.close() - if self._watcher: - try: - await self._watcher - except NetlinkError as exc: - if exc.code == 104: - # we called close() so that is expected - pass - else: - raise - self._watcher = None - self._ipr = None - - -async def main(iface: str, state: str) -> None: - '''Very basic entrypoint for commandline testing.''' - assert state in ('up', 'down') - async with InterfaceStateWatcher(iface) as watcher: - print('Will exit when', iface, 'is', state) - await getattr(watcher, state).wait() - - -if __name__ == '__main__': - try: - asyncio.run(main(sys.argv[1], sys.argv[2])) - except IndexError: - print('Usage:', sys.argv[0], ' ') diff --git a/pyroute2/dhcp/leases.py b/pyroute2/dhcp/leases.py deleted file mode 100644 index 8fa9815cc..000000000 --- a/pyroute2/dhcp/leases.py +++ /dev/null @@ -1,247 +0,0 @@ -'''Lease classes used by the dhcp client.''' - -import abc -import json -import time -from dataclasses import asdict, dataclass, field -from logging import getLogger -from pathlib import Path -from secrets import SystemRandom -from typing import Generic, Optional, TypeVar - -from pyroute2.common import dqn2int -from pyroute2.dhcp.dhcp4msg import dhcp4msg -from pyroute2.dhcp.enums.dhcp import Option - -LOG = getLogger(__name__) - -random = SystemRandom() - - -def _now() -> float: - '''The current timestamp.''' - return time.time() - - -class MissingOptionError(LookupError): - '''Raised when trying to access a missing option in a lease.''' - - def __init__(self, opt: Option): - super().__init__(f"Lease does not set {opt!r}") - - -LeaseOptionT = TypeVar('LeaseOptionT') - - -class LeaseOption(Generic[LeaseOptionT]): - '''Descriptor to factorize properties on leases that read DHCP options.''' - - def __init__(self, opt: Option) -> None: - '''Create a new instance that looks up the given option.''' - self.opt = opt - - def __get__( - self, obj: object, objtype: Optional[type] = None - ) -> LeaseOptionT: - '''Gets the option matching self.opt in the lease.''' - assert isinstance(obj, Lease) - opt_name = self.opt.name.lower() - if opt_name not in obj.ack['options']: - raise MissingOptionError(self.opt) - opt_value = obj.ack['options'][opt_name] - return opt_value - - -@dataclass -class Lease(abc.ABC): - '''Represents a lease obtained through DHCP.''' - - # The DHCP ack sent by the server which allocated this lease - ack: dhcp4msg - # Name of the interface for which this lease was requested - interface: str - # MAC address of the server that allocated the lease - server_mac: str - # Timestamp of when this lease was obtained - obtained: float = field(default_factory=_now) - - def _seconds_til_timer(self, value: int) -> Optional[float]: - '''Number of seconds to wait until the given timer value expires. - - If the value is -1, that means it is infinite and None is returned. - ''' - if value == -1: - # infinite timer - return None - return self.obtained + value - _now() - - @property - def expired(self) -> bool: - '''Whether this lease has expired (its expiration is in the past). - - When loading a persisted lease, this won't be correct if the clock - has been adjusted since the lease was written. - However the worst case scenario is that we send a REQUEST for it, - get a NAK and restart from scratch. - ''' - return self.expiration_in is not None and self.expiration_in <= 0 - - @property - def expiration_in(self) -> Optional[float]: - '''The amount of seconds before the lease expires. - - Computed from the `lease_time` option. - - Can be negative if it's past due, None if infinite, - or raise MissingOptionError if the server did not give a lease time. - ''' - return self._seconds_til_timer(self.lease_time) - - @property - def renewal_in(self) -> Optional[float]: - '''The amount of seconds before we have to renew the lease. - - Computed from the `renewal_time` option, defaults to ~.5 * lease exp. - - Can be negative if it's past due, or None if infinite. - ''' - try: - return self._seconds_til_timer(self.renewal_time) - except MissingOptionError: - # RFC section 4.4.5 says we need a fuzzy value around 0.5 - if self.expiration_in: - return self.expiration_in * random.uniform(0.4, 0.6) - return None - - @property - def rebinding_in(self) -> Optional[float]: - '''The amount of seconds before we have to rebind the lease. - - Computed from the `rebinding_time` option, defaults to ~.8 * lease exp. - - Can be negative if it's past due, or None if infinite. - ''' - try: - return self._seconds_til_timer(self.rebinding_time) - except MissingOptionError: - if self.expiration_in: - return self.expiration_in * random.uniform(0.75, 0.90) - return None - - @property - def ip(self) -> str: - '''The IP address assigned to the client.''' - return self.ack['yiaddr'] - - @property - def prefixlen(self) -> int: - '''The length of the subnet mask assigned to the client.''' - return dqn2int(self.subnet_mask) - - @property - def default_gateway(self) -> str: - '''The default gateway for this interface. - - As mentioned by the RFC, the first router is the most prioritary. - ''' - # TODO: unit test to make this crash - return self.routers[0] - - # timers - lease_time = LeaseOption[int](Option.LEASE_TIME) - renewal_time = LeaseOption[int](Option.RENEWAL_TIME) - rebinding_time = LeaseOption[int](Option.REBINDING_TIME) - - # The subnet mask assigned to the client. - subnet_mask = LeaseOption[str](Option.SUBNET_MASK) - - routers = LeaseOption[list[str]](Option.ROUTER) - - # The broadcast address for this network. - broadcast_address = LeaseOption[str](Option.BROADCAST_ADDRESS) - - # The MTU for this interface. - mtu = LeaseOption[int](Option.INTERFACE_MTU) - - name_servers = LeaseOption[list[str]](Option.NAME_SERVER) - - # The IP address of the server which allocated this lease. - server_id = LeaseOption[str](Option.SERVER_ID) - - domain_name = LeaseOption[str](Option.DOMAIN_NAME) - - domain_search = LeaseOption[list[str]](Option.DOMAIN_SEARCH) - - @abc.abstractmethod - def dump(self) -> None: - '''Write a lease, i.e. to disk or to stdout.''' - - @classmethod - @abc.abstractmethod - def load(cls, interface: str) -> 'Optional[Lease]': - '''Load an existing lease for an interface, if it exists. - - The lease is not checked for freshness, and will be None if no lease - could be loaded. - ''' - - -class JSONStdoutLease(Lease): - '''Just prints the lease to stdout when the client gets a new one.''' - - def dump(self) -> None: - '''Writes the lease as json to stdout.''' - print(json.dumps(asdict(self), indent=2)) - - @classmethod - def load(cls, interface: str) -> None: - '''Does not do anything.''' - return None - - -class JSONFileLease(Lease): - '''Write and load the lease from a JSON file in the working directory.''' - - @classmethod - def _get_lease_dir(cls) -> Path: - '''Where to store the lease file, i.e. the working directory.''' - return Path.cwd() - - @classmethod - def _get_path(cls, interface: str) -> Path: - '''The lease file, named after the interface.''' - return ( - cls._get_lease_dir() - .joinpath(interface) - .with_name(f'{interface}.lease.json') - ) - - def dump(self) -> None: - '''Dump the lease to a file. - - The lease file is named after the interface - and written in the working directory. - ''' - lease_path = self._get_path(self.interface) - LOG.info('Writing lease for %s to %s', self.interface, lease_path) - with lease_path.open('wt') as lf: - json.dump(asdict(self), lf, indent=2) - - @classmethod - def load(cls, interface: str) -> 'Optional[JSONFileLease]': - '''Load the lease from a file. - - The lease file is named after the interface - and read from the working directory. - ''' - lease_path = cls._get_path(interface) - try: - with lease_path.open('rt') as lf: - LOG.info('Loading lease for %s from %s', interface, lease_path) - return cls(**json.load(lf)) - except FileNotFoundError: - LOG.info('No existing lease at %s for %s', lease_path, interface) - return None - except (TypeError, ValueError) as err: - LOG.warning('Error loading lease: %s', err) - return None diff --git a/pyroute2/dhcp/messages.py b/pyroute2/dhcp/messages.py deleted file mode 100644 index 7b91d4ee3..000000000 --- a/pyroute2/dhcp/messages.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Helper functions to build dhcp client messages.""" - -from dataclasses import dataclass -from typing import Iterable, Literal, Optional - -from pyroute2.dhcp import enums -from pyroute2.dhcp.dhcp4msg import dhcp4msg -from pyroute2.dhcp.fsm import State -from pyroute2.dhcp.leases import Lease -from pyroute2.dhcp.xids import Xid - -Parameters = Iterable[enums.dhcp.Option] - - -@dataclass -class _DHCPMessage: - '''A DHCP message with some extra info from other layers.''' - - dhcp: dhcp4msg - eth_src: str - eth_dst: str = 'ff:ff:ff:ff:ff:ff' - ip_src: str = '0.0.0.0' - ip_dst: str = '255.255.255.255' - sport: int = 68 - dport: int = 67 - - @property - def message_type(self) -> enums.dhcp.MessageType: - '''The DHCP message type (DISCOVER, REQUEST, ACK...)''' - return self.dhcp['options']['message_type'] - - @property - def xid(self) -> Xid: - return Xid(self.dhcp['xid']) - - -@dataclass -class SentDHCPMessage(_DHCPMessage): - '''A DHCP message to be sent to a server or broadcast.''' - - eth_src: Optional[str] = None # type: ignore[assignment] - - def __str__(self) -> str: - type_name = self.message_type.name - return ( - f'{type_name} to {self.eth_dst}/{self.ip_dst}:{self.dport} ' - f'(xid {self.xid})' - ) - - -class ReceivedDHCPMessage(_DHCPMessage): - '''A DHCP message received by the client.''' - - def __str__(self) -> str: - type_name = self.dhcp['options']['message_type'].name - return ( - f'{type_name} from {self.eth_src}/{self.ip_src}:{self.sport} ' - f'(xid {self.xid})' - ) - - -def discover(parameter_list: Parameters) -> SentDHCPMessage: - '''Make a broadcast DISCOVER message for the given parameters.''' - # Default for SentDHCPMessage is broadcast which is what we want here - return SentDHCPMessage( - dhcp=dhcp4msg( - { - 'op': enums.bootp.MessageType.BOOTREQUEST, - 'flags': enums.bootp.Flag.BROADCAST, - 'options': { - 'message_type': enums.dhcp.MessageType.DISCOVER, - 'parameter_list': list(parameter_list), - }, - } - ) - ) - - -def request_for_offer( - parameter_list: Parameters, offer: ReceivedDHCPMessage -) -> SentDHCPMessage: - '''Make a REQUEST message for a given OFFER. - - Since we don't have an IP yet, the message is always broadcast. - When requesting an offer in the Selecting state, the server_id DHCP option - is always set as opposed to when a REQUEST is sent in other states. - - See RFC 2131 section 4.3.2. - ''' - return SentDHCPMessage( - dhcp=dhcp4msg( - { - 'op': enums.bootp.MessageType.BOOTREQUEST, - 'flags': enums.bootp.Flag.BROADCAST, - 'options': { - 'message_type': enums.dhcp.MessageType.REQUEST, - 'requested_ip': offer.dhcp['yiaddr'], - 'server_id': offer.dhcp['options']['server_id'], - 'parameter_list': list(parameter_list), - }, - } - ) - ) - - -def request_for_lease( - parameter_list: Parameters, - lease: Lease, - state: Literal[State.RENEWING, State.REBINDING, State.REBOOTING], -) -> SentDHCPMessage: - '''Make a REQUEST for an existing lease. - - This differs from REQUESTs in response to an OFFER in that the server_id - option is never set. - - When rebooting, the message is broadcast, and the requested_ip option is - set to the IP in the stored lease. The bootp client IP is left blank. - - When renewing, (i.e. T1 expires) the message is for the server that granted - the lease. The lease's IP is expected to be assigned to the client's - interface at this point. - - When rebinding (T2), the message is broadcast on the network. - - In both cases, the bootp client IP (ciaddr) is set to the lease's IP. - - See RFC 2131 section 4.3.6. - ''' - dhcp_msg = dhcp4msg( - { - 'op': enums.bootp.MessageType.BOOTREQUEST, - 'flags': enums.bootp.Flag.BROADCAST, - 'options': { - 'message_type': enums.dhcp.MessageType.REQUEST, - 'parameter_list': list(parameter_list), - }, - } - ) - if state == State.REBOOTING: - dhcp_msg['options']['requested_ip'] = lease.ip - else: - dhcp_msg['ciaddr'] = lease.ip - if state == State.RENEWING: - # T1 timer expired, send a request directly to the known server - dhcp_msg['flags'] = enums.bootp.Flag.UNICAST - return SentDHCPMessage( - dhcp=dhcp_msg, - eth_dst=lease.server_mac, - # default ip_dst to broadcast, it should not be necessary - # but it'll avoid a crash if the server_id is missing - ip_dst=lease.server_id or '255.255.255.255', - ip_src=lease.ip, - ) - # Reboot or rebind, broadcast the request - return SentDHCPMessage(dhcp=dhcp_msg) - - -def release(lease: Lease) -> SentDHCPMessage: - '''Make a RELEASE for an existing & active lease.''' - # RELEASE messages have nearly no allowed options, - # and the released IP address must be set in ciaddr - return SentDHCPMessage( - dhcp=dhcp4msg( - { - 'op': enums.bootp.MessageType.BOOTREQUEST, - 'flags': enums.bootp.Flag.UNICAST, - 'ciaddr': lease.ip, - 'options': { - 'message_type': enums.dhcp.MessageType.RELEASE, - 'server_id': lease.server_id, - }, - } - ), - # RELEASEs are unicast (see rfc section 4.4.4) - eth_dst=lease.server_mac, - # default ip_dst to broadcast, it should not be necessary - # but it'll avoid a crash if the server_id is missing - ip_dst=lease.server_id or '255.255.255.255', - ip_src=lease.ip, - ) diff --git a/pyroute2/dhcp/server_detector.py b/pyroute2/dhcp/server_detector.py deleted file mode 100644 index 14565c5a6..000000000 --- a/pyroute2/dhcp/server_detector.py +++ /dev/null @@ -1,234 +0,0 @@ -import argparse -import asyncio -import dataclasses -import json -import logging -import time -from asyncio.exceptions import CancelledError, TimeoutError -from secrets import SystemRandom -from typing import AsyncGenerator, Iterable - -from pyroute2.dhcp import messages -from pyroute2.dhcp.dhcp4socket import AsyncDHCP4Socket -from pyroute2.dhcp.enums.dhcp import Option - -DHCPResponse = tuple[str, messages.ReceivedDHCPMessage] - -LOG = logging.getLogger('dhcp-server-detector') - -DEFAULT_PARAMETERS = ( - Option.SUBNET_MASK, - Option.ROUTER, - Option.BROADCAST_ADDRESS, - Option.NAME_SERVER, -) - - -class DHCPServerDetector: - '''Sends `DISCOVER`s on interfaces and listens for responses.''' - - def __init__( - self, - *interfaces: str, - duration: float = 25.0, - interval: float = 4.0, - requested_parameters: messages.Parameters = DEFAULT_PARAMETERS, - sport: int = 68, - ): - self.interfaces = interfaces - self.interval = interval - self.duration = duration - self.sport = sport - # The DISCOVERs that will be sent repeatedly per interface - self.discover_messages = self._make_discover_msgs( - interfaces=self.interfaces, params=requested_parameters - ) - # All received responses are put here by the dedicated tasks, - # along with the name of the interface they were received on. - self._responses_queue: asyncio.Queue[DHCPResponse] = asyncio.Queue() - - @classmethod - def _make_discover_msgs( - cls, interfaces: Iterable[str], params: messages.Parameters - ) -> dict[str, messages.SentDHCPMessage]: - '''Generate DISCOVERs with a different xid for each interface .''' - msgs: dict[str, messages.SentDHCPMessage] = {} - rand = SystemRandom().randint - for i in interfaces: - msgs[i] = messages.discover(parameter_list=params) - # use a different xid per interface - msgs[i].dhcp['xid'] = rand(0xFF, 0xFFFFFFFF) - return msgs - - async def _send_forever(self, sock: AsyncDHCP4Socket): - '''Send the `DISCOVER` message at `interval` until cancelled.''' - msg = self.discover_messages[sock.ifname] - while True: - LOG.info('[%s] -> %s', sock.ifname, msg) - await sock.put(msg) - await asyncio.sleep(self.interval) - - async def _get_offers(self, interface: str): - '''Send DISCOVERs and wait for responses on an interface.''' - expected_xids = { - ifname: msg.dhcp['xid'] - for ifname, msg in self.discover_messages.items() - } - async with AsyncDHCP4Socket(ifname=interface, port=self.sport) as sock: - send_task = asyncio.create_task( - self._send_forever(sock), name=f'Send DISCOVERS on {interface}' - ) - # if send_task returns, that means _send_forever crashed - while not send_task.done(): - try: - get_next_msg = asyncio.Task( - sock.get(), name=f'Get message from {interface}' - ) - # wait for a message and put it in the queue - _, pending = await asyncio.wait( - [send_task, get_next_msg], - return_when=asyncio.FIRST_COMPLETED, - ) - # if get is still pending, that means send_task is over - if get_next_msg in pending: - get_next_msg.cancel() - continue - # we have a new message - next_msg = await get_next_msg - if next_msg.dhcp['xid'] != expected_xids[sock.ifname]: - LOG.debug( - '[%s] Got %s with xid mismatch, ignoring', - interface, - next_msg.message_type.name, - ) - continue - LOG.info('[%s] <- %s', interface, next_msg) - await self._responses_queue.put((interface, next_msg)) - except asyncio.CancelledError: - LOG.debug('[%s] stop discovery', interface) - send_task.cancel() - break - else: - await send_task - - async def detect_servers(self) -> AsyncGenerator[DHCPResponse, None]: - '''Detect DHCP servers on `interfaces` for `duration`. - - Yields tuples of (interface name, response). - ''' - discover_tasks = [ - asyncio.create_task(self._get_offers(i), name=i) - for i in self.interfaces - ] - started = time.time() - remaining = self.duration - while remaining > 0 and discover_tasks: - get_response = asyncio.create_task(self._responses_queue.get()) - try: - done, _ = await asyncio.wait( - [get_response, *discover_tasks], - timeout=remaining, - return_when=asyncio.FIRST_COMPLETED, - ) - if get_response in done: - yield await get_response - done.remove(get_response) - else: - get_response.cancel() - for i in done: - if task_exc := i.exception(): - LOG.error("%r: %s", i.get_name(), task_exc) - discover_tasks.remove(i) - remaining -= time.time() - started - except (TimeoutError, CancelledError): - break - for i in discover_tasks: - i.cancel() - - -def get_argparser() -> argparse.ArgumentParser: - psr = argparse.ArgumentParser( - description='Send DHCP DISCOVER messages on the given interface(s) ' - 'and collect all responses. ' - 'Responses and their metadata are printed as JSON to stdout.', - epilog='Exits successfully if at least one response was received.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - psr.add_argument( - 'interface', nargs='+', help='Interface(s) to DISCOVER on.' - ) - psr.add_argument( - '-d', - '--duration', - type=float, - default=30.0, - help='Number of seconds spent collecting responses.', - metavar='SEC', - ) - psr.add_argument( - '-i', - '--interval', - type=float, - default=4.0, - help='Interval in seconds between DISCOVERs.', - metavar='SEC', - ) - psr.add_argument( - '-s', - '--source-port', - type=int, - default=68, - help='Source port to bind to.', - metavar='PORT', - ) - psr.add_argument( - '-1', - '--exit-on-first-offer', - action='store_true', - default=False, - help="Exit as soon as a response is received.", - ) - psr.add_argument( - '-l', - '--log-level', - default='WARNING', - help="Log level. Set to INFO to log sent & received messages.", - metavar='LEVEL', - ) - return psr - - -async def main() -> int: - '''Commandline entrypoint. Returns the number of received responses.''' - args = get_argparser().parse_args() - LOG.setLevel(args.log_level) - detector = DHCPServerDetector( - *args.interface, - interval=args.interval, - duration=args.duration, - sport=args.source_port, - ) - response_count: int = 0 - async for interface, msg in detector.detect_servers(): - response_count += 1 - print( - json.dumps( - {'interface': interface, 'message': dataclasses.asdict(msg)}, - indent=2, - ) - ) - if args.exit_on_first_offer: - break - return response_count - - -def run(): - logging.basicConfig( - format='%(asctime)s %(levelname)s %(name)s %(message)s' - ) - # Exit on failure if there were no received responses - exit(asyncio.run(main()) == 0) - - -if __name__ == '__main__': - run() diff --git a/pyroute2/dhcp/timers.py b/pyroute2/dhcp/timers.py deleted file mode 100644 index f409f4299..000000000 --- a/pyroute2/dhcp/timers.py +++ /dev/null @@ -1,73 +0,0 @@ -'''Timers to manage lease rebinding, renewal & expiration.''' - -import asyncio -import dataclasses -from logging import getLogger -from typing import Optional - -from pyroute2.dhcp.fsm import AsyncCallback -from pyroute2.dhcp.leases import Lease - -LOG = getLogger(__name__) - - -@dataclasses.dataclass -class LeaseTimers: - '''Manage callbacks associated with DHCP leases.''' - - renewal: Optional[asyncio.TimerHandle] = None - rebinding: Optional[asyncio.TimerHandle] = None - expiration: Optional[asyncio.TimerHandle] = None - - def cancel(self) -> None: - '''Cancel all current timers.''' - for timer_name in ('renewal', 'rebinding', 'expiration'): - self._reset_timer(timer_name) - - def _reset_timer(self, timer_name: str) -> None: - '''Cancel a timer and set it to None.''' - if timer_name not in ('renewal', 'rebinding', 'expiration'): - raise ValueError(f'Unknown timer {timer_name!r}') - if timer := getattr(self, timer_name): - assert isinstance(timer, asyncio.TimerHandle) - if not timer.cancelled(): - # FIXME: how do we know a timer wasn't cancelled ? - # this causes spurious logs - LOG.debug('Canceling %s timer', timer_name) - timer.cancel() - setattr(self, timer_name, None) - - def arm(self, lease: Lease, **callbacks: AsyncCallback) -> None: - '''Reset & arm timers from a `Lease`. - - `callbacks` must be async callables with no arguments - that will be called when the associated timer expires. - ''' - self.cancel() - loop = asyncio.get_running_loop() - - for timer_name, async_callback in callbacks.items(): - self._reset_timer(timer_name) - lease_time = getattr(lease, f'{timer_name}_in') - if not lease_time: - LOG.debug('%s time is infinite', timer_name) - continue - if lease_time < 0.0: - LOG.debug('%s is in the past', timer_name) - continue - LOG.info('Scheduling %s in %.2fs', timer_name, lease_time) - timer = loop.call_later( - lease_time, self._create_timer_task, timer_name, async_callback - ) - setattr(self, timer_name, timer) - - def _create_timer_task( - self, timer_name: str, async_callback: AsyncCallback - ) -> None: - ''' 'Internal callback for loop.call_later. - - Creates a Task that runs the async_callback. - ''' - asyncio.create_task( - async_callback(), name=f"{timer_name} timer callback" - ) diff --git a/pyroute2/dhcp/xids.py b/pyroute2/dhcp/xids.py deleted file mode 100644 index ce70def16..000000000 --- a/pyroute2/dhcp/xids.py +++ /dev/null @@ -1,85 +0,0 @@ -from secrets import SystemRandom -from typing import Any, Optional, Union - -from pyroute2.dhcp.fsm import State - -random = SystemRandom() - - -def random_xid_prefix() -> int: - '''A (max) 32 bit random int with its lowest nibble set to 0. - - These last 4 bits are used by the client to store its state, in the form - of the associated `State` value, when sending a DHCP message. - ''' - return random.randint(0x00000010, 0xFFFFFFF0) - - -class Xid: - '''Transaction IDs used to identify responses to DHCP requests. - - We use the last nibble to store the state the message was sent in. - - The most significant bits store a random value we use to match requests - and responses, since the RFC specifies that the server must send back - the same value when answering. (see RFC section 4.1) - ''' - - def __init__(self, value: Optional[Union[int, 'Xid']] = None): - if isinstance(value, int): - int_value = value - elif isinstance(value, Xid): - int_value = int(value) - elif value is None: - int_value = random_xid_prefix() - else: - raise TypeError(f'{value!r} is not an xid') - assert int_value < 0xFFFFFFFF # we have 32 bits - self._value = int_value - - @property - def random_part(self) -> int: - '''The random part of the transaction id.''' - return self._value & 0xFFFFFFF0 - - @property - def request_state(self) -> Optional[State]: - '''The state in which the request was sent. - - Since servers answer with the same transaction ID as the request, - we can use this to know what client state does a response answer to. - ''' - try: - return State(self._value & 0xF) - except ValueError: - return None - - def for_state(self, state: State) -> 'Xid': - '''A new Xid built from the random part + the state.''' - return Xid(self.random_part | state) - - def __index__(self) -> int: - '''Allows xids to be used as int.''' - return self._value - - def matches(self, received_xid: 'Xid'): - '''Loose match, whether the random part of both XIDs match. - - This can be used to check if a message is indeed in response - to a request we sent. - ''' - return self.random_part == received_xid.random_part - - def __eq__(self, other: Any) -> bool: - '''Xids compare to other xids or ints.''' - if isinstance(other, Xid): - return self._value == other._value - elif isinstance(other, int): - return self._value == other - return False - - def __str__(self) -> str: - return hex(self._value) - - def __repr__(self) -> str: - return f"Xid({self})" diff --git a/pyroute2/ethtool/ethtool.py b/pyroute2/ethtool/ethtool.py index a9d707c20..22955af57 100644 --- a/pyroute2/ethtool/ethtool.py +++ b/pyroute2/ethtool/ethtool.py @@ -71,31 +71,6 @@ def to_ioctl(ioctl_features, eth_features): ioctl_features[feature.name] = feature.enable -class EthtoolChannels( - namedtuple( - 'EthtoolChannels', - ( - "max_rx", - "max_tx", - "max_other", - "max_combined", - "rx_count", - "tx_count", - "other_count", - "combined_count", - ), - ) -): - @classmethod - def from_ioctl(cls, channels): - return cls(**{k: getattr(channels, k) for k in cls._fields}) - - @staticmethod - def to_ioctl(ioctl_channels, eth_channel): - for key, val in eth_channel.items(): - setattr(ioctl_channels, key, val) - - class EthtoolWakeOnLan(namedtuple('EthtoolWolMode', ('modes', 'sopass'))): @classmethod def from_netlink(cls, nl_wol): @@ -490,16 +465,6 @@ def set_features(self, ifname, features): EthtoolFeatures.to_ioctl(ioctl_features, features) self._with_ioctl.set_features(ioctl_features) - def get_channels(self, ifname): - self._with_ioctl.change_ifname(ifname) - return EthtoolChannels.from_ioctl(self._with_ioctl.get_channels()) - - def set_channels(self, ifname, channels): - self._with_ioctl.change_ifname(ifname) - ioctl_channels = self._with_ioctl.get_channels() - EthtoolChannels.to_ioctl(ioctl_channels, channels) - self._with_ioctl.set_channels(ioctl_channels) - def get_coalesce(self, ifname): self._with_ioctl.change_ifname(ifname) return EthtoolCoalesce.from_ioctl(self._with_ioctl.get_coalesce()) diff --git a/pyroute2/ethtool/ioctl.py b/pyroute2/ethtool/ioctl.py index 0bf8f54e9..72be86f7d 100644 --- a/pyroute2/ethtool/ioctl.py +++ b/pyroute2/ethtool/ioctl.py @@ -17,8 +17,6 @@ ETHTOOL_GFLAGS = 0x00000025 ETHTOOL_GFEATURES = 0x0000003A ETHTOOL_SFEATURES = 0x0000003B -ETHTOOL_GCHANNELS = 0x0000003C -ETHTOOL_SCHANNELS = 0x0000003D ETHTOOL_GLINKSETTINGS = 0x0000004C ETHTOOL_GSTRINGS = 0x0000001B @@ -147,7 +145,6 @@ class EthtoolWolInfo(DictStruct): class EthtoolCmd(DictStruct): - _layout_ = 'ms' _pack_ = 1 _fields_ = [ ("cmd", ctypes.c_uint32), @@ -171,7 +168,6 @@ class EthtoolCmd(DictStruct): class IoctlEthtoolLinkSettings(DictStruct): - _layout_ = 'ms' _pack_ = 1 _fields_ = [ ("cmd", ctypes.c_uint32), @@ -195,7 +191,6 @@ class IoctlEthtoolLinkSettings(DictStruct): class EthtoolCoalesce(DictStruct): - _layout_ = 'ms' _pack_ = 1 _fields_ = [ # ETHTOOL_{G,S}COALESCE @@ -275,7 +270,6 @@ class EthtoolValue(ctypes.Structure): class EthtoolSsetInfo(ctypes.Structure): - _layout_ = 'ms' _pack_ = 1 _fields_ = [ ("cmd", ctypes.c_uint32), @@ -345,28 +339,11 @@ class EthtoolSfeatures(ctypes.Structure): ] -class EthtoolChannels(DictStruct): - _layout_ = 'ms' - _pack_ = 1 - _fields_ = [ - ("cmd", ctypes.c_uint32), - ("max_rx", ctypes.c_uint32), - ("max_tx", ctypes.c_uint32), - ("max_other", ctypes.c_uint32), - ("max_combined", ctypes.c_uint32), - ("rx_count", ctypes.c_uint32), - ("tx_count", ctypes.c_uint32), - ("other_count", ctypes.c_uint32), - ("combined_count", ctypes.c_uint32), - ] - - class FeatureState(ctypes.Structure): _fields_ = [("off_flags", ctypes.c_uint32), ("features", EthtoolGfeatures)] class EthtoolRingParam(DictStruct): - _layout_ = 'ms' _pack_ = 1 _fields_ = [ ("cmd", ctypes.c_uint32), @@ -392,7 +369,6 @@ class IfReqData(ctypes.Union): ("gstats", ctypes.POINTER(None)), ("gfeatures", ctypes.POINTER(EthtoolGfeatures)), ("sfeatures", ctypes.POINTER(EthtoolSfeatures)), - ("channels", ctypes.POINTER(EthtoolChannels)), ("glinksettings", ctypes.POINTER(IoctlEthtoolLinkSettings)), ("wolinfo", ctypes.POINTER(EthtoolWolInfo)), ("rings", ctypes.POINTER(EthtoolRingParam)), @@ -400,14 +376,12 @@ class IfReqData(ctypes.Union): class IfReq(ctypes.Structure): - _layout_ = 'ms' _pack_ = 1 _anonymous_ = ("u",) _fields_ = [("ifr_name", ctypes.c_uint8 * IFNAMSIZ), ("u", IfReqData)] class IfReqSsetInfo(ctypes.Structure): - _layout_ = 'ms' _pack_ = 1 _fields_ = [ ("ifr_name", ctypes.c_uint8 * IFNAMSIZ), @@ -532,7 +506,7 @@ def get_statistics(self): cmd=ETHTOOL_GSTATS ) self.ifreq.gstats = ctypes.cast( - ctypes.byref(gstats), ctypes.POINTER(None) + ctypes.pointer(gstats), ctypes.POINTER(None) ) self.ioctl() assert len(self.stat_names) == len(gstats.data) @@ -546,6 +520,7 @@ def get_stringset_length(self, set_id): ifreq_sset.ifr_name = (ctypes.c_uint8 * IFNAMSIZ)(*self.ifname) ifreq_sset.info = ctypes.pointer(sset_info) fcntl.ioctl(self.sock, SIOCETHTOOL, ifreq_sset) + assert sset_info.sset_mask return sset_info.data def get_stringset( @@ -559,7 +534,7 @@ def get_stringset( cmd=ETHTOOL_GSTRINGS, string_set=set_id, len=gstrings_length ) self.ifreq.gstrings = ctypes.cast( - ctypes.byref(gstrings), ctypes.POINTER(None) + ctypes.pointer(gstrings), ctypes.POINTER(None) ) self.ioctl() @@ -587,17 +562,6 @@ def set_features(self, features): self.ifreq.sfeatures = ctypes.pointer(features._cmd_set) return self.ioctl() - def get_channels(self): - cmd = EthtoolChannels(cmd=ETHTOOL_GCHANNELS) - self.ifreq.channels = ctypes.pointer(cmd) - self.ioctl() - return cmd - - def set_channels(self, channels): - channels.cmd = ETHTOOL_SCHANNELS - self.ifreq.channels = ctypes.pointer(channels) - return self.ioctl() - def get_cmd(self): cmd = EthtoolCmd(cmd=ETHTOOL_GSET) self.ifreq.ifr_data = ctypes.pointer(cmd) diff --git a/pyroute2/ext/bpf.py b/pyroute2/ext/bpf.py deleted file mode 100644 index c98465610..000000000 --- a/pyroute2/ext/bpf.py +++ /dev/null @@ -1,89 +0,0 @@ -from ctypes import ( - Structure, - addressof, - c_int, - c_ubyte, - c_ushort, - c_void_p, - sizeof, - string_at, -) -from enum import IntEnum - - -class sock_filter(Structure): - _fields_ = [ - ('code', c_ushort), # u16 - ('jt', c_ubyte), # u8 - ('jf', c_ubyte), # u8 - ('k', c_int), # can be signed or unsigned - ] - - -class sock_fprog(Structure): - _fields_ = [('len', c_ushort), ('filter', c_void_p)] - - -def compile(code: list[list[int]]): - ProgramType = sock_filter * len(code) - program = ProgramType(*[sock_filter(*line) for line in code]) - sfp = sock_fprog(len(code), addressof(program[0])) - return string_at(addressof(sfp), sizeof(sfp)), program - - -class BPF(IntEnum): - '''BPF constants. - - See: - - https://www.kernel.org/doc/Documentation/networking/filter.txt - - https://github.com/iovisor/bpf-docs/blob/master/eBPF.md - ''' - - # Operations - LD = 0x00 # Load - LDX = 0x01 # Load Index - ST = 0x02 # Store - STX = 0x03 # Store Index - ALU = 0x04 # Arithmetic Logic Unit - JMP = 0x05 # Jump - RET = 0x06 # Return - MISC = 0x07 # Miscellaneous - - # Sizes - W = 0x00 # Word (4 bytes) - H = 0x08 # Half-word (2 bytes) - B = 0x10 # Byte (1 byte) - - # Offsets - IMM = 0x00 # Immediate - ABS = 0x20 # Absolute - IND = 0x40 # Indirect - MEM = 0x60 # Memory - LEN = 0x80 # Packet Length - MSH = 0xA0 # Masked - - # gotos - JEQ = 0x10 # Jump if Equal - JGT = 0x20 # Jump if Greater - JGE = 0x30 # Jump if Greater or Equal - JSET = 0x40 # Jump if Bit is Set - - # Sources - K = 0x00 # Constant - X = 0x08 # Register - - # Operators - ADD = 0x00 - SUB = 0x10 - MUL = 0x20 - DIV = 0x30 - OR = 0x40 - AND = 0x50 - LSH = 0x60 - RSH = 0x70 - NEG = 0x80 - MOD = 0x90 - XOR = 0xA0 - MOV = 0xB0 - ARSH = 0xC0 - END = 0xD0 diff --git a/pyroute2/ext/rawsocket.py b/pyroute2/ext/rawsocket.py index f06215c26..902c24147 100644 --- a/pyroute2/ext/rawsocket.py +++ b/pyroute2/ext/rawsocket.py @@ -1,14 +1,18 @@ -import asyncio -import logging +import struct +from ctypes import ( + Structure, + addressof, + c_ubyte, + c_uint, + c_ushort, + c_void_p, + sizeof, + string_at, +) from socket import AF_PACKET, SOCK_RAW, SOL_SOCKET, errno, error, htons, socket -from typing import Optional -from pyroute2.ext import bpf -from pyroute2.iproute.linux import AsyncIPRoute -from pyroute2.netlink.rtnl import RTMGRP_LINK +from pyroute2.iproute.linux import IPRoute -LOG = logging.getLogger(__name__) -ETH_P_IP = 0x0800 ETH_P_ALL = 3 SO_ATTACH_FILTER = 26 SO_DETACH_FILTER = 27 @@ -17,7 +21,27 @@ total_filter = [[0x06, 0, 0, 0]] -class AsyncRawSocket(socket): +class sock_filter(Structure): + _fields_ = [ + ('code', c_ushort), # u16 + ('jt', c_ubyte), # u8 + ('jf', c_ubyte), # u8 + ('k', c_uint), + ] # u32 + + +class sock_fprog(Structure): + _fields_ = [('len', c_ushort), ('filter', c_void_p)] + + +def compile_bpf(code): + ProgramType = sock_filter * len(code) + program = ProgramType(*[sock_filter(*line) for line in code]) + sfp = sock_fprog(len(code), addressof(program[0])) + return string_at(addressof(sfp), sizeof(sfp)), program + + +class RawSocket(socket): ''' This raw socket binds to an interface and optionally installs a BPF filter. @@ -31,68 +55,28 @@ class AsyncRawSocket(socket): fprog = None - def __init__(self, ifname: str, bpf: Optional[list[list[int]]] = None): + def __init__(self, ifname, bpf=None): self.ifname = ifname - self.bpf = bpf - # start watching for mac addr changes - self._l2addr_watcher: Optional[asyncio.Task] = None - - async def __aexit__(self, *_): - self._l2addr_watcher.cancel() - self.close() - - async def __aenter__(self): # lookup the interface details - async with AsyncIPRoute() as ip: - async for link in await ip.get_links(): - if link.get_attr('IFLA_IFNAME') == self.ifname: + with IPRoute() as ip: + for link in ip.get_links(): + if link.get_attr('IFLA_IFNAME') == ifname: break else: raise IOError(2, 'Link not found') - self.l2addr: str = link.get_attr('IFLA_ADDRESS') - self.ifindex: int = link['index'] + self.l2addr = link.get_attr('IFLA_ADDRESS') + self.ifindex = link['index'] # bring up the socket socket.__init__(self, AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)) - socket.setblocking(self, False) socket.bind(self, (self.ifname, ETH_P_ALL)) - if self.bpf: + if bpf: self.clear_buffer() - fstring, self.fprog = bpf.compile(self.bpf) + fstring, self.fprog = compile_bpf(bpf) socket.setsockopt(self, SOL_SOCKET, SO_ATTACH_FILTER, fstring) else: - # FIXME: should be async self.clear_buffer(remove_total_filter=True) - # change self.l2addr if it changes - self._l2addr_watcher = asyncio.create_task( - self._watch_l2addr_changes(), - name=f'Watch {self.ifname} for l2addr changes', - ) - return self - - async def _watch_l2addr_changes(self): - '''Updates self.l2addr when the interfaces's mac changes. - - During the lifetime of the socket, the interface's mac can change, and - since it's read it at startup & used to build packets, they will then - have the wrong mac. - ''' - async with AsyncIPRoute() as ipr: - await ipr.bind(RTMGRP_LINK) - while True: - async for msg in ipr.get(): - if msg.get('IFLA_IFNAME') != self.ifname: - continue - new_l2addr = msg.get_attr('IFLA_ADDRESS') - if new_l2addr and new_l2addr != self.l2addr: - LOG.info( - 'l2addr for %s changed from %s to %s', - self.ifname, - self.l2addr, - new_l2addr, - ) - self.l2addr = new_l2addr - - def clear_buffer(self, remove_total_filter: bool = False): + + def clear_buffer(self, remove_total_filter=False): # there is a window of time after the socket has been created and # before bind/attaching a filter where packets can be queued onto the # socket buffer @@ -100,8 +84,9 @@ def clear_buffer(self, remove_total_filter: bool = False): # pcap-linux.c. libpcap sets a total filter which does not match any # packet. It then clears what is already in the socket # before setting the desired filter - total_fstring, prog = bpf.compile(total_filter) + total_fstring, prog = compile_bpf(total_filter) socket.setsockopt(self, SOL_SOCKET, SO_ATTACH_FILTER, total_fstring) + self.setblocking(0) while True: try: self.recvfrom(0) @@ -114,22 +99,22 @@ def clear_buffer(self, remove_total_filter: bool = False): break else: raise + self.setblocking(1) if remove_total_filter: # total_fstring ignored socket.setsockopt( self, SOL_SOCKET, SO_DETACH_FILTER, total_fstring ) - @staticmethod - def csum(data: bytes) -> int: - '''Compute the "Internet checksum" for the given bytes.''' + def csum(self, data): if len(data) % 2: data += b'\x00' - csum: int = 0 - # pretty much the fastest way to compute this in Python - for i in range(len(data) // 2): - offset = i * 2 - csum += (data[offset] << 8) + data[offset + 1] + csum = sum( + [ + struct.unpack('>H', data[x * 2 : x * 2 + 2])[0] + for x in range(len(data) // 2) + ] + ) csum = (csum >> 16) + (csum & 0xFFFF) csum += csum >> 16 return ~csum & 0xFFFF diff --git a/pyroute2/fixtures/__init__.py b/pyroute2/fixtures/__init__.py deleted file mode 100644 index 8b7d79e0d..000000000 --- a/pyroute2/fixtures/__init__.py +++ /dev/null @@ -1,132 +0,0 @@ -''' -.. - 8<----------------------------------------------------------- - Doctest setup. - - It is not rendered into html. For the docs, skip to tne - CI test fixtures section below. - -.. testsetup:: * - - from pyroute2.fixtures.doctest import * - -.. - End of the doctest setup. - 8<----------------------------------------------------------- - - -CI test fixtures ----------------- - -Added in version 0.9.1. - -The library provides a set of fixtures that can be used with pytest -to setup a simple test environment for functional network tests. - -Usage: - -.. testcode:: - - # file: conftest.py - pytest_plugins = [ - 'pyroute2.fixtures.iproute', - 'pyroute2.fixtures.ndb', - 'pyroute2.fixtures.plan9' - ] - - # file: my_test.py - def test_my_code(sync_context): - # here you have access to - # - sync_context.ipr # IPRoute instance running in a netns - sync_context.netns # ready to use netns with a test link up - sync_context.test_link # test link in the netns - sync_context.test_link.index # interface index - sync_context.test_link.ifname # interface name - sync_context.test_link.address # MAC address - -The fixtures set up a network namespace with a unique name, a dummy -interface within the namespace, and bring the interface up. They form -a tree of dependencies, so if you use e.g. `test_link_ifname` fixture, -you may be sure that the namespace and the interface are already set -up properly. - -Fixtures dependencies diagram: - -.. aafig:: - :scale: 80 - :textual: - - +---------------------+ - | `test_link` |--+ - +---------------------+ | - ^ | v - +---------------------+ | | +----------------------+ - | `test_link_index` |__+ | | `test_link_ifinfmsg` | - | | | | | | - +---------------------+ | | +----------------------+ - | | | - +---------------------+ | | v - | `test_link_address` |__+ | +----------------------+ - | | | +->| netns | - +---------------------+ | | | | - | | +----------------------+ - +---------------------+ | | - | `test_link_ifname` |__+ | - | | | | - +---------------------+ | | - | | - +---------------------+ | | - | `async_context` |__+ | - | |_ | ___ | - +---------------------+ | | | - | | | - +---------------------+ | | | - | `sync_context` |__| | | - | |_____ | | - +---------------------+ | | | - | | | - +---------------------+ | | | - | `sync_ipr` |<----+ | | - | |______ | _____+ - +---------------------+ | | - | | - +---------------------+ | | - | `async_ipr` |<------+ | - | |______________+ - +---------------------+ | - | - +---------------------+ | - | `ndb` |______________| - | | - +---------------------+ - -.. autofunction:: pyroute2.fixtures.iproute.nsname - -.. autofunction:: pyroute2.fixtures.iproute.test_link_ifinfmsg - -.. autofunction:: pyroute2.fixtures.iproute.test_link - -.. autofunction:: pyroute2.fixtures.iproute.test_link_address - -.. autofunction:: pyroute2.fixtures.iproute.test_link_index - -.. autofunction:: pyroute2.fixtures.iproute.test_link_ifname - -.. autofunction:: pyroute2.fixtures.iproute.async_ipr - -.. autofunction:: pyroute2.fixtures.iproute.sync_ipr - -.. autofunction:: pyroute2.fixtures.iproute.async_context - -.. autofunction:: pyroute2.fixtures.iproute.sync_context - -.. autofunction:: pyroute2.fixtures.ndb.ndb - -.. autoclass:: pyroute2.fixtures.iproute.TestInterface - :members: - -.. autoclass:: pyroute2.fixtures.iproute.TestContext - :members: - -''' diff --git a/pyroute2/fixtures/doctest.py b/pyroute2/fixtures/doctest.py deleted file mode 100644 index a8ebfecb5..000000000 --- a/pyroute2/fixtures/doctest.py +++ /dev/null @@ -1,24 +0,0 @@ -''' -This module only prepares the environment for doctests -in `pyroute2.fixtures`. -''' - -from unittest.mock import MagicMock - -from pyroute2 import config -from pyroute2.iproute.linux import AsyncIPRoute, IPRoute -from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg - -# config setup -config.mock_netlink = True -config.mock_netns = True - -# mock modules -subprocess = MagicMock() -pytest = MagicMock() - -# mock fixtures -nsname = MagicMock() -test_link_ifinfmsg = ifinfmsg() -async_ipr = AsyncIPRoute() -sync_ipr = IPRoute() diff --git a/pyroute2/fixtures/iproute/__init__.py b/pyroute2/fixtures/iproute/__init__.py deleted file mode 100644 index d2af2e9fa..000000000 --- a/pyroute2/fixtures/iproute/__init__.py +++ /dev/null @@ -1,402 +0,0 @@ -import errno -from collections.abc import AsyncGenerator, Generator -from typing import Generic, TypeVar - -import pytest -import pytest_asyncio - -from pyroute2 import netns -from pyroute2.common import uifname -from pyroute2.iproute.linux import AsyncIPRoute, IPRoute -from pyroute2.netlink.exceptions import NetlinkError -from pyroute2.netlink.rtnl import IFNAMSIZ -from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg - -T = TypeVar('T', AsyncIPRoute, IPRoute) - - -class TestInterface: - '''Test interface spec. - - Provided by `test_link` fixture. - - Provides shortcuts to some important interface properties, - like `TestInterface.index` or `TestInterface.netns`. - ''' - - def __init__(self, index: int, ifname: str, address: str, nsname: str): - if index <= 1: - raise TypeError('test interface index must be > 1') - if not 1 < len(ifname) <= IFNAMSIZ: - raise TypeError( - 'test interface ifname length must be from 2 to IFNAMSIZ' - ) - self._index = index - self._ifname = ifname - self._address = address - self._netns = nsname - - @property - def index(self) -> int: - '''Test interface index. - - Is always greater than 1, as index 1 has the the loopback interface. - ''' - return self._index - - @property - def ifname(self) -> str: - '''Test interface ifname. - - The name length must be greater than 1 and less or equal IFNAMSIZ. - ''' - return self._ifname - - @property - def address(self) -> str: - '''Test interface MAC address. - - In the form `xx:xx:xx:xx:xx:xx`.''' - return self._address - - @property - def netns(self) -> str: - '''Test interface netns. - - A string name of the network namespace.''' - return self._netns - - -class TestContext(Generic[T]): - '''The test context. - - Provided by `async_context` and `sync_context` fixtures. - - Provides convenient shortcuts to RTNL API, the network namespace - name and the test interface spec. - ''' - - __test__ = False # prevent pytest from looking for tests here - - def __init__(self, ipr: T, test_link: TestInterface): - self._ipr: T = ipr - self._test_link = test_link - - @property - def ipr(self) -> T: - '''RTNL API. - - Return RTNL API instance, either IPRoute, or AsyncIPRoute.''' - return self._ipr - - @property - def test_link(self) -> TestInterface: - '''Test interface spec. - - Return `TestInterface` object for the test interface.''' - return self._test_link - - @property - def netns(self) -> str: - '''Network namespace. - - A string name of the network namespace.''' - return self.ipr.status['netns'] - - -class SetNSContext: - '''A unique network namespace context. - - Provided by `setns_context` fixture. - - Sets a unique netns for the whole python process for the - fixture scope, and returns from the nets on the scope exit.''' - - def __init__(self, nsname: str): - self._netns = nsname - - def __enter__(self): - netns.pushns(self.netns) - - def __exit__(self, *_): - netns.popns() - - @property - def netns(self) -> str: - '''Network namespace. - - A string name of the network namespace.''' - return self._netns - - -@pytest.fixture -def nsname() -> Generator[str]: - '''Network namespace. - - * **Name**: nsname - * **Scope**: function - - Create a unique network namespace and yield its name. Remove - the netns on cleanup. - - It's safe to create and modify interfaces, addresses, routes etc. - in the test network namespace, as it is disconnected from the main - system, and the test cleanup will remove the namespace with all - its content. - - Example usage: - - .. testcode:: nsname - - def test_list_interfaces(nsname): - subprocess.Popen( - ['ip', 'netns', 'exec', nsname, 'ip', 'link'], - stdout=subprocess.PIPE, - ) - # ... - ''' - nsname = uifname() - netns.create(nsname) - with IPRoute(netns=nsname) as ipr: - ipr.link('set', index=1, state='up') - ipr.poll(ipr.addr, 'dump', address='127.0.0.1', timeout=5) - yield nsname - try: - netns.remove(nsname) - except OSError: - pass - - -@pytest.fixture -def test_link_ifinfmsg(nsname: str) -> Generator[ifinfmsg]: - '''Test interface ifinfmsg. - - * **Name**: test_link_ifinfmsg - * **Scope**: function - * **Depends**: nsname - - Create a test interface in the test netns and yield ifinfmsg. Remove - the interface on cleanup. - - This fixture depends on **nsname**, and it means that the network - namespace will be created automatically if you use this fixture. - - Example usage: - - .. testcode:: test_link_ifinfmsg - - def test_check_interface(nsname, test_link_ifinfmsg): - link = test_link_ifinfmsg - ns = ['ip', 'netns', 'exec', nsname] - up = ['ip', 'link', 'set', 'dev', link.get('ifname'), 'up'] - subprocess.Popen(ns + up) - # ... - - ''' - ifname = uifname() - with IPRoute(netns=nsname) as ipr: - ipr.link('add', ifname=ifname, kind='dummy', state='up') - (link,) = ipr.poll(ipr.link, 'dump', ifname=ifname, timeout=5) - yield link - try: - ipr.link('del', index=link.get('index')) - except NetlinkError as e: - if e.code != errno.ENODEV: - raise - - -@pytest.fixture -def test_link( - nsname: str, test_link_ifinfmsg: ifinfmsg -) -> Generator[TestInterface]: - '''Test interface spec. - - * **Name**: test_link - * **Scope**: function - * **Depends**: nsname, test_link_ifinfmsg - - Yield `TestInterface` object for the test interface, providing - a convenient way to access some important interface properties. - ''' - yield TestInterface( - index=test_link_ifinfmsg.get('index'), - ifname=test_link_ifinfmsg.get('ifname'), - address=test_link_ifinfmsg.get('address'), - nsname=nsname, - ) - - -@pytest.fixture -def test_link_address(test_link: TestInterface) -> Generator[str]: - '''Test interface MAC address. - - * **Name**: test_link_address - * **Scope**: function - * **Depends**: test_link - - Yield test interface MAC address. The network namespace and - the test interface exist at this point. - ''' - yield test_link.address - - -@pytest.fixture -def test_link_index(test_link: TestInterface) -> Generator[int]: - '''Test interface MAC index. - - * **Name**: test_link_index - * **Scope**: function - * **Depends**: test_link - - Yield test interface index. The network namespace and - the test interface exist at this point. - ''' - yield test_link.index - - -@pytest.fixture -def test_link_ifname(test_link: TestInterface) -> Generator[str]: - '''Test interface MAC ifname. - - * **Name**: test_link_ifname - * **Scope**: function - * **Depends**: test_link - - Yield test interface ifname. The network namespace and - the test interface exist at this point. - ''' - yield test_link.ifname - - -@pytest.fixture -def tmp_link_ifname(nsname: str) -> Generator[str]: - '''Temporary link name. - - * **Name**: tmp_link_ifname - * **Scope**: function - * **Depends**: nsname - - Yield tmp link ifname, but don't create it. Try to remove - the link on cleanup. - ''' - ifname = uifname() - with IPRoute(netns=nsname) as ipr: - yield ifname - try: - (link,) = ipr.link('get', ifname=ifname) - ipr.link('del', index=link.get('index')) - except NetlinkError as e: - if e.code != errno.ENODEV: - raise - - -@pytest_asyncio.fixture -async def async_ipr(request, nsname: str) -> AsyncGenerator[AsyncIPRoute]: - '''`AsyncIPRoute` instance. - - * **Name**: async_ipr - * **Scope**: function - * **Depends**: nsname - - Yield `AsyncIPRoute` instance, running within the test network namespace. - You can provide additional keyword arguments to `AsyncIPRoute`: - - .. testcode:: async_ipr - - @pytest.mark.parametrize( - 'async_ipr', - [ - { - 'ext_ack': True, - 'strict_check': True, - }, - ], - indirect=True - ) - @pytest.mark.asyncio - async def test_my_case(async_ipr): - await async_ipr.link('set', index=1, state='up') - - ''' - kwarg = getattr(request, 'param', {}) - async with AsyncIPRoute(netns=nsname, **kwarg) as ipr: - yield ipr - - -@pytest.fixture -def sync_ipr(request, nsname: str) -> Generator[IPRoute]: - '''`IPRoute` instance. - - * **Name**: sync_ipr - * **Scope**: function - * **Depends**: nsname - - Yield `IPRoute` instance, running within the test network namespace. - You can provide additional keyword arguments to `IPRoute`: - - .. testcode:: sync_ipr - - @pytest.mark.parametrize( - 'sync_ipr', - [ - { - 'ext_ack': True, - 'strict_check': True, - }, - ], - indirect=True - ) - def test_my_case(sync_ipr): - sync_ipr.link('set', index=1, state='up') - ''' - kwarg = getattr(request, 'param', {}) - with IPRoute(netns=nsname, **kwarg) as ipr: - yield ipr - - -@pytest_asyncio.fixture -async def async_context( - async_ipr: AsyncIPRoute, test_link: TestInterface -) -> AsyncGenerator[TestContext[AsyncIPRoute]]: - '''Asynchronous TestContext. - - * **Name**: async_context - * **Scope**: function - * **Depends**: async_ipr, test_link - - Yield `TestContext` with `AsyncIPRoute`. - ''' - yield TestContext[AsyncIPRoute](async_ipr, test_link) - - -@pytest.fixture -def sync_context( - sync_ipr: IPRoute, test_link: TestInterface -) -> Generator[TestContext[IPRoute]]: - '''Synchronous TestContext. - - * **Name**: sync_context - * **Scope**: function - * **Depends**: sync_ipr, test_link - - Yield `TestContext` with `IPRoute`. - ''' - yield TestContext[IPRoute](sync_ipr, test_link) - - -@pytest.fixture -def setns_context(nsname: str) -> Generator[SetNSContext]: - '''Set network namespace. - - * **Name**: setns_context - * **Scope**: function - * **Depends**: nsname - - Set a unique network namespace for the current process. Push the new - netns onto the stack, yield the network namespace context, and pop and - cleanup netns on exit. - - Please notice that `setns()` call affects the whole python process. - ''' - with SetNSContext(nsname) as ctx: - yield ctx diff --git a/pyroute2/fixtures/ndb/__init__.py b/pyroute2/fixtures/ndb/__init__.py deleted file mode 100644 index 71937714c..000000000 --- a/pyroute2/fixtures/ndb/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from collections.abc import Generator - -import pytest - -from pyroute2 import NDB - - -@pytest.fixture -def ndb(nsname: str) -> Generator[NDB]: - '''NDB instance. - - * **Name**: ndb - * **Scope**: function - * **Depends**: nsname - - Yield `NDB` instance running in the test network namespace. - ''' - with NDB(sources=[{'target': 'localhost', 'netns': nsname}]) as ndb: - yield ndb diff --git a/pyroute2/fixtures/plan9/__init__.py b/pyroute2/fixtures/plan9/__init__.py deleted file mode 100644 index 1863065d8..000000000 --- a/pyroute2/fixtures/plan9/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -import time -from collections.abc import AsyncGenerator -from socket import socketpair - -import pytest_asyncio - -from pyroute2.plan9.client import Plan9ClientSocket -from pyroute2.plan9.server import Plan9ServerSocket - - -def test_time(): - return time.time_ns() - - -class AsyncPlan9Context: - - server = None - client = None - shutdown_response = None - sample_data = b'Pi6raTaXuzohdu7n' - - def __init__(self): - self.server_sock, self.client_sock = socketpair() - self.server = Plan9ServerSocket(use_socket=self.server_sock) - self.client = Plan9ClientSocket(use_socket=self.client_sock) - self._task = None - with self.server.filesystem.create('test_file') as i: - i.data.write(self.sample_data) - with self.server.filesystem.create('test_time') as i: - i.metadata.call_on_read = True - i.register_function(test_time, loader=lambda x: {}) - - async def ensure_session(self): - self._task = await self.server.async_run() - await self.client.start_session() - - def close(self): - self._task.cancel() - self.client.close() - self.server.close() - - -@pytest_asyncio.fixture -async def async_p9_context() -> AsyncGenerator[AsyncPlan9Context]: - ctx = AsyncPlan9Context() - await ctx.ensure_session() - yield ctx - ctx.close() diff --git a/pyroute2/ipdb/__init__.py b/pyroute2/ipdb/__init__.py index 02740b277..e69de29bb 100644 --- a/pyroute2/ipdb/__init__.py +++ b/pyroute2/ipdb/__init__.py @@ -1,283 +0,0 @@ -''' -IPDB ----- - -Deprecated since 0.7.12, removed from the library. - -The current `pyroute2.ipdb` implementation is a wrapper around NDB -only to provide minimal compatibility for the old legacy code. Do -NOT use IPDB. -''' - -import errno -import logging -import warnings - -from pyroute2.ndb.main import NDB -from pyroute2.netlink.exceptions import NetlinkError - -log = logging.getLogger(__name__) - - -class CreateException(Exception): - pass - - -class CommitException(Exception): - pass - - -class ObjectProxy(dict): - - _translate_keys = {} - - def __init__(self, obj, ready=True): - self._obj = obj - self._ready = ready - - def __getattribute__(self, key): - if key[:4] == 'set_': - - def set_value(value): - self[key[4:]] = value - return self - - return set_value - try: - return self[key] - except KeyError: - return super(ObjectProxy, self).__getattribute__(key) - - def __setattr__(self, key, value): - if key in ('_obj', '_ready', '_translate_keys'): - super(ObjectProxy, self).__setattr__(key, value) - else: - super(ObjectProxy, self).__getattribute__('_obj')[key] = value - - def __getitem__(self, key): - tk = super().__getattribute__('_translate_keys') - if isinstance(key, str) and key in tk: - return super().__getattribute__('_obj')[tk[key](self)] - return super(ObjectProxy, self).__getattribute__('_obj')[key] - - def __setitem__(self, key, value): - tk = super().__getattribute__('_translate_keys') - if isinstance(key, str) and key in tk: - super().__getattribute__('_obj')[tk[key](self)] = value - super(ObjectProxy, self).__getattribute__('_obj')[key] = value - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - if hasattr(self, 'commit'): - self.commit() - - def __repr__(self): - return repr(super(ObjectProxy, self).__getattribute__('_obj')) - - def __contains__(self, key): - return key in super(ObjectProxy, self).__getattribute__('_obj') - - def get_ndb_object(self): - return self._obj - - def get(self, key, *argv): - return self._obj.get(key, *argv) - - def keys(self): - return self._obj.keys() - - def items(self): - return self._obj.items() - - def values(self): - return self._obj.values() - - def __iter__(self): - return self._obj.__iter__() - - @property - def _mode(self): - return 'implicit' - - -class Interface(ObjectProxy): - - _translate_keys = {'mode': lambda x: f'{x["kind"]}_mode'} - - def add_ip(self, address=None, prefixlen=None, **kwarg): - if address is not None: - kwarg['address'] = address - if prefixlen is not None: - kwarg['prefixlen'] = prefixlen - self._obj.add_ip(spec=kwarg) - return self - - def del_ip(self, address=None, prefixlen=None, **kwarg): - if address is not None: - kwarg['address'] = address - if prefixlen is not None: - kwarg['prefixlen'] = prefixlen - self._obj.del_ip(spec=kwarg) - return self - - def add_port(self, *argv, **kwarg): - self._obj.add_port(*argv, **kwarg) - return self - - def del_port(self, *argv, **kwarg): - self._obj.del_port(*argv, **kwarg) - return self - - def commit(self, *argv, **kwarg): - try: - self._obj.commit(*argv, **kwarg) - except Exception as e: - if self._ready: - raise CommitException(e) - else: - raise CreateException(e) - self._ready = True - return self - - def up(self): - self._obj.set('state', 'up') - return self - - def down(self): - self._obj.set('state', 'down') - return self - - def remove(self): - self._obj.remove() - return self - - @property - def if_master(self): - return self._obj.get('master', None) - - @property - def ipaddr(self): - report = self._obj.ipaddr.dump() - report.select_fields('address', 'prefixlen') - return tuple(report) - - -class Interfaces(ObjectProxy): - text_create = ''' -When `create().commit()` fails, the failed interface object behaves -differently in IPDB and NDB. IPDB saves the failed object in the database, -while the NDB database contains only the system reflection, and the failed -object may stay only being referenced by a variable. -''' - - def __getitem__(self, key): - return Interface(super(Interfaces, self).__getitem__(key)) - - def __iter__(self): - return iter(self.keys()) - - def add(self, *argv, **kwarg): - return self.create(*argv, **kwarg) - - def get(self, spec, *argv): - try: - return self[spec] - except KeyError: - if len[argv] > 0: - return argv[0] - raise - - def create(self, *argv, **kwarg): - log.warning(self.text_create) - key = dict( - filter(lambda x: x[0] in ('ifname', 'index'), kwarg.items()) - ) - if key in self: - if kwarg.get('reuse'): - return self[key] - raise CreateException(NetlinkError(errno.EEXIST, 'object exists')) - return Interface(self._obj.create(*argv, **kwarg), ready=False) - - def keys(self): - ret = [] - for record in self._obj.dump(): - ret += [record.ifname, record.index] - return ret - - def has_key(self, key): - return key in self.keys() - - -class IPDB(object): - text_create = ''' -IPDB has a shortcut method to create interfaces: `ipdb.create(...)`. - -NDB has `create()` methods only under respective views: -`ndb.interfaces.create(...)`, `ndb.addresses.create(...)` etc. -''' - - text_nl = ''' -Unlike IPDB, NDB can work with many netlink sources. The default one -referenced as `localhost`:: - - # - # these two statements are equivalent: - # - ndb.sources['localhost'].nl.get_links() - ipdb.nl.get_links() - -''' - - def __init__(self, *argv, **kwarg): - warnings.warn( - ''' - IPDB module is deprecated and removed. - This IPDB instance is just a wrapper around - NDB. See more: - - https://github.com/svinota/pyroute2/wiki/IPDB-EOL - ''', - DeprecationWarning, - ) - sources = kwarg.pop('sources', [{'target': 'localhost'}]) - if argv or kwarg: - log.warning( - '%s does not support IPDB parameters, ignoring', - self.__class__.__name__, - ) - if len(argv) > 0 or 'nl' in kwarg: - log.warning( - '%s does not support shared netlink sources,' - ' ignoring `nl` and starting with local IPRoute', - self.__class__.__name__, - ) - - self._ndb = NDB(sources=sources) - self.interfaces = Interfaces(self._ndb.interfaces) - - def __enter__(self): - return self - - def __exit__(self, *_): - self.release() - - @property - def nl(self): - log.warning(self.text_nl) - return self._ndb.sources['localhost'].nl - - @property - def ipaddr(self): - ret = dict([(x.index, []) for x in self._ndb.interfaces.dump()]) - for record in self._ndb.addresses.dump(): - ret[record.index].append((record.address, record.prefixlen)) - return ret - - def create(self, *argv, **kwarg): - log.warning(self.text_create) - return self.interfaces.create(*argv, **kwarg) - - def release(self): - self._ndb.close() diff --git a/pyroute2/ipdb/exceptions.py b/pyroute2/ipdb/exceptions.py new file mode 100644 index 000000000..b0ec8782f --- /dev/null +++ b/pyroute2/ipdb/exceptions.py @@ -0,0 +1,18 @@ +class DeprecationException(Exception): + pass + + +class CommitException(Exception): + pass + + +class CreateException(Exception): + pass + + +class PartialCommitException(Exception): + pass + + +class ShutdownException(Exception): + pass diff --git a/pyroute2/ipdb/interfaces.py b/pyroute2/ipdb/interfaces.py new file mode 100644 index 000000000..69cb3c7b1 --- /dev/null +++ b/pyroute2/ipdb/interfaces.py @@ -0,0 +1,1500 @@ +import errno +import time +import traceback +from socket import AF_INET, AF_INET6 +from socket import error as socket_error +from socket import inet_ntop, inet_pton + +from pyroute2 import config +from pyroute2.common import Dotkeys, View, basestring, dqn2int +from pyroute2.config import AF_BRIDGE +from pyroute2.ipdb.exceptions import ( + CommitException, + CreateException, + PartialCommitException, +) +from pyroute2.ipdb.linkedset import LinkedSet +from pyroute2.ipdb.transactional import ( + SYNC_TIMEOUT, + Transactional, + with_transaction, +) +from pyroute2.netlink import rtnl +from pyroute2.netlink.exceptions import NetlinkError +from pyroute2.netlink.rtnl.ifinfmsg import IFF_MASK, ifinfmsg + +supported_kinds = ( + 'bridge', + 'bond', + 'tuntap', + 'vxlan', + 'gre', + 'gretap', + 'ip6gre', + 'ip6gretap', + 'macvlan', + 'macvtap', + 'ipvlan', + 'vrf', + 'vti', +) + +groups = ( + rtnl.RTMGRP_LINK + | rtnl.RTMGRP_NEIGH + | rtnl.RTMGRP_IPV4_IFADDR + | rtnl.RTMGRP_IPV6_IFADDR +) + + +def _get_data_fields(): + global supported_kinds + + ret = [] + for data in supported_kinds: + msg = ifinfmsg.ifinfo.data_map.get(data) + if msg is not None: + if getattr(msg, 'prefix', None) is not None: + ret += [msg.nla2name(i[0]) for i in msg.nla_map] + else: + ret += [ifinfmsg.nla2name(i[0]) for i in msg.nla_map] + return ret + + +def _br_time_check(x, y): + return abs(x - y) < 5 + + +class Interface(Transactional): + ''' + Objects of this class represent network interface and + all related objects: + * addresses + * (todo) neighbours + * (todo) routes + + Interfaces provide transactional model and can act as + context managers. Any attribute change implicitly + starts a transaction. The transaction can be managed + with three methods: + * review() -- review changes + * rollback() -- drop all the changes + * commit() -- try to apply changes + + If anything will go wrong during transaction commit, + it will be rolled back authomatically and an + exception will be raised. Failed transaction review + will be attached to the exception. + ''' + + _fields_cmp = { + 'flags': lambda x, y: x & y & IFF_MASK == y & IFF_MASK, + 'br_hello_time': _br_time_check, + 'br_max_age': _br_time_check, + 'br_ageing_time': _br_time_check, + 'br_forward_delay': _br_time_check, + 'br_mcast_membership_intvl': _br_time_check, + 'br_mcast_querier_intvl': _br_time_check, + 'br_mcast_query_intvl': _br_time_check, + 'br_mcast_query_response_intvl': _br_time_check, + 'br_mcast_startup_query_intvl': _br_time_check, + } + _virtual_fields = [ + 'ipdb_scope', + 'ipdb_priority', + 'vlans', + 'ipaddr', + 'ports', + 'vlan_flags', + 'net_ns_fd', + 'net_ns_pid', + ] + _fields = [ifinfmsg.nla2name(i[0]) for i in ifinfmsg.nla_map] + for name in ('bridge_slave_data',): + data = getattr(ifinfmsg.ifinfo, name) + _fields.extend([ifinfmsg.nla2name(i[0]) for i in data.nla_map]) + _fields.append('index') + _fields.append('flags') + _fields.append('mask') + _fields.append('change') + _fields.append('kind') + _fields.append('peer') + _fields.append('vlan_id') + _fields.append('vlan_protocol') + _fields.append('bond_mode') + _fields.extend(_get_data_fields()) + _fields.extend(_virtual_fields) + + def __init__(self, ipdb, mode=None, parent=None, uid=None): + ''' + Parameters: + * ipdb -- ipdb() reference + * mode -- transaction mode + ''' + Transactional.__init__(self, ipdb, mode) + self.cleanup = ( + 'header', + 'linkinfo', + 'protinfo', + 'af_spec', + 'attrs', + 'event', + 'map', + 'stats', + 'stats64', + 'change', + '__align', + ) + self.ingress = None + self.egress = None + self.nlmsg = None + self.errors = [] + self.partial = False + self._exception = None + self._deferred_link = None + self._tb = None + self._linked_sets.add('ipaddr') + self._linked_sets.add('ports') + self._linked_sets.add('vlans') + self._freeze = None + self._delay_add_port = set() + self._delay_del_port = set() + # 8<----------------------------------- + # local setup: direct state is required + with self._direct_state: + for i in ('change', 'mask'): + del self[i] + self['ipaddr'] = self.ipdb._ipaddr_set() + self['ports'] = LinkedSet() + self['vlans'] = LinkedSet() + self['ipdb_priority'] = 0 + # 8<----------------------------------- + + def __hash__(self): + return self['index'] + + @property + def if_master(self): + ''' + [property] Link to the parent interface -- if it exists + ''' + return self.get('master', None) + + def detach(self): + self.ipdb.interfaces._detach(self['ifname'], self['index'], self.nlmsg) + return self + + def freeze(self): + if self._freeze is not None: + raise RuntimeError("the interface is frozen already") + + dump = self.pick() + + def cb(ipdb, msg, action): + if msg.get('index', -1) == dump['index']: + try: + # important: that's a rollback, so do not + # try to revert changes in the case of failure + self.commit( + transaction=dump, commit_phase=2, commit_mask=2 + ) + except Exception: + pass + + self._freeze = self.ipdb.register_callback(cb) + return self + + def unfreeze(self): + self.ipdb.unregister_callback(self._freeze) + self._freeze = None + return self + + def load(self, data): + ''' + Load the data from a dictionary to an existing + transaction. Requires `commit()` call, or must be + called from within a `with` statement. + + Sample:: + + data = json.loads(...) + with ipdb.interfaces['dummy1'] as i: + i.load(data) + + Sample, mode `explicit:: + + data = json.loads(...) + i = ipdb.interfaces['dummy1'] + i.begin() + i.load(data) + i.commit() + ''' + for key in data: + if data[key] is None: + continue + if key == 'ipaddr': + for addr in self['ipaddr']: + self.del_ip(*addr) + for addr in data[key]: + if isinstance(addr, basestring): + addr = (addr,) + self.add_ip(*addr) + elif key == 'ports': + for port in self['ports']: + self.del_port(port) + for port in data[key]: + self.add_port(port) + elif key == 'vlans': + for vlan in self['vlans']: + self.del_vlan(vlan) + for vlan in data[key]: + if vlan != 1: + self.add_vlan(vlan) + elif key in ('neighbours', 'family'): + # ignore on load + pass + else: + self[key] = data[key] + return self + + def load_dict(self, data): + ''' + Update the interface info from a dictionary. + + This call always bypasses open transactions, loading + changes directly into the interface data. + ''' + with self._direct_state: + self.load(data) + + def load_netlink(self, dev): + ''' + Update the interface info from RTM_NEWLINK message. + + This call always bypasses open transactions, loading + changes directly into the interface data. + ''' + global supported_kinds + + with self._direct_state: + if self['ipdb_scope'] == 'locked': + # do not touch locked interfaces + return + + if self['ipdb_scope'] in ('shadow', 'create'): + # ignore non-broadcast messages + if dev['header']['sequence_number'] != 0: + return + # ignore ghost RTM_NEWLINK messages + if (config.kernel[0] < 3) and ( + not dev.get_attr('IFLA_AF_SPEC') + ): + return + + for name, value in dev.items(): + self[name] = value + for cell in dev['attrs']: + # + # Parse on demand + # + # At that moment, being not referenced, the + # NLA is not decoded (yet). Calling + # `__getitem__()` on nla_slot triggers the + # NLA decoding, if the nla is referenced: + # + norm = ifinfmsg.nla2name(cell[0]) + if norm not in self.cleanup: + self[norm] = cell[1] + # load interface kind + linkinfo = dev.get_attr('IFLA_LINKINFO') + if linkinfo is not None: + kind = linkinfo.get_attr('IFLA_INFO_KIND') + if kind is not None: + self['kind'] = kind + if kind == 'vlan': + data = linkinfo.get_attr('IFLA_INFO_DATA') + self['vlan_id'] = data.get_attr('IFLA_VLAN_ID') + self['vlan_protocol'] = data.get_attr( + 'IFLA_VLAN_PROTOCOL' + ) + self['vlan_flags'] = data.get_attr( + 'IFLA_VLAN_FLAGS', {} + ).get('flags', 0) + if kind in supported_kinds: + data = linkinfo.get_attr('IFLA_INFO_DATA') or {} + for nla in data.get('attrs', []): + norm = ifinfmsg.nla2name(nla[0]) + self[norm] = nla[1] + # load vlans + if dev['family'] == AF_BRIDGE: + spec = dev.get_attr('IFLA_AF_SPEC') + if spec is not None: + vlans = spec.get_attrs('IFLA_BRIDGE_VLAN_INFO') + vmap = {} + for vlan in vlans: + vmap[vlan['vid']] = vlan + vids = set(vmap.keys()) + # remove vids we do not have anymore + for vid in self['vlans'] - vids: + self.del_vlan(vid) + for vid in vids - self['vlans']: + self.add_vlan(vmap[vid]) + protinfo = dev.get_attr('IFLA_PROTINFO') + if protinfo is not None: + for attr, value in protinfo['attrs']: + attr = attr[5:].lower() + self[attr] = value + # the rest is possible only when interface + # is used in IPDB, not standalone + if self.ipdb is not None: + self['ipaddr'] = self.ipdb.ipaddr[self['index']] + self['neighbours'] = self.ipdb.neighbours[self['index']] + # finally, cleanup all not needed + for item in self.cleanup: + if item in self: + del self[item] + + # AF_BRIDGE messages for bridges contain + # IFLA_MASTER == self.index, we should fix it + if self.get('master', None) == self['index']: + self['master'] = None + + self['ipdb_scope'] = 'system' + + def wait_ip(self, *argv, **kwarg): + return self['ipaddr'].wait_ip(*argv, **kwarg) + + @with_transaction + def add_ip(self, ip, mask=None, broadcast=None, anycast=None, scope=None): + ''' + Add IP address to an interface + + Address formats: + + with ipdb.interfaces.eth0 as i: + i.add_ip('192.168.0.1', 24) + i.add_ip('192.168.0.2/24') + i.add_ip('192.168.0.3/255.255.255.0') + i.add_ip('192.168.0.4/24', + broadcast='192.168.0.255', + scope=254) + ''' + family = 0 + # split mask + if mask is None: + ip, mask = ip.split('/') + + if ip.find(':') > -1: + family = AF_INET6 + # normalize IPv6 format + ip = inet_ntop(AF_INET6, inet_pton(AF_INET6, ip)) + else: + family = AF_INET + + if isinstance(mask, basestring): + try: + mask = int(mask, 0) + except: + mask = dqn2int(mask, family) + + # if it is a transaction or an interface update, apply the change + self['ipaddr'].unlink((ip, mask)) + request = {} + if broadcast is not None: + request['broadcast'] = broadcast + if anycast is not None: + request['anycast'] = anycast + if scope is not None: + request['scope'] = scope + self['ipaddr'].add((ip, mask), raw=request) + + @with_transaction + def del_ip(self, ip, mask=None): + ''' + Delete IP address from an interface + ''' + if mask is None: + ip, mask = ip.split('/') + if mask.find('.') > -1: + mask = dqn2int(mask) + else: + mask = int(mask, 0) + # normalize the address + if ip.find(':') > -1: + ip = inet_ntop(AF_INET6, inet_pton(AF_INET6, ip)) + if (ip, mask) in self['ipaddr']: + self['ipaddr'].unlink((ip, mask)) + self['ipaddr'].remove((ip, mask)) + + @with_transaction + def add_vlan(self, vlan, flags=None): + if isinstance(vlan, dict): + vid = vlan['vid'] + else: + vid = vlan + vlan = {'vid': vlan, 'flags': 0} + self['vlans'].unlink(vid) + self['vlans'].add(vid, raw=(vlan, flags)) + + @with_transaction + def del_vlan(self, vlan): + if vlan in self['vlans']: + self['vlans'].unlink(vlan) + self['vlans'].remove(vlan) + + @with_transaction + def add_port(self, port): + ''' + Add port to a bridge or bonding + ''' + ifindex = self._resolve_port(port) + if not ifindex: + self._delay_add_port.add(port) + else: + self['ports'].unlink(ifindex) + self['ports'].add(ifindex) + + @with_transaction + def del_port(self, port): + ''' + Remove port from a bridge or bonding + ''' + ifindex = self._resolve_port(port) + if not ifindex: + self._delay_del_port.add(port) + else: + self['ports'].unlink(ifindex) + self['ports'].remove(ifindex) + + def reload(self): + ''' + Reload interface information + ''' + countdown = 3 + while countdown: + links = self.nl.get_links(self['index']) + if links: + self.load_netlink(links[0]) + break + else: + countdown -= 1 + time.sleep(1) + return self + + def review(self): + ret = super(Interface, self).review() + last = self.current_tx + if self['ipdb_scope'] == 'create': + ret['+ipaddr'] = last['ipaddr'] + ret['+ports'] = last['ports'] + ret['+vlans'] = last['vlans'] + del ret['ports'] + del ret['ipaddr'] + del ret['vlans'] + if last._delay_add_port: + ports = set(['*%s' % x for x in last._delay_add_port]) + if '+ports' in ret: + ret['+ports'] |= ports + else: + ret['+ports'] = ports + if last._delay_del_port: + ports = set(['*%s' % x for x in last._delay_del_port]) + if '-ports' in ret: + ret['-ports'] |= ports + else: + ret['-ports'] = ports + return ret + + def _run(self, cmd, *argv, **kwarg): + try: + return cmd(*argv, **kwarg) + except Exception as error: + if self.partial: + self.errors.append(error) + return [] + raise error + + def _resolve_port(self, port): + # for now just a stupid resolver, will be + # improved later with search by mac, etc. + if isinstance(port, Interface): + return port['index'] + else: + return self.ipdb.interfaces.get(port, {}).get('index', None) + + def commit( + self, + tid=None, + transaction=None, + commit_phase=1, + commit_mask=0xFF, + newif=False, + ): + ''' + Commit transaction. In the case of exception all + changes applied during commit will be reverted. + ''' + + if not commit_phase & commit_mask: + return self + + error = None + added = None + removed = None + drop = self.ipdb.txdrop + notx = True + + init = None + debug = {'traceback': None, 'transaction': None, 'next_stage': None} + + if tid or transaction: + notx = False + + if tid: + transaction = self.global_tx[tid] + else: + transaction = transaction or self.current_tx + + if transaction.partial: + transaction.errors = [] + + with self._write_lock: + # if the interface does not exist, create it first ;) + if self['ipdb_scope'] != 'system': + # a special case: transition "create" -> "remove" + if ( + transaction['ipdb_scope'] == 'remove' + and self['ipdb_scope'] == 'create' + ): + self.invalidate() + return self + + newif = True + self.set_target('ipdb_scope', 'system') + try: + # 8<--------------------------------------------------- + # link resolve + if self._deferred_link: + link_key, link_obj = self._deferred_link + transaction[link_key] = self._resolve_port(link_obj) + self._deferred_link = None + + # 8<---------------------------------------------------- + # ACHTUNG: hack for old platforms + if self['address'] == '00:00:00:00:00:00': + with self._direct_state: + self['address'] = None + self['broadcast'] = None + # 8<---------------------------------------------------- + init = self.pick() + try: + request = { + key: transaction[key] + for key in filter( + lambda x: x[:5] != 'bond_' + and x[:7] != 'brport_' + and x[:3] != 'br_', + transaction, + ) + if transaction[key] is not None + } + for key in ('net_ns_fd', 'net_ns_pid'): + if key in request: + with self._direct_state: + self[key] = None + del request[key] + self.nl.link('add', **request) + except NetlinkError as x: + # File exists + if x.code == errno.EEXIST: + # A bit special case, could be one of two cases: + # + # 1. A race condition between two different IPDB + # processes + # 2. An attempt to create dummy0, gre0, bond0 when + # the corrseponding module is not loaded. Being + # loaded, the module creates a default interface + # by itself, causing the request to fail + # + # The exception in that case can cause the DB + # inconsistence, since there can be queued not only + # the interface creation, but also IP address + # changes etc. + # + # So we ignore this particular exception and try to + # continue, as it is created by us. + # + # 3. An attempt to create VLAN or VXLAN interface + # with the same ID but under different name + # + # In that case we should forward error properly + if self['kind'] in ('vlan', 'vxlan'): + newif = x + + else: + raise + except Exception as e: + if transaction.partial: + transaction.errors.append(e) + raise PartialCommitException() + else: + # If link('add', ...) raises an exception, no netlink + # broadcast will be sent, and the object is unmodified. + # After the exception forwarding, the object is ready + # to repeat the commit() call. + if drop and notx: + self.drop(transaction.uid) + raise + + if transaction['ipdb_scope'] == 'create' and commit_phase > 1: + if self['index']: + wd = self.ipdb.watchdog('RTM_DELLINK', ifname=self['ifname']) + with self._direct_state: + self['ipdb_scope'] = 'locked' + self.nl.link('delete', index=self['index']) + wd.wait() + self.load_dict(transaction) + return self + + elif newif: + # Here we come only if a new interface is created + # + if commit_phase == 1 and not self.wait_target('ipdb_scope'): + if drop and notx: + self.drop(transaction.uid) + self.invalidate() + if isinstance(newif, Exception): + raise newif + else: + raise CreateException() + + # Re-populate transaction.ipaddr to have a proper IP target + # + # The reason behind the code is that a new interface in the + # "up" state will have automatic IPv6 addresses, that aren't + # reflected in the transaction. This may cause a false IP + # target mismatch and a commit failure. + # + # To avoid that, collect automatic addresses to the + # transaction manually, since it is not yet properly linked. + # + for addr in self.ipdb.ipaddr[self['index']]: + transaction['ipaddr'].add(addr) + + # Reload the interface data + try: + self.load_netlink(self.nl.link('get', **request)[0]) + except Exception: + pass + + # now we have our index and IP set and all other stuff + snapshot = self.pick() + + # make snapshots of all dependent routes + if commit_phase == 1 and hasattr(self.ipdb, 'routes'): + self.routes = [] + for record in self.ipdb.routes.filter({'oif': self['index']}): + # For MPLS routes the key is an integer + # They should match anyways + if getattr(record['key'], 'table', None) != 255: + self.routes.append( + (record['route'], record['route'].pick()) + ) + + # resolve all delayed ports + def resolve_ports(transaction, ports, callback, self, drop): + def error(x): + return KeyError('can not resolve port %s' % x) + + for port in tuple(ports): + ifindex = self._resolve_port(port) + if not ifindex: + if transaction.partial: + transaction.errors.append(error(port)) + else: + if drop: + self.drop(transaction.uid) + raise error(port) + else: + ports.remove(port) + with transaction._direct_state: # ???? + callback(ifindex) + + resolve_ports( + transaction, + transaction._delay_add_port, + transaction.add_port, + self, + drop and notx, + ) + resolve_ports( + transaction, + transaction._delay_del_port, + transaction.del_port, + self, + drop and notx, + ) + + try: + removed, added = snapshot // transaction + + run = transaction._run + nl = transaction.nl + + # 8<--------------------------------------------- + # Port vlans + if removed['vlans'] or added['vlans']: + self['vlans'].set_target(transaction['vlans']) + + for i in removed['vlans']: + # remove vlan from the port + run( + nl.vlan_filter, + 'del', + index=self['index'], + vlan_info=self['vlans'][i][0], + ) + + for i in added['vlans']: + # add vlan to the port + vinfo = transaction['vlans'][i][0] + flags = transaction['vlans'][i][1] + req = {'index': self['index'], 'vlan_info': vinfo} + if flags == 'self': + req['vlan_flags'] = flags + # this request will NOT give echo, + # so bypass the check + with self._direct_state: + self.add_vlan(vinfo['vid']) + run(nl.vlan_filter, 'add', **req) + + self['vlans'].target.wait(SYNC_TIMEOUT) + if not self['vlans'].target.is_set(): + raise CommitException('vlans target is not set') + + # 8<--------------------------------------------- + # Ports + if removed['ports'] or added['ports']: + self['ports'].set_target(transaction['ports']) + + for i in removed['ports']: + # detach port + if i in self.ipdb.interfaces: + ( + self.ipdb.interfaces[i] + .set_target('master', None) + .mirror_target('master', 'link') + ) + run(nl.link, 'update', index=i, master=0) + else: + transaction.errors.append(KeyError(i)) + + for i in added['ports']: + # attach port + if i in self.ipdb.interfaces: + ( + self.ipdb.interfaces[i] + .set_target('master', self['index']) + .mirror_target('master', 'link') + ) + run(nl.link, 'update', index=i, master=self['index']) + else: + transaction.errors.append(KeyError(i)) + + self['ports'].target.wait(SYNC_TIMEOUT) + if self['ports'].target.is_set(): + for msg in self.nl.get_vlans(index=self['index']): + self.load_netlink(msg) + else: + raise CommitException('ports target is not set') + + # 1. wait for proper targets on ports + # 2. wait for mtu sync + # + # the bridge mtu is set from the port, if the latter is smaller + # the bond mtu sets the port mtu, if the latter is smaller + # + # FIXME: team interfaces? + for i in list(added['ports']) + list(removed['ports']): + port = self.ipdb.interfaces[i] + # port update + target = port._local_targets['master'] + target.wait(SYNC_TIMEOUT) + with port._write_lock: + del port._local_targets['master'] + del port._local_targets['link'] + if not target.is_set(): + raise CommitException('master target failed') + if i in added['ports']: + if port.if_master != self['index']: + raise CommitException('master set failed') + else: + if port.if_master == self['index']: + raise CommitException('master unset failed') + # master update + if self['kind'] == 'bridge' and self['mtu'] > port['mtu']: + self.set_target('mtu', port['mtu']) + self.wait_target('mtu') + + # 8<--------------------------------------------- + # Interface changes + request = {} + brequest = {} + prequest = {} + # preseed requests with the interface kind + request['kind'] = self['kind'] + brequest['kind'] = self['kind'] + wait_all = False + for key, value in added.items(): + if ( + value is not None + and (key not in self._virtual_fields) + and (key != 'kind') + ): + if key[:3] == 'br_': + brequest[key] = added[key] + elif key[:7] == 'brport_': + prequest[key[7:]] = added[key] + else: + if key == 'address' and added[key] is not None: + self[key] = added[key].lower() + request[key] = added[key] + # FIXME: flush the interface type so the next two conditions + # will work correctly + request['kind'] = None + brequest['kind'] = None + + # apply changes only if there is something to apply + if (self['kind'] == 'bridge') and any( + [brequest[item] is not None for item in brequest] + ): + brequest['index'] = self['index'] + brequest['kind'] = self['kind'] + brequest['family'] = AF_BRIDGE + wait_all = True + run(nl.link, 'set', **brequest) + + if any([request[item] is not None for item in request]): + request['index'] = self['index'] + request['kind'] = self['kind'] + if request.get('address', None) == '00:00:00:00:00:00': + request.pop('address') + request.pop('broadcast', None) + wait_all = True + run(nl.link, 'update', **request) + # Yet another trick: setting ifalias doesn't cause + # netlink updates + if 'ifalias' in request: + self.reload() + + if any([prequest[item] is not None for item in prequest]): + prequest['index'] = self['index'] + run(nl.brport, 'set', **prequest) + + if (wait_all) and (not transaction.partial): + transaction.wait_all_targets() + + # 8<--------------------------------------------- + # VLAN flags -- a dirty hack, pls do something with it + if added.get('vlan_flags') is not None: + run( + nl.link, + 'set', + **{ + 'kind': 'vlan', + 'index': self['index'], + 'vlan_flags': added['vlan_flags'], + } + ) + + # 8<--------------------------------------------- + # IP address changes + for _ in range(3): + ip2add = transaction['ipaddr'] - self['ipaddr'] + ip2remove = self['ipaddr'] - transaction['ipaddr'] + + if not ip2add and not ip2remove: + break + + self['ipaddr'].set_target(transaction['ipaddr']) + ### + # Remove + # + # The promote_secondaries sysctl causes the kernel + # to add secondary addresses back after the primary + # address is removed. + # + # The library can not tell this from the result of + # an external program. + # + # One simple way to work that around is to remove + # secondaries first. + rip = sorted( + ip2remove, + key=lambda x: self['ipaddr'][x]['flags'], + reverse=True, + ) + # 8<-------------------------------------- + for i in rip: + # When you remove a primary IP addr, all the + # subnetwork can be removed. In this case you + # will fail, but it is OK, no need to roll back + try: + run( + nl.addr, + 'delete', + index=self['index'], + address=i[0], + prefixlen=i[1], + ) + except NetlinkError as x: + # bypass only errno 99, + # 'Cannot assign address' + if x.code != errno.EADDRNOTAVAIL: + raise + except socket_error as x: + # bypass illegal IP requests + if isinstance(x.args[0], basestring) and x.args[ + 0 + ].startswith('illegal IP'): + continue + raise + ### + # Add addresses + # 8<-------------------------------------- + for i in ip2add: + # Try to fetch additional address attributes + try: + kwarg = dict( + [ + k + for k in transaction['ipaddr'][i].items() + if k[0] in ('broadcast', 'anycast', 'scope') + ] + ) + except KeyError: + kwarg = None + try: + # feed the address to the OS + kwarg = kwarg or {} + kwarg['index'] = self['index'] + kwarg['address'] = i[0] + kwarg['prefixlen'] = i[1] + run(nl.addr, 'add', **kwarg) + except NetlinkError as x: + if x.code != errno.EEXIST: + raise + + # 8<-------------------------------------- + # some interfaces do not send IPv6 address + # updates, when are down + # + # beside of that, bridge interfaces are + # down by default, so they never send + # address updates from beginning + # + # FIXME: + # + # that all is a dirtiest hack ever, pls do + # something with it + # + if (not self['flags'] & 1) or hasattr(self.ipdb.nl, 'netns'): + # 1. flush old IPv6 addresses + for addr in list(self['ipaddr'].ipv6): + self['ipaddr'].remove(addr) + # 2. reload addresses + for addr in self.nl.get_addr( + index=self['index'], family=AF_INET6 + ): + self.ipdb.ipaddr._new(addr) + # if there are tons of IPv6 addresses, it may take a + # really long time, and that's bad, but it's broken in + # the kernel :| + + # 8<-------------------------------------- + self['ipaddr'].target.wait(SYNC_TIMEOUT) + if self['ipaddr'].target.is_set(): + break + else: + raise CommitException('ipaddr target is not set') + + # 8<--------------------------------------------- + # Iterate callback chain + for ch in self._commit_hooks: + # An exception will rollback the transaction + ch(self.dump(), snapshot.dump(), transaction.dump()) + + # 8<--------------------------------------------- + # Move the interface to a netns + if ('net_ns_fd' in added) or ('net_ns_pid' in added): + request = {} + for key in ('net_ns_fd', 'net_ns_pid'): + if key in added: + request[key] = added[key] + + request['index'] = self['index'] + run(nl.link, 'update', **request) + + countdown = 10 + while countdown: + # wait until the interface will disappear + # from the current network namespace -- + # up to 1 second (make it configurable?) + try: + self.nl.get_links(self['index']) + except NetlinkError as e: + if e.code == errno.ENODEV: + break + raise + except Exception: + raise + countdown -= 1 + time.sleep(0.1) + + # 8<--------------------------------------------- + # Interface removal + if added.get('ipdb_scope') in ('shadow', 'remove'): + wd = self.ipdb.watchdog('RTM_DELLINK', ifname=self['ifname']) + with self._direct_state: + self['ipdb_scope'] = 'locked' + self.nl.link('delete', index=self['index']) + wd.wait() + + with self._direct_state: + self['ipdb_scope'] = 'shadow' + + # system-wide checks + if commit_phase == 1: + self.ipdb.ensure('run') + + if added.get('ipdb_scope') == 'remove': + self.ipdb.interfaces._detach(None, self['index'], None) + + if notx: + self.drop(transaction.uid) + + return self + # 8<--------------------------------------------- + + # system-wide checks + if commit_phase == 1: + self.ipdb.ensure('run') + + # so far all's ok + drop = True + + except Exception as e: + error = e + # log the error environment + debug['traceback'] = traceback.format_exc() + debug['transaction'] = transaction + debug['next_stage'] = None + + # something went wrong: roll the transaction back + if commit_phase == 1: + if newif: + drop = False + try: + self.commit( + transaction=init if newif else snapshot, + commit_phase=2, + commit_mask=commit_mask, + newif=newif, + ) + + except Exception as i_e: + debug['next_stage'] = i_e + error = RuntimeError() + else: + # reload all the database -- it can take a long time, + # but it is required since we have no idea, what is + # the result of the failure + links = self.nl.get_links() + for link in links: + self.ipdb.interfaces._new(link) + links = self.nl.get_vlans() + for link in links: + self.ipdb.interfaces._new(link) + for addr in self.nl.get_addr(): + self.ipdb.ipaddr._new(addr) + + for key in ('ipaddr', 'ports', 'vlans'): + self[key].clear_target() + + # raise partial commit exceptions + if transaction.partial and transaction.errors: + error = PartialCommitException('partial commit error') + + # drop only if required + if drop and notx: + # drop last transaction in any case + self.drop(transaction.uid) + + # raise exception for failed transaction + if error is not None: + error.debug = debug + raise error + + # restore dependent routes for successful rollback + if commit_phase == 2: + for route in self.routes: + with route[0]._direct_state: + route[0]['ipdb_scope'] = 'restore' + try: + route[0].commit( + transaction=route[1], commit_phase=2, commit_mask=2 + ) + except RuntimeError as x: + # RuntimeError is raised due to phase 2, so + # an additional check is required + if ( + isinstance(x.cause, NetlinkError) + and x.cause.code == errno.EEXIST + ): + pass + + time.sleep(config.commit_barrier) + + # drop all collected errors, if any + self.errors = [] + return self + + def up(self): + ''' + Shortcut: change the interface state to 'up'. + ''' + self['state'] = 'up' + return self + + def down(self): + ''' + Shortcut: change the interface state to 'down'. + ''' + self['state'] = 'down' + return self + + def remove(self): + ''' + Mark the interface for removal + ''' + self['ipdb_scope'] = 'remove' + return self + + def shadow(self): + ''' + Remove the interface from the OS, but leave it in the + database. When one will try to re-create interface with + the same name, all the old saved attributes will apply + to the new interface, incl. MAC-address and even the + interface index. Please be aware, that the interface + index can be reused by OS while the interface is "in the + shadow state", in this case re-creation will fail. + ''' + self['ipdb_scope'] = 'shadow' + return self + + +class InterfacesDict(Dotkeys): + def __init__(self, ipdb): + self.ipdb = ipdb + self._event_map = {'RTM_NEWLINK': self._new, 'RTM_DELLINK': self._del} + + def _register(self): + links = self.ipdb.nl.get_links() + # iterate twice to map port/master relations + for link in links: + self._new(link, skip_master=True) + for link in links: + self._new(link) + # load bridge vlan information + links = self.ipdb.nl.get_vlans() + for link in links: + self._new(link) + + def add(self, kind, ifname, reuse=False, **kwarg): + ''' + Create new network interface + ''' + with self.ipdb.exclusive: + # check for existing interface + if ifname in self: + if (self[ifname]['ipdb_scope'] == 'shadow') or reuse: + device = self[ifname] + kwarg['kind'] = kind + device.load_dict(kwarg) + if self[ifname]['ipdb_scope'] == 'shadow': + with device._direct_state: + device['ipdb_scope'] = 'create' + device.begin() + else: + raise CreateException("interface %s exists" % ifname) + else: + device = self[ifname] = Interface( + ipdb=self.ipdb, mode='snapshot' + ) + # delay link resolve? + for key in kwarg: + # any /.+link$/ attr + if key[-4:] == 'link': + if isinstance(kwarg[key], Interface): + kwarg[key] = kwarg[key].get('index') or kwarg[ + key + ].get('ifname') + if not isinstance(kwarg[key], int): + device._deferred_link = (key, kwarg[key]) + device._mode = self.ipdb.mode + with device._direct_state: + device['kind'] = kind + device['index'] = kwarg.get('index', 0) + device['ifname'] = ifname + device['ipdb_scope'] = 'create' + # set some specific attrs + for attr in ( + 'peer', + 'uid', + 'gid', + 'ifr', + 'mode', + 'bond_mode', + 'address', + ): + if attr in kwarg: + device[attr] = kwarg.pop(attr) + device.begin() + device.load(kwarg) + return device + + def _del(self, msg): + target = self.get(msg['index']) + if target is None: + return + + if msg['family'] == AF_BRIDGE: + with target._direct_state: + for vlan in tuple(target['vlans']): + target.del_vlan(vlan) + + # check for freezed devices + if getattr(target, '_freeze', None): + with target._direct_state: + target['ipdb_scope'] = 'shadow' + return + + # check for locked devices + if target.get('ipdb_scope') in ('locked', 'shadow'): + return + + self._detach(None, msg['index'], msg) + + def _new(self, msg, skip_master=False): + # check, if a record exists + index = msg.get('index', None) + ifname = msg.get_attr('IFLA_IFNAME', None) + device = None + cleanup = None + + # scenario #1: no matches for both: new interface + # + # scenario #2: ifname exists, index doesn't: + # index changed + # scenario #3: index exists, ifname doesn't: + # name changed + # scenario #4: both exist: assume simple update and + # an optional name change + + if (index not in self) and (ifname not in self): + # scenario #1, new interface + device = self[index] = self[ifname] = Interface(ipdb=self.ipdb) + elif (index not in self) and (ifname in self): + # scenario #2, index change + old_index = self[ifname]['index'] + device = self[index] = self[ifname] + if old_index in self: + cleanup = old_index + + if old_index in self.ipdb.ipaddr: + self.ipdb.ipaddr[index] = self.ipdb.ipaddr[old_index] + del self.ipdb.ipaddr[old_index] + + if old_index in self.ipdb.neighbours: + self.ipdb.neighbours[index] = self.ipdb.neighbours[old_index] + del self.ipdb.neighbours[old_index] + else: + # scenario #3, interface rename + # scenario #4, assume rename + old_name = self[index]['ifname'] + if old_name != ifname: + # unlink old name + cleanup = old_name + device = self[ifname] = self[index] + + if index not in self.ipdb.ipaddr: + self.ipdb.ipaddr[index] = self.ipdb._ipaddr_set() + + if index not in self.ipdb.neighbours: + self.ipdb.neighbours[index] = LinkedSet() + + # update port references + old_master = device.get('master', None) + new_master = msg.get_attr('IFLA_MASTER') + + if old_master != new_master: + if old_master in self: + with self[old_master]._direct_state: + if index in self[old_master]['ports']: + self[old_master].del_port(index) + if new_master in self and new_master != index: + with self[new_master]._direct_state: + self[new_master].add_port(index) + + if cleanup is not None: + del self[cleanup] + + if skip_master: + msg.strip('IFLA_MASTER') + + device.load_netlink(msg) + if new_master is None: + with device._direct_state: + device['master'] = None + + def _detach(self, name, idx, msg=None): + with self.ipdb.exclusive: + if msg is not None: + if ( + msg['event'] == 'RTM_DELLINK' + and msg['change'] != 0xFFFFFFFF + ): + return + if idx is None or idx < 1: + target = self[name] + idx = target['index'] + else: + target = self[idx] + name = target['ifname'] + # clean up port, if exists + master = target.get('master', None) + if master in self and target['index'] in self[master]['ports']: + with self[master]._direct_state: + self[master].del_port(target) + self.pop(name, None) + self.pop(idx, None) + self.ipdb.ipaddr.pop(idx, None) + self.ipdb.neighbours.pop(idx, None) + with target._direct_state: + target['ipdb_scope'] = 'detached' + + +class AddressesDict(dict): + def __init__(self, ipdb): + self.ipdb = ipdb + self._event_map = {'RTM_NEWADDR': self._new, 'RTM_DELADDR': self._del} + + def _register(self): + for msg in self.ipdb.nl.get_addr(): + self._new(msg) + + def reload(self): + # Reload addresses from the kernel. + # (This is a workaround to reorder primary and secondary addresses.) + for k in self.keys(): + self[k] = self.ipdb._ipaddr_set() + for msg in self.ipdb.nl.get_addr(): + self._new(msg) + for idx in self.keys(): + iff = self.ipdb.interfaces[idx] + with iff._direct_state: + iff['ipaddr'] = self[idx] + + def _new(self, msg): + if msg['family'] == AF_INET: + addr = msg.get_attr('IFA_LOCAL') + elif msg['family'] == AF_INET6: + addr = msg.get_attr('IFA_LOCAL') + if not addr: + addr = msg.get_attr('IFA_ADDRESS') + else: + return + raw = { + 'local': msg.get_attr('IFA_LOCAL'), + 'broadcast': msg.get_attr('IFA_BROADCAST'), + 'address': msg.get_attr('IFA_ADDRESS'), + 'flags': msg.get_attr('IFA_FLAGS') or msg.get('flags'), + 'prefixlen': msg['prefixlen'], + 'family': msg['family'], + 'cacheinfo': msg.get_attr('IFA_CACHEINFO'), + } + try: + self[msg['index']].add(key=(addr, raw['prefixlen']), raw=raw) + except: + pass + + def _del(self, msg): + if msg['family'] == AF_INET: + addr = msg.get_attr('IFA_LOCAL') + elif msg['family'] == AF_INET6: + addr = msg.get_attr('IFA_ADDRESS') + else: + return + try: + self[msg['index']].remove((addr, msg['prefixlen'])) + except: + pass + + +class NeighboursDict(dict): + def __init__(self, ipdb): + self.ipdb = ipdb + self._event_map = { + 'RTM_NEWNEIGH': self._new, + 'RTM_DELNEIGH': self._del, + } + + def _register(self): + for msg in self.ipdb.nl.get_neighbours(): + self._new(msg) + + def _new(self, msg): + if msg['family'] == AF_BRIDGE: + return + + try: + ( + self[msg['ifindex']].add( + key=msg.get_attr('NDA_DST'), + raw={'lladdr': msg.get_attr('NDA_LLADDR')}, + ) + ) + except: + pass + + def _del(self, msg): + if msg['family'] == AF_BRIDGE: + return + try: + (self[msg['ifindex']].remove(msg.get_attr('NDA_DST'))) + except: + pass + + +spec = [ + {'name': 'interfaces', 'class': InterfacesDict, 'kwarg': {}}, + { + 'name': 'by_name', + 'class': View, + 'kwarg': { + 'path': 'interfaces', + 'constraint': lambda k, v: isinstance(k, basestring), + }, + }, + { + 'name': 'by_index', + 'class': View, + 'kwarg': { + 'path': 'interfaces', + 'constraint': lambda k, v: isinstance(k, int), + }, + }, + {'name': 'ipaddr', 'class': AddressesDict, 'kwarg': {}}, + {'name': 'neighbours', 'class': NeighboursDict, 'kwarg': {}}, +] diff --git a/pyroute2/ipdb/linkedset.py b/pyroute2/ipdb/linkedset.py new file mode 100644 index 000000000..b90bca8e7 --- /dev/null +++ b/pyroute2/ipdb/linkedset.py @@ -0,0 +1,307 @@ +''' +''' + +import struct +import threading +from collections import OrderedDict +from socket import AF_INET, AF_INET6, inet_pton + +from pyroute2.common import basestring + + +class LinkedSet(set): + ''' + Utility class, used by `Interface` to track ip addresses + and ports. Called "linked" as it automatically updates all + instances, linked with it. + + Target filter is a function, that returns `True` if a set + member should be counted in target checks (target methods + see below), or `False` if it should be ignored. + ''' + + def target_filter(self, x): + return True + + def __init__(self, *argv, **kwarg): + set.__init__(self, *argv, **kwarg) + + def _check_default_target(self): + if self._ct is not None: + if set(filter(self.target_filter, self)) == set( + filter(self.target_filter, self._ct) + ): + self._ct = None + return True + return False + + self.lock = threading.RLock() + self.target = threading.Event() + self.targets = {self.target: _check_default_target} + self._ct = None + self.raw = OrderedDict() + self.links = [] + self.exclusive = set() + + def __getitem__(self, key): + return self.raw[key] + + def clear_target(self, target=None): + with self.lock: + if target is None: + self._ct = None + self.target.clear() + else: + target.clear() + del self.targets[target] + + def set_target(self, value, ignore_state=False): + ''' + Set target state for the object and clear the target + event. Once the target is reached, the event will be + set, see also: `check_target()` + + Args: + - value (set): the target state to compare with + ''' + with self.lock: + if isinstance(value, (set, tuple, list)): + self._ct = value + self.target.clear() + # immediately check, if the target already + # reached -- otherwise you will miss the + # target forever + if not ignore_state: + self.check_target() + elif hasattr(value, '__call__'): + new_target = threading.Event() + self.targets[new_target] = value + if not ignore_state: + self.check_target() + return new_target + else: + raise TypeError("target type not supported") + + def check_target(self): + ''' + Check the target state and set the target event in the + case the state is reached. Called from mutators, `add()` + and `remove()` + ''' + with self.lock: + for evt in self.targets: + if self.targets[evt](self): + evt.set() + + def add(self, key, raw=None, cascade=False): + ''' + Add an item to the set and all connected instances, + check the target state. + + Args: + - key: any hashable object + - raw (optional): raw representation of the object + + Raw representation is not required. It can be used, e.g., + to store RTM_NEWADDR RTNL messages along with + human-readable ip addr representation. + ''' + with self.lock: + if cascade and (key in self.exclusive): + return + if key not in self: + self.raw[key] = raw + super(LinkedSet, self).add(key) + for link in self.links: + link.add(key, raw, cascade=True) + self.check_target() + + def remove(self, key, raw=None, cascade=False): + ''' + Remove an item from the set and all connected instances, + check the target state. + ''' + with self.lock: + if cascade and (key in self.exclusive): + return + super(LinkedSet, self).remove(key) + self.raw.pop(key, None) + for link in self.links: + if key in link: + link.remove(key, cascade=True) + self.check_target() + + def unlink(self, key): + ''' + Exclude key from cascade updates. + ''' + self.exclusive.add(key) + + def relink(self, key): + ''' + Do not ignore key on cascade updates. + ''' + self.exclusive.remove(key) + + def connect(self, link): + ''' + Connect a LinkedSet instance to this one. Connected + sets will be updated together with this instance. + ''' + if not isinstance(link, LinkedSet): + raise TypeError() + self.links.append(link) + + def disconnect(self, link): + self.links.remove(link) + + def __repr__(self): + return repr(tuple(self)) + + +class IPaddrSet(LinkedSet): + ''' + LinkedSet child class with different target filter. The + filter ignores link local IPv6 addresses when sets and checks + the target. + + The `wait_ip()` routine by default does not ignore link local + IPv6 addresses, but it may be changed with the `ignore_link_local` + argument. + ''' + + @property + def ipv4(self): + ret = IPaddrSet() + for x in self: + if self[x]['family'] == AF_INET: + ret.add(x, self[x]) + return ret + + @property + def ipv6(self): + ret = IPaddrSet() + for x in self: + if self[x]['family'] == AF_INET6: + ret.add(x, self[x]) + return ret + + def wait_ip(self, net, mask=None, timeout=None, ignore_link_local=False): + family = AF_INET6 if net.find(':') >= 0 else AF_INET + alen = 32 if family == AF_INET else 128 + net = inet_pton(family, net) + if mask is None: + mask = alen + if family == AF_INET: + net = struct.unpack('>I', net)[0] + else: + na, nb = struct.unpack('>QQ', net) + net = (na << 64) | nb + match = net & (((1 << mask) - 1) << (alen - mask)) + + def match_ip(ipset): + for rnet, rmask in ipset: + rfamily = AF_INET6 if rnet.find(':') >= 0 else AF_INET + if family != rfamily: + continue + if ( + family == AF_INET6 + and ignore_link_local + and rnet[:4] == 'fe80' + and rmask == 64 + ): + continue + rnet = inet_pton(family, rnet) + if family == AF_INET: + rnet = struct.unpack('>I', rnet)[0] + else: + rna, rnb = struct.unpack('>QQ', rnet) + rnet = (rna << 64) | rnb + if (rnet & (((1 << mask) - 1) << (alen - mask))) == match: + return True + return False + + target = self.set_target(match_ip) + target.wait(timeout) + ret = target.is_set() + self.clear_target(target) + return ret + + def __getitem__(self, key): + if isinstance(key, (tuple, list)): + return self.raw[key] + elif isinstance(key, int): + return self.raw[tuple(self.raw.keys())[key]] + elif isinstance(key, basestring): + key = key.split('/') + key = (key[0], int(key[1])) + return self.raw[key] + else: + TypeError('wrong key type') + + +class SortedIPaddrSet(IPaddrSet): + def __init__(self, *argv, **kwarg): + super(SortedIPaddrSet, self).__init__(*argv, **kwarg) + if argv and isinstance(argv[0], SortedIPaddrSet): + # Re-initialize self.raw from argv[0].raw to preserve order: + self.raw = OrderedDict(argv[0].raw) + + def __and__(self, other): + nset = SortedIPaddrSet(self) + return nset.__iand__(other) + + def __iand__(self, other): + for key in self.raw: + if key not in other: + self.remove(key) + return self + + def __rand__(self, other): + return self.__and__(other) + + def __xor__(self, other): + nset = SortedIPaddrSet(self) + return nset.__ixor__(other) + + def __ixor__(self, other): + if not isinstance(other, SortedIPaddrSet): + return RuntimeError('SortedIPaddrSet instance required') + xor_keys = set(self.raw.keys()) ^ set(other.raw.keys()) + for key in xor_keys: + if key in self: + self.remove(key) + else: + self.add(key, raw=other.raw[key], cascade=False) + return self + + def __rxor__(self, other): + return self.__xor__(other) + + def __or__(self, other): + nset = SortedIPaddrSet(self) + return nset.__ior__(other) + + def __ior__(self, other): + if not isinstance(other, SortedIPaddrSet): + return RuntimeError('SortedIPaddrSet instance required') + for key, value in other.raw.items(): + if key not in self: + self.add(key, raw=value, cascade=False) + return self + + def __ror__(self, other): + return self.__or__(other) + + def __sub__(self, other): + nset = SortedIPaddrSet(self) + return nset.__isub__(other) + + def __isub__(self, other): + for key in other: + if key in self: + self.remove(key) + return self + + def __iter__(self): + return iter(self.raw) diff --git a/pyroute2/ipdb/main.py b/pyroute2/ipdb/main.py new file mode 100644 index 000000000..5de906c1d --- /dev/null +++ b/pyroute2/ipdb/main.py @@ -0,0 +1,1525 @@ +# -*- coding: utf-8 -*- +''' +IPDB guide +========== + +.. warning:: + The IPDB module has design issues that may not be + fixed. It is recommended to switch to NDB wherever + it's possible. + +Basically, IPDB is a transactional database, containing +records, that represent network stack objects. Any change +in the database is not reflected immediately in OS, but +waits until `commit()` is called. One failed operation +during `commit()` rolls back all the changes, has been made +so far. Moreover, IPDB has commit hooks API, that allows +you to roll back changes depending on your own function +calls, e.g. when a host or a network becomes unreachable. + +Limitations +----------- + +One of the major issues with IPDB is its memory footprint. It +proved not to be suitable for environments with thousands of +routes or neighbours. Being a design issue, it could not be +fixed, so a new module was started, NDB, that aims to replace +IPDB. IPDB is still more feature rich, but NDB is already more +fast and stable. + +IPDB, NDB, IPRoute +------------------ + +These modules use different approaches. + +* IPRoute just forwards requests to the kernel, and doesn't + wait for the system state. So it's up to developer to check, + whether the requested object is really set up or not. +* IPDB is an asynchronously updated database, that starts + several additional threads by default. If your project's policy + doesn't allow implicit threads, keep it in mind. But unlike IPRoute, + the IPDB ensures the changes to be reflected in the system. +* NDB is like IPDB, and will obsolete it in the future. The difference + is that IPDB creates Python object for every RTNL object, while + NDB stores everything in an SQL DB, and creates objects on demand. + +Being asynchronously updated, IPDB does sync on commit:: + + with IPDB() as ipdb: + with ipdb.interfaces['eth0'] as i: + i.up() + i.add_ip('192.168.0.2/24') + i.add_ip('192.168.0.3/24') + # ---> <--- here you can expect `eth0` is up + # and has these two addresses, so + # the following code can rely on that + +NB: *In the example above `commit()` is implied with the +`__exit__()` of the `with` statement.* + +IPDB and other software +----------------------- + +IPDB is designed to be a non-exclusive network settings database. +There may be several IPDB instances on the same OS, as well as +other network management software, such as NetworkManager etc. + +The IPDB transactions should not interfere with other software +settings, unless they touch the same objects. E.g., if IPDB +brings an interface up, while NM shuts it down, there will be +a race condition. + +An example:: + + # IPDB code # NetworkManager at the same time: + ipdb.interfaces['eth0'].up() # + ipdb.interfaces['eth0'].commit() # $ sudo nmcli con down eth0 + # ---> <--- + # The eth0 state here is undefined. Some of the commands + # above will fail + +But as long as the software doesn't touch the same objects, there +will be no conflicts. Another example:: + + # IPDB code # At the same time, NetworkManager + with ipdb.interfaces['eth0'] as i: # adds addresses: + i.add_ip('172.16.254.2/24') # * 10.0.0.2/24 + i.add_ip('172.16.254.3/24') # * 10.0.0.3/24 + # ---> <--- + # At this point the eth0 interface will have all four addresses. + # If the IPDB transaction fails by some reason, only IPDB addresses + # will be rolled back. + +There may be a need to prevent other software from changing the network +settings. There is no locking at the kernel level, but IPDB can revert +all the changes as soon as they appear on the interface:: + + # IPDB code + ipdb.interfaces['eth0'].freeze() + # Here some other software tries to + # add an address, or to remove the old + # one + # ---> <--- + # At this point the eth0 interface will have all the same settings as + # at the `freeze()` call moment. Newly added addresses will be removed, + # all the deleted addresses will be restored. + # + # Please notice, that an address removal may cause also a routes removal, + # and that is the thing that IPDB can not neither prevent, nor revert. + + ipdb.interfaces['eth0'].unfreeze() + +Quickstart +---------- + +Simple tutorial:: + + from pyroute2 import IPDB + # several IPDB instances are supported within on process + ipdb = IPDB() + + # commit is called automatically upon the exit from `with` + # statement + with ipdb.interfaces.eth0 as i: + i.address = '00:11:22:33:44:55' + i.ifname = 'bala' + i.txqlen = 2000 + + # basic routing support + ipdb.routes.add({'dst': 'default', + 'gateway': '10.0.0.1'}).commit() + + # do not forget to shutdown IPDB + ipdb.release() + +Please, notice `ip.release()` call in the end. Though it is +not forced in an interactive python session for the better +user experience, it is required in the scripts to sync the +IPDB state before exit. + +IPDB supports functional-like syntax also:: + + from pyroute2 import IPDB + with IPDB() as ipdb: + intf = (ipdb.interfaces['eth0'] + .add_ip('10.0.0.2/24') + .add_ip('10.0.0.3/24') + .set_address('00:11:22:33:44:55') + .set_mtu(1460) + .set_name('external') + .commit()) + # ---> <--- here you have the interface reference with + # all the changes applied: renamed, added ipaddr, + # changed macaddr and mtu. + ... # some code + + # pls notice, that the interface reference will not work + # outside of `with IPDB() ...` + +Transaction modes +----------------- +IPDB has several operating modes: + + - 'implicit' (default) -- the first change starts an implicit + transaction, that have to be committed + - 'explicit' -- you have to begin() a transaction prior to + make any change + +The default is to use implicit transaction. This behaviour +can be changed in the future, so use 'mode' argument when +creating IPDB instances. + +The sample session with explicit transactions:: + + In [1]: from pyroute2 import IPDB + In [2]: ip = IPDB(mode='explicit') + In [3]: ifdb = ip.interfaces + In [4]: ifdb.tap0.begin() + Out[3]: UUID('7a637a44-8935-4395-b5e7-0ce40d31d937') + In [5]: ifdb.tap0.up() + In [6]: ifdb.tap0.address = '00:11:22:33:44:55' + In [7]: ifdb.tap0.add_ip('10.0.0.1', 24) + In [8]: ifdb.tap0.add_ip('10.0.0.2', 24) + In [9]: ifdb.tap0.review() + Out[8]: + {'+ipaddr': set([('10.0.0.2', 24), ('10.0.0.1', 24)]), + '-ipaddr': set([]), + 'address': '00:11:22:33:44:55', + 'flags': 4099} + In [10]: ifdb.tap0.commit() + + +Note, that you can `review()` the `current_tx` transaction, +and `commit()` or `drop()` it. Also, multiple transactions +are supported, use uuid returned by `begin()` to identify +them. + +Actually, the form like 'ip.tap0.address' is an eye-candy. +The IPDB objects are dictionaries, so you can write the code +above as that:: + + ipdb.interfaces['tap0'].down() + ipdb.interfaces['tap0']['address'] = '00:11:22:33:44:55' + ... + +Context managers +---------------- + +Transactional objects (interfaces, routes) can act as context +managers in the same way as IPDB does itself:: + + with ipdb.interfaces.tap0 as i: + i.address = '00:11:22:33:44:55' + i.ifname = 'vpn' + i.add_ip('10.0.0.1', 24) + i.add_ip('10.0.0.1', 24) + +On exit, the context manager will automatically `commit()` +the transaction. + +Read-only interface views +------------------------- + +Using an interface as a context manager **will** start a +transaction. Sometimes it is not what one needs. To avoid +unnecessary transactions, and to avoid the risk to occasionally +change interface attributes, one can use read-only views:: + + with ipdb.interfaces[1].ro as iface: + print(iface.ifname) + print(iface.address) + +The `.ro` view neither starts transactions, nor allows to +change anything, raising the `RuntimeError` exception. + +The same read-only views are available for routes and rules. + +Create interfaces +----------------- + +IPDB can also create virtual interfaces:: + + with ipdb.create(kind='bridge', ifname='control') as i: + i.add_port(ip.interfaces.eth1) + i.add_port(ip.interfaces.eth2) + i.add_ip('10.0.0.1/24') + + +The `IPDB.create()` call has the same syntax as +`IPRoute.link('add', ...)`, except you shouldn't specify +the `'add'` command. Refer to `IPRoute` docs for details. + +Please notice, that the interface object stays in the database +even if there was an error during the interface creation. It is +done so to make it possible to fix the interface object and try +to run `commit()` again. Or you can drop the interface object +with the `.remove().commit()` call. + +IP address management +--------------------- + +IP addresses on interfaces may be managed using `add_ip()` and +`del_ip()`:: + + with ipdb.interfaces['eth0'] as eth: + eth.add_ip('10.0.0.1/24') + eth.add_ip('10.0.0.2/24') + eth.add_ip('2001:4c8:1023:108::39/64') + eth.del_ip('172.16.12.5/24') + +The address format may be either a string with `'address/mask'` +notation, or a pair of `'address', mask`:: + + with ipdb.interfaces['eth0'] as eth: + eth.add_ip('10.0.0.1', 24) + eth.del_ip('172.16.12.5', 24) + +The `ipaddr` attribute contains all the IP addresses of the +interface, which are accessible in different ways. Getting an +iterator from `ipaddr` gives you a sequence of tuples +`('address', mask)`: + +.. doctest:: + :skipif: True + + >>> for addr in ipdb.interfaces['eth0'].ipaddr: + ... print(ipaddr) + ... + ('10.0.0.2', 24) + ('10.0.0.1', 24) + +Getting one IP from `ipaddr` returns a dict object with full spec: + +.. doctest:: + :skipif: True + + >>> ipdb.interfaces['eth0'].ipaddr[0] + {'family': 2, + 'broadcast': None, + 'flags': 128, + 'address': '10.0.0.2', + 'prefixlen': 24, + 'local': '10.0.0.2'} + + >>> ipdb.intefaces['eth0'].ipaddr['10.0.0.2/24'] + {'family': 2, + 'broadcast': None, + 'flags': 128, + 'address': '10.0.0.2', + 'prefixlen': 24, + 'local': '10.0.0.2'} + +The API is a bit weird, but it's because of historical reasons. In +the future it may be changed. + +Another feature of the `ipaddr` attribute is views: + +.. doctest:: + :skipif: True + + >>> ipdb.interfaces['eth0'].ipaddr.ipv4: + (('10.0.0.2', 24), ('10.0.0.1', 24)) + >>> ipdb.interfaces['eth0'].ipaddr.ipv6: + (('2001:4c8:1023:108::39', 64),) + +The views, as well as the `ipaddr` attribute itself are not supposed +to be changed by user, but only by the internal API. + +Bridge interfaces +----------------- + +Modern kernels provide possibility to manage bridge +interface properties such as STP, forward delay, ageing +time etc. Names of these properties start with `br_`, like +`br_ageing_time`, `br_forward_delay` e.g.:: + + [x for x in dir(ipdb.interfaces.virbr0) if x.startswith('br_')] + +Bridge ports +------------ + +IPDB supports specific bridge port parameters, such as proxyarp, +unicast/multicast flood, cost etc.:: + + with ipdb.interfaces['br-port0'] as p: + p.brport_cost = 200 + p.brport_unicast_flood = 0 + p.brport_proxyarp = 0 + +Ports management +---------------- + +IPDB provides a uniform API to manage bridge, bond and vrf ports:: + + with ipdb.interfaces['br-int'] as br: + br.add_port('veth0') + br.add_port(ipdb.interfaces.veth1) + br.add_port(700) + br.del_port('veth2') + +Both `add_port()` and `del_port()` accept three types of arguments: + + * `'veth0'` -- interface name as a string + * `ipdb.interfaces.veth1` -- IPDB interface object + * `700` -- interface index, an integer + +Routes management +----------------- + +IPDB has a simple yet useful routing management interface. + +Create a route +~~~~~~~~~~~~~~ + +To add a route, there is an easy to use syntax:: + + # spec as a dictionary + spec = {'dst': '172.16.1.0/24', + 'oif': 4, + 'gateway': '192.168.122.60', + 'metrics': {'mtu': 1400, + 'advmss': 500}} + + # pass spec as is + ipdb.routes.add(spec).commit() + + # pass spec as kwargs + ipdb.routes.add(**spec).commit() + + # use keyword arguments explicitly + ipdb.routes.add(dst='172.16.1.0/24', oif=4, ...).commit() + +Please notice, that the device can be specified with `oif` +(output interface) or `iif` (input interface), the `device` +keyword is not supported anymore. + +More examples:: + + # specify table and priority + (ipdb.routes + .add(dst='172.16.1.0/24', + gateway='192.168.0.1', + table=100, + priority=10) + .commit()) + +The `priority` field is what the `iproute2` utility calls +`metric` -- see also below. + +Get a route +~~~~~~~~~~~ + +To access and change the routes, one can use notations as +follows:: + + # default table (254) + # + # change the route gateway and mtu + # + with ipdb.routes['172.16.1.0/24'] as route: + route.gateway = '192.168.122.60' + route.metrics.mtu = 1500 + + # access the default route + print(ipdb.routes['default']) + + # change the default gateway + with ipdb.routes['default'] as route: + route.gateway = '10.0.0.1' + +By default, the path `ipdb.routes` reflects only the main +routing table (254). But Linux supports much more routing +tables, so does IPDB:: + + In [1]: ipdb.routes.tables.keys() + Out[1]: [0, 254, 255] + + In [2]: len(ipdb.routes.tables[255]) + Out[2]: 11 # => 11 automatic routes in the table local + +It is important to understand, that routing tables keys in +IPDB are not only the destination prefix. The key consists +of 'prefix/mask' string and the route priority (if any):: + + In [1]: ipdb.routes.tables[254].idx.keys() + Out[1]: + [RouteKey(dst='default', table=254, family=2, ...), + RouteKey(dst='172.17.0.0/16', table=254, ...), + RouteKey(dst='172.16.254.0/24', table=254, ...), + RouteKey(dst='192.168.122.0/24', table=254, ...), + RouteKey(dst='fe80::/64', table=254, family=10, ...)] + +But a routing table in IPDB allows several variants of the +route spec. The simplest case is to retrieve a route by +prefix, if there is only one match:: + + # get route by prefix + ipdb.routes['172.16.1.0/24'] + + # get route by a special name + ipdb.routes['default'] + +If there are more than one route that matches the spec, only +the first one will be retrieved. One should iterate all the +records and filter by a key to retrieve all matches:: + + # only one route will be retrieved + ipdb.routes['fe80::/64'] + + # get all routes by this prefix + [ x for x in ipdb.routes if x['dst'] == 'fe80::/64' ] + +It is also possible to use dicts as specs:: + + # get IPv4 default route + ipdb.routes[{'dst': 'default', 'family': AF_INET}] + + # get IPv6 default route + ipdb.routes[{'dst': 'default', 'family': AF_INET6}] + + # get route by priority + ipdb.routes.table[100][{'dst': '10.0.0.0/24', 'priority': 10}] + +While this notation returns one route, there is a method to get +all the routes matching the spec:: + + # get all the routes from all the tables via some interface + ipdb.routes.filter({'oif': idx}) + + # get all IPv6 routes from some table + ipdb.routes.table[tnum].filter({'family': AF_INET6}) + +Route metrics +~~~~~~~~~~~~~ + +A special object is dedicated to route metrics, one can +access it via `route.metrics` or `route['metrics']`:: + + # these two statements are equal: + with ipdb.routes['172.16.1.0/24'] as route: + route['metrics']['mtu'] = 1400 + + with ipdb.routes['172.16.1.0/24'] as route: + route.metrics.mtu = 1400 + +Possible metrics are defined in `rtmsg.py:rtmsg.metrics`, +e.g. `RTAX_HOPLIMIT` means `hoplimit` metric etc. + +Multipath routing +~~~~~~~~~~~~~~~~~ + +Multipath nexthops are managed via `route.add_nh()` and +`route.del_nh()` methods. They are available to review via +`route.multipath`, but one should not directly +add/remove/modify nexthops in `route.multipath`, as the +changes will not be committed correctly. + +To create a multipath route:: + + ipdb.routes.add({'dst': '172.16.232.0/24', + 'multipath': [{'gateway': '172.16.231.2', + 'hops': 2}, + {'gateway': '172.16.231.3', + 'hops': 1}, + {'gateway': '172.16.231.4'}]}).commit() + +To change a multipath route:: + + with ipdb.routes['172.16.232.0/24'] as r: + r.add_nh({'gateway': '172.16.231.5'}) + r.del_nh({'gateway': '172.16.231.4'}) + +Another possible way is to create a normal route and turn +it into multipath by `add_nh()`:: + + # create a non-MP route with one gateway: + (ipdb + .routes + .add({'dst': '172.16.232.0/24', + 'gateway': '172.16.231.2'}) + .commit()) + + # turn it to become a MP route: + (ipdb + .routes['172.16.232.0/24'] + .add_nh({'gateway': '172.16.231.3'}) + .commit()) + + # here the route will contain two NH records, with + # gateways 172.16.231.2 and 172.16.231.3 + + # remove one NH and turn the route to be a normal one + (ipdb + .routes['172.16.232.0/24'] + .del_nh({'gateway': '172.16.231.2'}) + .commit()) + + # thereafter the traffic to 172.16.232.0/24 will go only + # via 172.16.231.3 + +Differences from the iproute2 syntax +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By historical reasons, `iproute2` uses names that differs +from what the kernel uses. E.g., `iproute2` uses `weight` +for multipath route hops instead of `hops`, where +`weight == (hops + 1)`. Thus, a route created with +`hops == 2` will be listed by `iproute2` as `weight 3`. + +Another significant difference is `metrics`. The `pyroute2` +library uses the kernel naming scheme, where `metrics` means +mtu, rtt, window etc. The `iproute2` utility uses `metric` +(not `metrics`) as a name for the `priority` field. + +In examples:: + + # ------------------------------------------------------- + # iproute2 command: + $ ip route add default \\ + nexthop via 172.16.0.1 weight 2 \\ + nexthop via 172.16.0.2 weight 9 + + # pyroute2 code: + (ipdb + .routes + .add({'dst': 'default', + 'multipath': [{'gateway': '172.16.0.1', 'hops': 1}, + {'gateway': '172.16.0.2', 'hops': 8}]) + .commit()) + + # ------------------------------------------------------- + # iproute2 command: + $ ip route add default via 172.16.0.2 metric 200 + + # pyroute2 code: + (ipdb + .routes + .add({'dst': 'default', + 'gateway': '172.16.0.2', + 'priority': 200}) + .commit()) + + # ------------------------------------------------------- + # iproute2 command: + $ ip route add default via 172.16.0.2 mtu 1460 + + # pyroute2 code: + (ipdb + .routes + .add({'dst': 'default', + 'gateway': '172.16.0.2', + 'metrics': {'mtu': 1460}}) + .commit()) + +Multipath default routes +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. warning:: + As of the merge of kill_rtcache into the kernel, and it's + release in ~3.6, weighted default routes no longer work + in Linux. + +Please refer to +https://github.com/svinota/pyroute2/issues/171#issuecomment-149297244 +for details. + +Rules management +---------------- + +IPDB provides a basic IP rules management system. + +Create a rule +~~~~~~~~~~~~~ + +Syntax is almost the same as for routes:: + + # rule spec + spec = {'src': '172.16.1.0/24', + 'table': 200, + 'priority': 15000} + + ipdb.rules.add(spec).commit() + +Get a rule +~~~~~~~~~~ + +The way IPDB handles IP rules is almost the same as routes, +but rule keys are more complicated -- the Linux kernel +doesn't use keys for rules, but instead iterates all the +records until the first one w/o any attribute mismatch. + +The fields that the kernel uses to compare rules, IPDB uses +as the key fields (see `pyroute2/ipdb/rule.py:RuleKey`) + +There are also more ways to find a record, as with routes:: + + # 1. iterate all the records + for record in ipdb.rules: + match(record) + + # 2. an integer as the key matches the first + # rule with that priority + ipdb.rules[32565] + + # 3. a dict as the key returns the first match + # for all the specified attrs + ipdb.rules[{'dst': '10.0.0.0/24', 'table': 200}] + +Priorities +~~~~~~~~~~ + +Thus, the rule priority is **not** a key, neither in the +kernel, nor in IPDB. One should **not** rely on priorities +as on keys, there may be several rules with the same +priority, and it often happens, e.g. on Android systems. + +Persistence +~~~~~~~~~~~ + +There is no *change* operation for the rule records in the +kernel, so only *add/del* work. When IPDB changes a record, +it effectively deletes the old one and creates the new with +new parameters, but the object, referring the record, stays +the same. Also that means, that IPDB can not recognize the +situation, when someone else does the same. So if there is +another program changing records by *del/add* operations, +even another IPDB instance, referring objects in the IPDB +will be recreated. + +Performance issues +------------------ + +In the case of bursts of Netlink broadcast messages, all +the activity of the pyroute2-based code in the async mode +becomes suppressed to leave more CPU resources to the +packet reader thread. So please be ready to cope with +delays in the case of Netlink broadcast storms. It means +also, that IPDB state will be synchronized with OS also +after some delay. + +The class API +------------- +''' +import atexit +import logging +import sys +import threading +import traceback +import warnings +import weakref + +try: + import queue +except ImportError: + import Queue as queue # The module is called 'Queue' in Python2 +# prepare to deprecate the module +# import warnings +from functools import partial +from pprint import pprint + +from pyroute2 import config +from pyroute2.common import basestring, uuid32 +from pyroute2.ipdb import interfaces, routes, rules +from pyroute2.ipdb.exceptions import ShutdownException +from pyroute2.ipdb.linkedset import IPaddrSet, SortedIPaddrSet +from pyroute2.ipdb.routes import BaseRoute +from pyroute2.ipdb.transactional import SYNC_TIMEOUT +from pyroute2.ipdb.utils import test_reachable_icmp +from pyroute2.iproute import IPRoute +from pyroute2.netlink.rtnl import RTM_GETLINK, RTMGRP_DEFAULTS +from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg + +log = logging.getLogger(__name__) + + +class Watchdog(object): + def __init__(self, ipdb, action, kwarg): + self.event = threading.Event() + self.is_set = False + self.ipdb = ipdb + + def cb(ipdb, msg, _action): + if _action != action: + return + + for key in kwarg: + if (msg.get(key, None) != kwarg[key]) and ( + msg.get_attr(msg.name2nla(key)) != kwarg[key] + ): + return + + self.is_set = True + self.event.set() + + self.cb = cb + # register callback prior to other things + self.uuid = self.ipdb.register_callback(self.cb) + + def wait(self, timeout=SYNC_TIMEOUT): + ret = self.event.wait(timeout=timeout) + self.cancel() + return ret + + def cancel(self): + self.ipdb.unregister_callback(self.uuid) + + +class _evq_context(object): + ''' + Context manager class for the event queue used by the event loop + ''' + + def __init__(self, ipdb, qsize, block, timeout): + self._ipdb = ipdb + self._qsize = qsize + self._block = block + self._timeout = timeout + + def __enter__(self): + # Context manager protocol + self._ipdb._evq_lock.acquire() + self._ipdb._evq = queue.Queue(maxsize=self._qsize) + self._ipdb._evq_drop = 0 + return self + + def __exit__(self, exc_type, exc_value, traceback): + # Context manager protocol + self._ipdb._evq = None + self._ipdb._evq_drop = 0 + self._ipdb._evq_lock.release() + + def __iter__(self): + # Iterator protocol + if not self._ipdb._evq: + raise RuntimeError( + 'eventqueue must be used ' 'as a context manager' + ) + return self + + def next(self): + # Iterator protocol -- Python 2.x compatibility + return self.__next__() + + def __next__(self): + # Iterator protocol -- Python 3.x + msg = self._ipdb._evq.get(self._block, self._timeout) + self._ipdb._evq.task_done() + if isinstance(msg, Exception): + raise msg + return msg + + +class IPDB(object): + ''' + The class that maintains information about network setup + of the host. Monitoring netlink events allows it to react + immediately. It uses no polling. + ''' + + def __init__( + self, + nl=None, + mode='implicit', + restart_on_error=None, + nl_async=None, + sndbuf=1048576, + rcvbuf=1048576, + nl_bind_groups=RTMGRP_DEFAULTS, + ignore_rtables=None, + callbacks=None, + sort_addresses=False, + plugins=None, + deprecation_warning=True, + ): + msg = 'https://docs.pyroute2.org/ipdb_toc.html' + log.warning('Deprecation warning ' + msg) + if deprecation_warning: + log.warning( + 'To remove this DeprecationWarning exception, ' + 'start IPDB(deprecation_warning=False, ...)' + ) + warnings.warn( + 'IPDB module is deprecated and will be removed in 0.7.1', + DeprecationWarning, + ) + plugins = plugins or ['interfaces', 'routes', 'rules'] + pmap = {'interfaces': interfaces, 'routes': routes, 'rules': rules} + self.mode = mode + self.txdrop = False + self._stdout = sys.stdout + self._ipaddr_set = SortedIPaddrSet if sort_addresses else IPaddrSet + self._event_map = {} + self._deferred = {} + self._ensure = [] + self._loaded = set() + self._mthread = None + self._nl_own = nl is None + self._nl_async = config.ipdb_nl_async if nl_async is None else True + self.mnl = None + self.nl = nl + self._sndbuf = sndbuf + self._rcvbuf = rcvbuf + self.nl_bind_groups = nl_bind_groups + self._plugins = [pmap[x] for x in plugins if x in pmap] + if isinstance(ignore_rtables, int): + self._ignore_rtables = [ignore_rtables] + elif isinstance(ignore_rtables, (list, tuple, set)): + self._ignore_rtables = ignore_rtables + else: + self._ignore_rtables = [] + self._stop = False + # see also 'register_callback' + self._post_callbacks = {} + self._pre_callbacks = {} + + # local event queues + # - callbacks event queue + self._cbq = queue.Queue(maxsize=8192) + self._cbq_drop = 0 + # - users event queue + self._evq = None + self._evq_lock = threading.Lock() + self._evq_drop = 0 + + # locks and events + self.exclusive = threading.RLock() + self._shutdown_lock = threading.Lock() + + # register callbacks + # + # examples:: + # def cb1(ipdb, msg, event): + # print(event, msg) + # def cb2(...): + # ... + # + # # default mode: post + # IPDB(callbacks=[cb1, cb2]) + # # specify the mode explicitly + # IPDB(callbacks=[(cb1, 'pre'), (cb2, 'post')]) + # + for cba in callbacks or []: + if not isinstance(cba, (tuple, list, set)): + cba = (cba,) + self.register_callback(*cba) + + # load information + self.restart_on_error = ( + restart_on_error if restart_on_error is not None else nl is None + ) + + # init the database + self.initdb() + + # init the dir() cache + self.__dir_cache__ = [ + i for i in self.__class__.__dict__.keys() if i[0] != '_' + ] + self.__dir_cache__.extend(list(self._deferred.keys())) + + def cleanup(ref): + ipdb_obj = ref() + if (ipdb_obj is not None) and (not ipdb_obj._stop): + ipdb_obj.release() + + atexit.register(cleanup, weakref.ref(self)) + + def __dir__(self): + return self.__dir_cache__ + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.release() + + def _flush_db(self): + def flush(idx): + for key in tuple(idx.keys()): + try: + del idx[key] + except KeyError: + pass + + idx_list = [] + if 'interfaces' in self._loaded: + for key, dev in self.by_name.items(): + try: + # FIXME + self.interfaces._detach(key, dev['index'], dev.nlmsg) + except KeyError: + pass + idx_list.append(self.ipaddr) + idx_list.append(self.neighbours) + if 'routes' in self._loaded: + idx_list.extend( + [self.routes.tables[x] for x in self.routes.tables.keys()] + ) + if 'rules' in self._loaded: + idx_list.append(self.rules) + for idx in idx_list: + flush(idx) + + def initdb(self): + # flush all the DB objects + with self.exclusive: + # explicitly cleanup object references + for event in tuple(self._event_map): + del self._event_map[event] + + self._flush_db() + + # if the command socket is not provided, create it + if self._nl_own: + if self.nl is not None: + self.nl.close() + self.nl = IPRoute( + sndbuf=self._sndbuf, rcvbuf=self._rcvbuf, async_qsize=0 + ) # OBS: legacy design + # setup monitoring socket + if self.mnl is not None: + self._flush_mnl() + self.mnl.close() + self.mnl = self.nl.clone() + try: + self.mnl.bind( + groups=self.nl_bind_groups, async_cache=self._nl_async + ) + except: + self.mnl.close() + if self._nl_own is None: + self.nl.close() + raise + + # explicitly cleanup references + for key in tuple(self._deferred): + del self._deferred[key] + + for module in self._plugins: + if (module.groups & self.nl_bind_groups) != module.groups: + continue + for plugin in module.spec: + self._deferred[plugin['name']] = module.spec + if plugin['name'] in self._loaded: + delattr(self, plugin['name']) + self._loaded.remove(plugin['name']) + + # start service threads + for tspec in ( + ('_mthread', '_serve_main', 'IPDB main event loop'), + ('_cthread', '_serve_cb', 'IPDB cb event loop'), + ): + tg = getattr(self, tspec[0], None) + if not getattr(tg, 'is_alive', lambda: False)(): + tx = threading.Thread( + name=tspec[2], target=getattr(self, tspec[1]) + ) + setattr(self, tspec[0], tx) + tx.daemon = True + tx.start() + + def __getattribute__(self, name): + deferred = super(IPDB, self).__getattribute__('_deferred') + if name in deferred: + register = [] + spec = deferred[name] + for plugin in spec: + obj = plugin['class'](self, **plugin['kwarg']) + setattr(self, plugin['name'], obj) + register.append(obj) + self._loaded.add(plugin['name']) + del deferred[plugin['name']] + for obj in register: + if hasattr(obj, '_register'): + obj._register() + if hasattr(obj, '_event_map'): + for event in obj._event_map: + if event not in self._event_map: + self._event_map[event] = [] + self._event_map[event].append(obj._event_map[event]) + return super(IPDB, self).__getattribute__(name) + + def register_callback(self, callback, mode='post'): + ''' + IPDB callbacks are routines executed on a RT netlink + message arrival. There are two types of callbacks: + "post" and "pre" callbacks. + + ... + + "Post" callbacks are executed after the message is + processed by IPDB and all corresponding objects are + created or deleted. Using ipdb reference in "post" + callbacks you will access the most up-to-date state + of the IP database. + + "Post" callbacks are executed asynchronously in + separate threads. These threads can work as long + as you want them to. Callback threads are joined + occasionally, so for a short time there can exist + stopped threads. + + ... + + "Pre" callbacks are synchronous routines, executed + before the message gets processed by IPDB. It gives + you the way to patch arriving messages, but also + places a restriction: until the callback exits, the + main event IPDB loop is blocked. + + Normally, only "post" callbacks are required. But in + some specific cases "pre" also can be useful. + + ... + + The routine, `register_callback()`, takes two arguments: + - callback function + - mode (optional, default="post") + + The callback should be a routine, that accepts three + arguments:: + + cb(ipdb, msg, action) + + Arguments are: + + - **ipdb** is a reference to IPDB instance, that invokes + the callback. + - **msg** is a message arrived + - **action** is just a msg['event'] field + + E.g., to work on a new interface, you should catch + action == 'RTM_NEWLINK' and with the interface index + (arrived in msg['index']) get it from IPDB:: + + index = msg['index'] + interface = ipdb.interfaces[index] + ''' + lock = threading.Lock() + + def safe(*argv, **kwarg): + with lock: + callback(*argv, **kwarg) + + safe.hook = callback + safe.lock = lock + safe.uuid = uuid32() + + if mode == 'post': + self._post_callbacks[safe.uuid] = safe + elif mode == 'pre': + self._pre_callbacks[safe.uuid] = safe + else: + raise KeyError('Unknown callback mode') + return safe.uuid + + def unregister_callback(self, cuid, mode='post'): + if mode == 'post': + cbchain = self._post_callbacks + elif mode == 'pre': + cbchain = self._pre_callbacks + else: + raise KeyError('Unknown callback mode') + safe = cbchain[cuid] + with safe.lock: + ret = cbchain.pop(cuid) + return ret + + def eventqueue(self, qsize=8192, block=True, timeout=None): + ''' + Initializes event queue and returns event queue context manager. + Once the context manager is initialized, events start to be collected, + so it is possible to read initial state from the system without losing + last moment changes, and once that is done, start processing events. + + Example:: + + ipdb = IPDB() + with ipdb.eventqueue() as evq: + my_state = ipdb.... + for msg in evq: + update_state_by_msg(my_state, msg) + ''' + return _evq_context(self, qsize, block, timeout) + + def eventloop(self, qsize=8192, block=True, timeout=None): + """ + Event generator for simple cases when there is no need for initial + state setup. Initialize event queue and yield events as they happen. + """ + with self.eventqueue(qsize=qsize, block=block, timeout=timeout) as evq: + for msg in evq: + yield msg + + def release(self): + ''' + Shutdown IPDB instance and sync the state. Since + IPDB is asyncronous, some operations continue in the + background, e.g. callbacks. So, prior to exit the + script, it is required to properly shutdown IPDB. + + The shutdown sequence is not forced in an interactive + python session, since it is easier for users and there + is enough time to sync the state. But for the scripts + the `release()` call is required. + ''' + with self._shutdown_lock: + if self._stop: + log.warning("shutdown in progress") + return + self._stop = True + self._cbq.put(ShutdownException("shutdown")) + + if self._mthread is not None: + self._flush_mnl() + self._mthread.join() + + if self.mnl is not None: + self.mnl.close() + self.mnl = None + + if self._nl_own: + self.nl.close() + self.nl = None + + self._flush_db() + + def _flush_mnl(self): + if self.mnl is not None: + # terminate the main loop + for t in range(3): + try: + msg = ifinfmsg() + msg['index'] = 1 + msg.reset() + self.mnl.put(msg, RTM_GETLINK) + except Exception as e: + log.error("shutdown error: %s", e) + # Just give up. + # We can not handle this case + + def create(self, kind, ifname, reuse=False, **kwarg): + return self.interfaces.add(kind, ifname, reuse, **kwarg) + + def ensure(self, cmd='add', reachable=None, condition=None): + if cmd == 'reset': + self._ensure = [] + elif cmd == 'run': + for f in self._ensure: + f() + elif cmd == 'add': + if isinstance(reachable, basestring): + reachable = reachable.split(':') + if len(reachable) == 1: + f = partial(test_reachable_icmp, reachable[0]) + else: + raise NotImplementedError() + self._ensure.append(f) + else: + if sys.stdin.isatty(): + pprint(self._ensure, stream=self._stdout) + elif cmd == 'print': + pprint(self._ensure, stream=self._stdout) + elif cmd == 'get': + return self._ensure + else: + raise NotImplementedError() + + def items(self): + # TODO: add support for filters? + + # iterate interfaces + for ifname in getattr(self, 'by_name', {}): + yield (('interfaces', ifname), self.interfaces[ifname]) + + # iterate routes + for table in getattr(getattr(self, 'routes', None), 'tables', {}): + for key, route in self.routes.tables[table].items(): + yield (('routes', table, key), route) + + def dump(self): + ret = {} + for key, obj in self.items(): + ptr = ret + for step in key[:-1]: + if step not in ptr: + ptr[step] = {} + ptr = ptr[step] + ptr[key[-1]] = obj + return ret + + def load(self, config, ptr=None): + if ptr is None: + ptr = self + + for key in config: + obj = getattr(ptr, key, None) + if obj is not None: + if hasattr(obj, 'load'): + obj.load(config[key]) + else: + self.load(config[key], ptr=obj) + elif hasattr(ptr, 'add'): + ptr.add(**config[key]) + + return self + + def review(self): + ret = {} + for key, obj in self.items(): + ptr = ret + try: + rev = obj.review() + except TypeError: + continue + + for step in key[:-1]: + if step not in ptr: + ptr[step] = {} + ptr = ptr[step] + ptr[key[-1]] = rev + + if not ret: + raise TypeError('no transaction started') + return ret + + def drop(self): + ok = False + for key, obj in self.items(): + try: + obj.drop() + except TypeError: + continue + ok = True + if not ok: + raise TypeError('no transaction started') + + def commit(self, transactions=None, phase=1): + # what to commit: either from transactions argument, or from + # started transactions on existing objects + if transactions is None: + # collect interface transactions + txlist = [ + (x, x.current_tx) + for x in getattr(self, 'by_name', {}).values() + if x.local_tx.values() + ] + # collect route transactions + for table in getattr( + getattr(self, 'routes', None), 'tables', {} + ).keys(): + txlist.extend( + [ + (x, x.current_tx) + for x in self.routes.tables[table] + if x.local_tx.values() + ] + ) + transactions = txlist + + snapshots = [] + removed = [] + + tx_ipdb_prio = [] + tx_main = [] + tx_prio1 = [] + tx_prio2 = [] + tx_prio3 = [] + for target, tx in transactions: + # 8<------------------------------ + # first -- explicit priorities + if tx['ipdb_priority']: + tx_ipdb_prio.append((target, tx)) + continue + # 8<------------------------------ + # routes + if isinstance(target, BaseRoute): + tx_prio3.append((target, tx)) + continue + # 8<------------------------------ + # intefaces + kind = target.get('kind', None) + if kind in ( + 'vlan', + 'vxlan', + 'gre', + 'tuntap', + 'vti', + 'vti6', + 'vrf', + 'xfrm', + ): + tx_prio1.append((target, tx)) + elif kind in ('bridge', 'bond'): + tx_prio2.append((target, tx)) + else: + tx_main.append((target, tx)) + # 8<------------------------------ + + # explicitly sorted transactions + tx_ipdb_prio = sorted( + tx_ipdb_prio, key=lambda x: x[1]['ipdb_priority'], reverse=True + ) + + # FIXME: this should be documented + # + # The final transactions order: + # 1. any txs with ipdb_priority (sorted by that field) + # + # Then come default priorities (no ipdb_priority specified): + # 2. all the rest + # 3. vlan, vxlan, gre, tuntap, vti, vrf + # 4. bridge, bond + # 5. routes + transactions = tx_ipdb_prio + tx_main + tx_prio1 + tx_prio2 + tx_prio3 + + try: + for target, tx in transactions: + if target['ipdb_scope'] == 'detached': + continue + if tx['ipdb_scope'] == 'remove': + tx['ipdb_scope'] = 'shadow' + removed.append((target, tx)) + if phase == 1: + s = (target, target.pick(detached=True)) + snapshots.append(s) + # apply the changes, but NO rollback -- only phase 1 + target.commit( + transaction=tx, commit_phase=phase, commit_mask=phase + ) + # if the commit above fails, the next code + # branch will run rollbacks + except Exception: + if phase == 1: + # run rollbacks for ALL the collected transactions, + # even successful ones + self.fallen = transactions + txs = filter( + lambda x: not ( + 'create' == x[0]['ipdb_scope'] == x[1]['ipdb_scope'] + ), + snapshots, + ) + self.commit(transactions=txs, phase=2) + raise + else: + if phase == 1: + for target, tx in removed: + target['ipdb_scope'] = 'detached' + target.detach() + finally: + if phase == 1: + for target, tx in transactions: + target.drop(tx.uid) + + return self + + def watchdog(self, wdops='RTM_NEWLINK', **kwarg): + return Watchdog(self, wdops, kwarg) + + def _serve_cb(self): + ### + # Callbacks thread working on a dedicated event queue. + ### + + while not self._stop: + msg = self._cbq.get() + self._cbq.task_done() + if isinstance(msg, ShutdownException): + return + elif isinstance(msg, Exception): + raise msg + for cb in tuple(self._post_callbacks.values()): + try: + cb(self, msg, msg['event']) + except: + pass + + def _serve_main(self): + ### + # Main monitoring cycle. It gets messages from the + # default iproute queue and updates objects in the + # database. + ### + + while not self._stop: + try: + messages = self.mnl.get() + ## + # Check it again + # + # NOTE: one should not run callbacks or + # anything like that after setting the + # _stop flag, since IPDB is not valid + # anymore + if self._stop: + break + except Exception as e: + with self.exclusive: + if self._evq: + self._evq.put(e) + return + if self.restart_on_error: + log.error( + 'Restarting IPDB instance after ' 'error:\n%s', + traceback.format_exc(), + ) + try: + self.initdb() + except: + log.error( + 'Error restarting DB:\n%s', traceback.format_exc() + ) + return + continue + else: + log.error('Emergency shutdown, cleanup manually') + raise RuntimeError('Emergency shutdown') + + for msg in messages: + # Run pre-callbacks + # NOTE: pre-callbacks are synchronous + for cuid, cb in tuple(self._pre_callbacks.items()): + try: + cb(self, msg, msg['event']) + except: + pass + + with self.exclusive: + event = msg.get('event', None) + if event in self._event_map: + for func in self._event_map[event]: + func(msg) + + # Post-callbacks + try: + self._cbq.put_nowait(msg) + if self._cbq_drop: + log.warning('dropped %d events', self._cbq_drop) + self._cbq_drop = 0 + except queue.Full: + self._cbq_drop += 1 + except Exception: + log.error('Emergency shutdown, cleanup manually') + raise RuntimeError('Emergency shutdown') + + # + # Why not to put these two pieces of the code + # it in a routine? + # + # TODO: run performance tests with routines + + # Users event queue + if self._evq: + try: + self._evq.put_nowait(msg) + if self._evq_drop: + log.warning( + "dropped %d events", self._evq_drop + ) + self._evq_drop = 0 + except queue.Full: + self._evq_drop += 1 + except Exception: + log.error('Emergency shutdown, cleanup manually') + raise RuntimeError('Emergency shutdown') diff --git a/pyroute2/ipdb/routes.py b/pyroute2/ipdb/routes.py new file mode 100644 index 000000000..7eb5fcdfd --- /dev/null +++ b/pyroute2/ipdb/routes.py @@ -0,0 +1,1323 @@ +import logging +import struct +import threading +import time +import traceback +import types +from collections import namedtuple +from socket import AF_INET, AF_INET6, AF_UNSPEC, inet_ntop, inet_pton + +from pyroute2.common import AF_MPLS, basestring +from pyroute2.ipdb.exceptions import CommitException +from pyroute2.ipdb.linkedset import LinkedSet +from pyroute2.ipdb.transactional import ( + SYNC_TIMEOUT, + Transactional, + with_transaction, +) +from pyroute2.netlink import NLM_F_CREATE, NLM_F_MULTI, nlmsg, nlmsg_base, rtnl +from pyroute2.netlink.rtnl import encap_type, rt_proto, rt_type +from pyroute2.netlink.rtnl.ifaddrmsg import IFA_F_SECONDARY +from pyroute2.netlink.rtnl.rtmsg import rtmsg +from pyroute2.requests.main import RequestProcessor +from pyroute2.requests.route import RouteFieldFilter + +log = logging.getLogger(__name__) +groups = ( + rtnl.RTMGRP_IPV4_ROUTE | rtnl.RTMGRP_IPV6_ROUTE | rtnl.RTMGRP_MPLS_ROUTE +) +IP6_RT_PRIO_USER = 1024 + + +class Metrics(Transactional): + _fields = [rtmsg.metrics.nla2name(i[0]) for i in rtmsg.metrics.nla_map] + + +class Encap(Transactional): + _fields = ['type', 'labels'] + + +class Via(Transactional): + _fields = ['family', 'addr'] + + +class NextHopSet(LinkedSet): + def __init__(self, prime=None): + super(NextHopSet, self).__init__() + prime = prime or [] + for v in prime: + self.add(v) + + def __sub__(self, vs): + ret = type(self)() + sub = set(self.raw.keys()) - set(vs.raw.keys()) + for v in sub: + ret.add(self[v], raw=self.raw[v]) + return ret + + def __make_nh(self, prime): + if isinstance(prime, BaseRoute): + return prime.make_nh_key(prime) + elif isinstance(prime, dict): + if prime.get('family', None) == AF_MPLS: + return MPLSRoute.make_nh_key(prime) + else: + return Route.make_nh_key(prime) + elif isinstance(prime, tuple): + return prime + else: + raise TypeError("unknown prime type %s" % type(prime)) + + def __getitem__(self, key): + return self.raw[key] + + def __iter__(self): + def NHIterator(): + for x in tuple(self.raw.values()): + yield x + + return NHIterator() + + def add(self, prime, raw=None, cascade=False): + key = self.__make_nh(prime) + req = key._required + fields = key._fields + skey = key[:req] + (None,) * (len(fields) - req) + if skey in self.raw: + del self.raw[skey] + return super(NextHopSet, self).add(key, raw=prime) + + def remove(self, prime, raw=None, cascade=False): + key = self.__make_nh(prime) + try: + super(NextHopSet, self).remove(key) + except KeyError as e: + req = key._required + fields = key._fields + skey = key[:req] + (None,) * (len(fields) - req) + for rkey in tuple(self.raw.keys()): + if skey == rkey[:req] + (None,) * (len(fields) - req): + break + else: + raise e + super(NextHopSet, self).remove(rkey) + + +class WatchdogMPLSKey(dict): + def __init__(self, route): + dict.__init__(self) + self['oif'] = route['oif'] + self['dst'] = [{'ttl': 0, 'bos': 1, 'tc': 0, 'label': route['dst']}] + + +class WatchdogKey(dict): + ''' + Construct from a route a dictionary that could be used as + a match for IPDB watchdogs. + ''' + + def __init__(self, route): + dict.__init__( + self, + [ + x + for x in RequestProcessor( + RouteFieldFilter(), context=route, prime=route + ).items() + if x[0] + in ( + 'dst', + 'dst_len', + 'src', + 'src_len', + 'tos', + 'priority', + 'gateway', + 'table', + ) + and x[1] + ], + ) + + +# Universal route key +# Holds the fields that the kernel uses to uniquely identify routes. +# IPv4 allows redundant routes with different 'tos' but IPv6 does not, +# so 'tos' is used for IPv4 but not IPv6. +# For reference, see fib_table_insert() in +# https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/ipv4/fib_trie.c#n1147 +# and fib6_add_rt2node() in +# https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/ipv6/ip6_fib.c#n765 +RouteKey = namedtuple( + 'RouteKey', ('dst', 'table', 'family', 'priority', 'tos') +) + +# IP multipath NH key +IPNHKey = namedtuple('IPNHKey', ('gateway', 'encap', 'oif')) +IPNHKey._required = 2 + +# MPLS multipath NH key +MPLSNHKey = namedtuple('MPLSNHKey', ('newdst', 'via', 'oif')) +MPLSNHKey._required = 2 + + +def _normalize_ipaddr(x, y): + if isinstance(y, basestring) and y.find(':') > -1: + y = inet_ntop(AF_INET6, inet_pton(AF_INET6, y)) + return x == y + + +def _normalize_ipnet(x, y): + # + # x -- incoming value + # y -- transaction value + # + if isinstance(y, basestring) and y.find(':') > -1: + s = y.split('/') + ip = inet_ntop(AF_INET6, inet_pton(AF_INET6, s[0])) + if len(s) > 1: + y = '%s/%s' % (ip, s[1]) + else: + y = ip + return x == y + + +class BaseRoute(Transactional): + ''' + Persistent transactional route object + ''' + + _fields = [rtmsg.nla2name(i[0]) for i in rtmsg.nla_map] + for key, _ in rtmsg.fields: + _fields.append(key) + _fields.append('removal') + _virtual_fields = ['ipdb_scope', 'ipdb_priority'] + _fields.extend(_virtual_fields) + _linked_sets = ['multipath'] + _nested = [] + _gctime = None + cleanup = ('attrs', 'header', 'event', 'cacheinfo') + _fields_cmp = { + 'src': _normalize_ipnet, + 'dst': _normalize_ipnet, + 'gateway': _normalize_ipaddr, + 'prefsrc': _normalize_ipaddr, + } + + def __init__(self, ipdb, mode=None, parent=None, uid=None): + Transactional.__init__(self, ipdb, mode, parent, uid) + with self._direct_state: + self['ipdb_priority'] = 0 + + @with_transaction + def add_nh(self, prime): + with self._write_lock: + # if the multipath chain is empty, copy the current + # nexthop as the first in the multipath + if not self['multipath']: + first = {} + for key in ('oif', 'gateway', 'newdst'): + if self[key]: + first[key] = self[key] + if first: + if self['family']: + first['family'] = self['family'] + for key in ('encap', 'via', 'metrics'): + if self[key] and any(self[key].values()): + first[key] = self[key] + self[key] = None + self['multipath'].add(first) + # cleanup key fields + for key in ('oif', 'gateway', 'newdst'): + self[key] = None + # add the prime as NH + if self['family'] == AF_MPLS: + prime['family'] = AF_MPLS + self['multipath'].add(prime) + + @with_transaction + def del_nh(self, prime): + with self._write_lock: + if not self['multipath']: + raise KeyError( + 'attempt to delete nexthop from ' 'non-multipath route' + ) + nh = dict(prime) + if self['family'] == AF_MPLS: + nh['family'] = AF_MPLS + self['multipath'].remove(nh) + + def load_netlink(self, msg): + with self._direct_state: + if self['ipdb_scope'] == 'locked': + # do not touch locked interfaces + return + + self['ipdb_scope'] = 'system' + + # IPv6 multipath via several devices (not networks) is a very + # special case, since we get only the first hop notification. Ask + # the kernel guys why. I've got no idea. + # + # So load all the rest + flags = msg.get('header', {}).get('flags', 0) + family = msg.get('family', 0) + clean_mp = True + table = msg.get_attr('RTA_TABLE') or msg.get('table') + dst = msg.get_attr('RTA_DST') + # + # It MAY be a multipath hop + # + if family == AF_INET6 and not msg.get_attr('RTA_MULTIPATH'): + # + # It is a notification about the route created + # + if flags == NLM_F_CREATE: + # + # This routine can significantly slow down the IPDB + # instance, but I see no way around. Some are born + # to endless night. + # + clean_mp = False + msgs = self.nl.route( + 'show', table=table, dst=dst, family=family + ) + for nhmsg in msgs: + nh = type(self)(ipdb=self.ipdb, parent=self) + nh.load_netlink(nhmsg) + with nh._direct_state: + del nh['dst'] + del nh['ipdb_scope'] + del nh['ipdb_priority'] + del nh['multipath'] + del nh['metrics'] + self.add_nh(nh) + # + # it IS a multipath hop loaded during IPDB init + # + elif flags == NLM_F_MULTI and self.get('dst'): + nh = type(self)(ipdb=self.ipdb, parent=self) + nh.load_netlink(msg) + with nh._direct_state: + del nh['dst'] + del nh['ipdb_scope'] + del nh['ipdb_priority'] + del nh['multipath'] + del nh['metrics'] + self.add_nh(nh) + return + + for key, value in msg.items(): + self[key] = value + + # cleanup multipath NH + if clean_mp: + for nh in self['multipath']: + self.del_nh(nh) + + for cell in msg['attrs']: + # + # Parse on demand + # + norm = rtmsg.nla2name(cell[0]) + if norm in self.cleanup: + continue + value = cell[1] + # normalize RTAX + if norm == 'metrics': + with self['metrics']._direct_state: + for metric in tuple(self['metrics'].keys()): + del self['metrics'][metric] + for rtax, rtax_value in value['attrs']: + rtax_norm = rtmsg.metrics.nla2name(rtax) + self['metrics'][rtax_norm] = rtax_value + elif norm == 'multipath': + for record in value: + nh = type(self)(ipdb=self.ipdb, parent=self) + nh.load_netlink(record) + with nh._direct_state: + del nh['dst'] + del nh['ipdb_scope'] + del nh['ipdb_priority'] + del nh['multipath'] + del nh['metrics'] + self['multipath'].add(nh) + elif norm == 'encap': + with self['encap']._direct_state: + # WIP: should support encap_types other than MPLS + if value.get_attr('MPLS_IPTUNNEL_DST'): + ret = [] + for dst in value.get_attr('MPLS_IPTUNNEL_DST'): + ret.append(str(dst['label'])) + if ret: + self['encap']['labels'] = '/'.join(ret) + elif norm == 'via': + with self['via']._direct_state: + self['via'] = value + elif norm == 'newdst': + self['newdst'] = [x['label'] for x in value] + else: + self[norm] = value + + if msg.get('family', 0) == AF_MPLS: + dst = msg.get_attr('RTA_DST') + if dst: + dst = dst[0]['label'] + else: + if msg.get_attr('RTA_DST'): + dst = '%s/%s' % (msg.get_attr('RTA_DST'), msg['dst_len']) + else: + dst = 'default' + self['dst'] = dst + + # fix RTA_ENCAP_TYPE if needed + if msg.get_attr('RTA_ENCAP'): + if self['encap_type'] is not None: + with self['encap']._direct_state: + self['encap']['type'] = self['encap_type'] + self['encap_type'] = None + # or drop encap, if there is no RTA_ENCAP in msg + elif self['encap'] is not None: + self['encap_type'] = None + with self['encap']._direct_state: + self['encap'] = {} + + # drop metrics, if there is no RTA_METRICS in msg + if not msg.get_attr('RTA_METRICS') and self['metrics'] is not None: + with self['metrics']._direct_state: + self['metrics'] = {} + + # same for via + if not msg.get_attr('RTA_VIA') and self['via'] is not None: + with self['via']._direct_state: + self['via'] = {} + + # one hop -> multihop transition + if not msg.get_attr('RTA_GATEWAY') and self['gateway'] is not None: + self['gateway'] = None + if ( + 'oif' not in msg + and not msg.get_attr('RTA_OIF') + and self['oif'] is not None + ): + self['oif'] = None + + # finally, cleanup all not needed + for item in self.cleanup: + if item in self: + del self[item] + + def commit( + self, tid=None, transaction=None, commit_phase=1, commit_mask=0xFF + ): + if not commit_phase & commit_mask: + return self + + error = None + drop = self.ipdb.txdrop + devop = 'set' + cleanup = [] + # FIXME -- make a debug object + debug = {'traceback': None, 'next_stage': None} + notx = True + + if tid or transaction: + notx = False + + if tid: + transaction = self.global_tx[tid] + else: + transaction = transaction or self.current_tx + + # ignore global rollbacks on invalid routes + if self['ipdb_scope'] == 'create' and commit_phase > 1: + return + + # create a new route + if self['ipdb_scope'] != 'system': + devop = 'add' + + # work on an existing route + snapshot = self.pick() + added, removed = transaction // snapshot + added.pop('ipdb_scope', None) + removed.pop('ipdb_scope', None) + + try: + # route set + if self['family'] != AF_MPLS: + cleanup = [ + any(snapshot['metrics'].values()) + and not any(added.get('metrics', {}).values()), + any(snapshot['encap'].values()) + and not any(added.get('encap', {}).values()), + ] + if ( + any(added.values()) + or any(cleanup) + or removed.get('multipath', None) + or devop == 'add' + ): + # prepare multipath target sync + wlist = [] + if transaction['multipath']: + mplen = len(transaction['multipath']) + if mplen == 1: + # set up local targets + for nh in transaction['multipath']: + for key in ('oif', 'gateway', 'newdst'): + if nh.get(key, None): + self.set_target(key, nh[key]) + wlist.append(key) + mpt = None + else: + + def mpcheck(mpset): + return len(mpset) == mplen + + mpt = self['multipath'].set_target(mpcheck, True) + else: + mpt = None + + # prepare the anchor key to catch *possible* route update + old_key = self.make_key(self) + new_key = self.make_key(transaction) + if old_key != new_key: + # assume we can not move routes between tables (yet ;) + if self['family'] == AF_MPLS: + route_index = self.ipdb.routes.tables['mpls'].idx + else: + route_index = self.ipdb.routes.tables[ + self['table'] or 254 + ].idx + # re-link the route record + if new_key in route_index: + raise CommitException('route idx conflict') + else: + route_index[new_key] = {'key': new_key, 'route': self} + # wipe the old key, if needed + if old_key in route_index: + del route_index[old_key] + self.nl.route(devop, **transaction) + # delete old record, if required + if (old_key != new_key) and (devop == 'set'): + req = dict(old_key._asdict()) + # update the request with the scope. + # + # though the scope isn't a part of the + # key, it is required for the correct + # removal -- only if it is set + req['scope'] = self.get('scope', 0) + self.nl.route('del', **req) + transaction.wait_all_targets() + for key in ('metrics', 'via'): + if transaction[key] and transaction[key]._targets: + transaction[key].wait_all_targets() + if mpt is not None: + mpt.wait(SYNC_TIMEOUT) + if not mpt.is_set(): + raise CommitException('multipath target is not set') + self['multipath'].clear_target(mpt) + for key in wlist: + self.wait_target(key) + # route removal + if (transaction['ipdb_scope'] in ('shadow', 'remove')) or ( + (transaction['ipdb_scope'] == 'create') and commit_phase == 2 + ): + if transaction['ipdb_scope'] == 'shadow': + with self._direct_state: + self['ipdb_scope'] = 'locked' + # create watchdog + wd = self.ipdb.watchdog( + 'RTM_DELROUTE', **self.wd_key(snapshot) + ) + for route in self.nl.route('delete', **snapshot): + self.ipdb.routes.load_netlink(route) + wd.wait() + if transaction['ipdb_scope'] == 'shadow': + with self._direct_state: + self['ipdb_scope'] = 'shadow' + + # success, so it's safe to drop the transaction + drop = True + + except Exception as e: + error = e + # prepare postmortem + debug['traceback'] = traceback.format_exc() + debug['error_stack'] = [] + debug['next_stage'] = None + + if commit_phase == 1: + try: + self.commit( + transaction=snapshot, + commit_phase=2, + commit_mask=commit_mask, + ) + except Exception as i_e: + debug['next_stage'] = i_e + error = RuntimeError() + + if drop and notx: + self.drop(transaction.uid) + + if error is not None: + error.debug = debug + raise error + + self.ipdb.routes.gc() + return self + + def remove(self): + self['ipdb_scope'] = 'remove' + return self + + def shadow(self): + self['ipdb_scope'] = 'shadow' + return self + + def detach(self): + if self.get('family') == AF_MPLS: + table = 'mpls' + else: + table = self.get('table', 254) + del self.ipdb.routes.tables[table][self.make_key(self)] + + +class Route(BaseRoute): + _nested = ['encap', 'metrics'] + wd_key = WatchdogKey + + @classmethod + def make_encap(cls, encap): + ''' + Normalize encap object + ''' + labels = encap.get('labels', None) + if isinstance(labels, (list, tuple, set)): + labels = '/'.join( + map( + lambda x: ( + str(x['label']) if isinstance(x, dict) else str(x) + ), + labels, + ) + ) + if not isinstance(labels, basestring): + raise TypeError('labels struct not supported') + return {'type': encap.get('type', 'mpls'), 'labels': labels} + + @classmethod + def make_nh_key(cls, msg): + ''' + Construct from a netlink message a multipath nexthop key + ''' + values = [] + if isinstance(msg, nlmsg_base): + for field in IPNHKey._fields: + v = msg.get_attr(msg.name2nla(field)) + if field == 'encap': + # 1. encap type + if msg.get_attr('RTA_ENCAP_TYPE') != 1: # FIXME + values.append(None) + continue + # 2. encap_type == 'mpls' + v = '/'.join( + [ + str(x['label']) + for x in v.get_attr('MPLS_IPTUNNEL_DST') + ] + ) + elif v is None: + v = msg.get(field, None) + values.append(v) + elif isinstance(msg, dict): + for field in IPNHKey._fields: + v = msg.get(field, None) + if field == 'encap' and v and v['labels']: + v = v['labels'] + elif (field == 'encap') and ( + len(msg.get('multipath', []) or []) == 1 + ): + v = ( + tuple(msg['multipath'].raw.values())[0] + .get('encap', {}) + .get('labels', None) + ) + elif field == 'encap': + v = None + elif ( + (field == 'gateway') + and (len(msg.get('multipath', []) or []) == 1) + and not v + ): + v = tuple(msg['multipath'].raw.values())[0].get( + 'gateway', None + ) + + if field == 'encap' and isinstance(v, (list, tuple, set)): + v = '/'.join( + map( + lambda x: ( + str(x['label']) + if isinstance(x, dict) + else str(x) + ), + v, + ) + ) + values.append(v) + else: + raise TypeError('prime not supported: %s' % type(msg)) + return IPNHKey(*values) + + @classmethod + def make_key(cls, msg): + ''' + Construct from a netlink message a key that can be used + to locate the route in the table + ''' + values = [] + if isinstance(msg, nlmsg_base): + for field in RouteKey._fields: + v = msg.get_attr(msg.name2nla(field)) + if field == 'dst': + if v is not None: + v = '%s/%s' % (v, msg['dst_len']) + else: + v = 'default' + elif field == 'tos' and msg.get('family') != AF_INET: + # ignore tos field for non-IPv6 routes, + # as it used as a key only there + v = None + elif v is None: + v = msg.get(field, None) + values.append(v) + elif isinstance(msg, dict): + for field in RouteKey._fields: + v = msg.get(field, None) + if ( + field == 'dst' + and isinstance(v, basestring) + and v.find(':') > -1 + ): + v = v.split('/') + ip = inet_ntop(AF_INET6, inet_pton(AF_INET6, v[0])) + if len(v) > 1: + v = '%s/%s' % (ip, v[1]) + else: + v = ip + elif field == 'tos' and msg.get('family') != AF_INET: + # ignore tos field for non-IPv6 routes, + # as it used as a key only there + v = None + values.append(v) + else: + raise TypeError('prime not supported: %s' % type(msg)) + return RouteKey(*values) + + def __setitem__(self, key, value): + ret = value + if (key in ('encap', 'metrics')) and isinstance(value, dict): + # transactionals attach as is + if type(value) in (Encap, Metrics): + with self._direct_state: + return Transactional.__setitem__(self, key, value) + + # check, if it exists already + ret = Transactional.__getitem__(self, key) + # it doesn't + # (plain dict can be safely discarded) + if isinstance(ret, dict) or not ret: + # bake transactionals in place + if key == 'encap': + ret = Encap(parent=self) + elif key == 'metrics': + ret = Metrics(parent=self) + # attach transactional to the route + with self._direct_state: + Transactional.__setitem__(self, key, ret) + # begin() works only if the transactional is attached + if any(value.values()): + if self._mode in ('implicit', 'explicit'): + ret._begin(tid=self.current_tx.uid) + [ + ret.__setitem__(k, v) + for k, v in value.items() + if v is not None + ] + # corresponding transactional exists + else: + # set fields + for k in ret: + ret[k] = value.get(k, None) + return + elif key == 'multipath': + cur = Transactional.__getitem__(self, key) + if isinstance(cur, NextHopSet): + # load entries + vs = NextHopSet(value) + for key in vs - cur: + cur.add(key) + for key in cur - vs: + cur.remove(key) + else: + # drop any result of `update()` + Transactional.__setitem__(self, key, NextHopSet(value)) + return + elif key == 'encap_type' and not isinstance(value, int): + ret = encap_type.get(value, value) + elif key == 'type' and not isinstance(value, int): + ret = rt_type.get(value, value) + elif key == 'proto' and not isinstance(value, int): + ret = rt_proto.get(value, value) + elif ( + key == 'dst' + and isinstance(value, basestring) + and value in ('0.0.0.0/0', '::/0') + ): + ret = 'default' + Transactional.__setitem__(self, key, ret) + + def __getitem__(self, key): + ret = Transactional.__getitem__(self, key) + if (key in ('encap', 'metrics', 'multipath')) and (ret is None): + with self._direct_state: + self[key] = [] if key == 'multipath' else {} + ret = self[key] + return ret + + +class MPLSRoute(BaseRoute): + wd_key = WatchdogMPLSKey + _nested = ['via'] + + @classmethod + def make_nh_key(cls, msg): + ''' + Construct from a netlink message a multipath nexthop key + ''' + return MPLSNHKey( + newdst=tuple(msg['newdst']), + via=msg.get('via', {}).get('addr', None), + oif=msg.get('oif', None), + ) + + @classmethod + def make_key(cls, msg): + ''' + Construct from a netlink message a key that can be used + to locate the route in the table + ''' + ret = None + if isinstance(msg, nlmsg): + ret = msg.get_attr('RTA_DST') + elif isinstance(msg, dict): + ret = msg.get('dst', None) + else: + raise TypeError('prime not supported') + if isinstance(ret, list): + ret = ret[0]['label'] + return ret + + def __setitem__(self, key, value): + if key == 'via' and isinstance(value, dict): + # replace with a new transactional + if isinstance(value, Via): + with self._direct_state: + return BaseRoute.__setitem__(self, key, value) + # or load the dict + ret = BaseRoute.__getitem__(self, key) + if not isinstance(ret, Via): + ret = Via(parent=self) + # attach new transactional -- replace any + # non-Via object (may be a result of update()) + with self._direct_state: + BaseRoute.__setitem__(self, key, ret) + # load value into the new object + if any(value.values()): + if self._mode in ('implicit', 'explicit'): + ret._begin(tid=self.current_tx.uid) + [ + ret.__setitem__(k, v) + for k, v in value.items() + if v is not None + ] + else: + # load value into existing object + for k in ret: + ret[k] = value.get(k, None) + return + elif key == 'multipath': + cur = BaseRoute.__getitem__(self, key) + if isinstance(cur, NextHopSet): + # load entries + vs = NextHopSet(value) + for key in vs - cur: + cur.add(key) + for key in cur - vs: + cur.remove(key) + else: + BaseRoute.__setitem__(self, key, NextHopSet(value)) + else: + BaseRoute.__setitem__(self, key, value) + + def __getitem__(self, key): + with self._direct_state: + ret = BaseRoute.__getitem__(self, key) + if key == 'multipath' and ret is None: + self[key] = [] + ret = self[key] + elif key == 'via' and ret is None: + self[key] = {} + ret = self[key] + return ret + + +class RoutingTable(object): + route_class = Route + + def __init__(self, ipdb, prime=None): + self.ipdb = ipdb + self.lock = threading.Lock() + self.idx = {} + self.kdx = {} + + def __nogc__(self): + return self.filter(lambda x: x['route']['ipdb_scope'] != 'gc') + + def __repr__(self): + return repr([x['route'] for x in self.__nogc__()]) + + def __len__(self): + return len(self.keys()) + + def __iter__(self): + for record in self.__nogc__(): + yield record['route'] + + def gc(self): + now = time.time() + for route in self.filter({'ipdb_scope': 'gc'}): + if now - route['route']._gctime < 2: + continue + try: + if not self.ipdb.nl.route('dump', **route['route']): + raise + with route['route']._direct_state: + route['route']['ipdb_scope'] = 'system' + except: + del self.idx[route['key']] + + def keys(self, key='dst'): + with self.lock: + return [x['route'][key] for x in self.__nogc__()] + + def items(self): + for key in self.keys(): + yield (key, self[key]) + + def filter(self, target, oneshot=False): + # + if isinstance(target, types.FunctionType): + return filter(target, [x for x in tuple(self.idx.values())]) + + if isinstance(target, basestring): + target = {'dst': target} + + if not isinstance(target, dict): + raise TypeError('target type not supported: %s' % type(target)) + + ret = [] + for record in tuple(self.idx.values()): + for key, value in tuple(target.items()): + if (key not in record['route']) or ( + value != record['route'][key] + ): + break + else: + ret.append(record) + if oneshot: + return ret + + return ret + + def describe(self, target, forward=False): + # match the route by index -- a bit meaningless, + # but for compatibility + if isinstance(target, int): + keys = [x['key'] for x in self.__nogc__()] + return self.idx[keys[target]] + + # match the route by key + if isinstance(target, (tuple, list)): + # full match + return self.idx[RouteKey(*target)] + + if isinstance(target, nlmsg): + return self.idx[Route.make_key(target)] + + # match the route by filter + ret = self.filter(target, oneshot=True) + if ret: + return ret[0] + + if not forward: + raise KeyError('record not found') + + # match the route by dict spec + if not isinstance(target, dict): + raise TypeError('lookups can be done only with dict targets') + + # split masks + if target.get('dst', '').find('/') >= 0: + dst = target['dst'].split('/') + target['dst'] = dst[0] + target['dst_len'] = int(dst[1]) + + if target.get('src', '').find('/') >= 0: + src = target['src'].split('/') + target['src'] = src[0] + target['src_len'] = int(src[1]) + + # load and return the route, if exists + route = Route(self.ipdb) + ret = self.ipdb.nl.get_routes(**target) + if not ret: + raise KeyError('record not found') + route.load_netlink(ret[0]) + return {'route': route, 'key': None} + + def __delitem__(self, key): + with self.lock: + item = self.describe(key, forward=False) + del self.idx[self.route_class.make_key(item['route'])] + + def load(self, msg): + key = self.route_class.make_key(msg) + self[key] = msg + return key + + def __setitem__(self, key, value): + with self.lock: + try: + record = self.describe(key, forward=False) + except KeyError: + record = {'route': self.route_class(self.ipdb), 'key': None} + + if isinstance(value, nlmsg): + record['route'].load_netlink(value) + elif isinstance(value, self.route_class): + record['route'] = value + elif isinstance(value, dict): + with record['route']._direct_state: + record['route'].update(value) + + key = self.route_class.make_key(record['route']) + if record['key'] is None: + self.idx[key] = {'route': record['route'], 'key': key} + else: + self.idx[key] = record + if record['key'] != key: + del self.idx[record['key']] + record['key'] = key + + def __getitem__(self, key): + with self.lock: + return self.describe(key, forward=False)['route'] + + def __contains__(self, key): + try: + with self.lock: + self.describe(key, forward=False) + return True + except KeyError: + return False + + +class MPLSTable(RoutingTable): + route_class = MPLSRoute + + def keys(self): + return self.idx.keys() + + def describe(self, target, forward=False): + # match by key + if isinstance(target, int): + return self.idx[target] + + # match by rtmsg + if isinstance(target, rtmsg): + return self.idx[self.route_class.make_key(target)] + + raise KeyError('record not found') + + +class RoutingTableSet(object): + def __init__(self, ipdb): + self.ipdb = ipdb + self._gctime = time.time() + self.ignore_rtables = ipdb._ignore_rtables or [] + self.tables = {254: RoutingTable(self.ipdb)} + self._event_map = { + 'RTM_NEWROUTE': self.load_netlink, + 'RTM_DELROUTE': self.load_netlink, + 'RTM_NEWLINK': self.gc_mark_link, + 'RTM_DELLINK': self.gc_mark_link, + 'RTM_DELADDR': self.gc_mark_addr, + } + + def _register(self): + for msg in self.ipdb.nl.get_routes( + family=AF_INET, match={'family': AF_INET} + ): + self.load_netlink(msg) + for msg in self.ipdb.nl.get_routes( + family=AF_INET6, match={'family': AF_INET6} + ): + self.load_netlink(msg) + for msg in self.ipdb.nl.get_routes( + family=AF_MPLS, match={'family': AF_MPLS} + ): + self.load_netlink(msg) + + def add(self, spec=None, **kwarg): + ''' + Create a route from a dictionary + ''' + spec = dict(spec or kwarg) + gateway = spec.get('gateway') or '' + dst = spec.get('dst') or '' + if 'tos' not in spec: + spec['tos'] = 0 + if 'scope' not in spec: + spec['scope'] = 0 + if 'table' not in spec: + spec['table'] = 254 + if 'family' not in spec: + if (dst.find(':') > -1) or (gateway.find(':') > -1): + spec['family'] = AF_INET6 + else: + spec['family'] = AF_INET + if not dst: + raise ValueError('dst not specified') + if ( + isinstance(dst, basestring) + and (dst not in ('', 'default')) + and ('/' not in dst) + ): + if spec['family'] == AF_INET: + spec['dst'] = dst + '/32' + elif spec['family'] == AF_INET6: + spec['dst'] = dst + '/128' + if 'priority' not in spec: + if spec['family'] == AF_INET6: + spec['priority'] = IP6_RT_PRIO_USER + else: + spec['priority'] = None + multipath = spec.pop('multipath', []) + if spec.get('family', 0) == AF_MPLS: + table = 'mpls' + if table not in self.tables: + self.tables[table] = MPLSTable(self.ipdb) + route = MPLSRoute(self.ipdb) + else: + table = spec.get('table', 254) + if table not in self.tables: + self.tables[table] = RoutingTable(self.ipdb) + route = Route(self.ipdb) + route.update(spec) + with route._direct_state: + route['ipdb_scope'] = 'create' + for nh in multipath: + if 'encap' in nh: + nh['encap'] = route.make_encap(nh['encap']) + if table == 'mpls': + nh['family'] = AF_MPLS + route.add_nh(nh) + route.begin() + for key, value in spec.items(): + if key == 'encap': + route[key] = route.make_encap(value) + else: + route[key] = value + self.tables[table][route.make_key(route)] = route + return route + + def load_netlink(self, msg): + ''' + Loads an existing route from a rtmsg + ''' + if not isinstance(msg, rtmsg): + return + + if msg['family'] == AF_MPLS: + table = 'mpls' + else: + table = msg.get_attr('RTA_TABLE', msg['table']) + + if table in self.ignore_rtables: + return + + now = time.time() + if now - self._gctime > 5: + self._gctime = now + self.gc() + + # RTM_DELROUTE + if msg['event'] == 'RTM_DELROUTE': + try: + # locate the record + record = self.tables[table][msg] + # delete the record + if record['ipdb_scope'] not in ('locked', 'shadow'): + del self.tables[table][msg] + with record._direct_state: + record['ipdb_scope'] = 'detached' + except Exception as e: + # just ignore this failure for now + log.debug("delroute failed for %s", e) + return + + # RTM_NEWROUTE + if table not in self.tables: + if table == 'mpls': + self.tables[table] = MPLSTable(self.ipdb) + else: + self.tables[table] = RoutingTable(self.ipdb) + self.tables[table].load(msg) + + def gc_mark_addr(self, msg): + ## + # Find invalid IPv4 route records after addr delete + # + # Example:: + # $ sudo ip link add test0 type dummy + # $ sudo ip link set dev test0 up + # $ sudo ip addr add 172.18.0.5/24 dev test0 + # $ sudo ip route add 10.1.2.0/24 via 172.18.0.1 + # ... + # $ sudo ip addr flush dev test0 + # + # The route {'dst': '10.1.2.0/24', 'gateway': '172.18.0.1'} + # will stay in the routing table being removed from the system. + # That's because the kernel doesn't send IPv4 route updates in + # that case, so we have to calculate the update here -- or load + # all the routes from scratch. The latter may be far too + # expensive. + # + # See http://www.spinics.net/lists/netdev/msg254186.html for + # background on this kernel behavior. + + # Simply ignore secondary addresses, as they don't matter + if msg['flags'] & IFA_F_SECONDARY: + return + + # When the primary address is removed, corresponding routes + # may be silently discarded. But if promote_secondaries is set + # to 1, the next secondary becomes a new primary, and routes + # stay. There is no way to know here, whether promote_secondaries + # was set at the moment of the address removal, so we have to + # act as if it wasn't. + + # Get the removed address: + family = msg['family'] + + if family == AF_INET: + addr = msg.get_attr('IFA_LOCAL') + net = struct.unpack('>I', inet_pton(family, addr))[0] & ( + 0xFFFFFFFF << (32 - msg['prefixlen']) + ) + + # now iterate all registered routes and mark those with + # gateway from that network + for record in self.filter({'family': family}): + gw = record['route'].get('gateway') + if gw: + gwnet = struct.unpack('>I', inet_pton(family, gw))[0] & net + if gwnet == net: + with record['route']._direct_state: + record['route']['ipdb_scope'] = 'gc' + record['route']._gctime = time.time() + + elif family == AF_INET6: + # Unlike IPv4, IPv6 route updates are sent after addr + # delete, so no need to delete them here. + pass + else: + # ignore not (IPv4 or IPv6) + return + + def gc_mark_link(self, msg): + ### + # mark route records for GC after link delete + # + if msg['family'] != 0 or msg['state'] != 'down': + return + + for record in self.filter({'oif': msg['index']}): + with record['route']._direct_state: + record['route']['ipdb_scope'] = 'gc' + record['route']._gctime = time.time() + for record in self.filter({'iif': msg['index']}): + with record['route']._direct_state: + record['route']['ipdb_scope'] = 'gc' + record['route']._gctime = time.time() + + def gc(self): + for table in self.tables.keys(): + self.tables[table].gc() + + def remove(self, route, table=None): + if isinstance(route, Route): + table = route.get('table', 254) or 254 + route = route.get('dst', 'default') + else: + table = table or 254 + self.tables[table][route].remove() + + def filter(self, target): + # FIXME: turn into generator! + ret = [] + for table in tuple(self.tables.values()): + if table is not None: + ret.extend(table.filter(target)) + return ret + + def describe(self, spec, table=254): + return self.tables[table].describe(spec) + + def get(self, dst, table=None): + table = table or 254 + return self.tables[table][dst] + + def keys(self, table=254, family=AF_UNSPEC): + return [ + x['dst'] + for x in self.tables[table] + if (x.get('family') == family) or (family == AF_UNSPEC) + ] + + def has_key(self, key, table=254): + return key in self.tables[table] + + def __contains__(self, key): + return key in self.tables[254] + + def __getitem__(self, key): + return self.get(key) + + def __setitem__(self, key, value): + if key != value['dst']: + raise ValueError("dst doesn't match key") + return self.add(value) + + def __delitem__(self, key): + return self.remove(key) + + def __repr__(self): + return repr(self.tables[254]) + + +spec = [{'name': 'routes', 'class': RoutingTableSet, 'kwarg': {}}] diff --git a/pyroute2/ipdb/rules.py b/pyroute2/ipdb/rules.py new file mode 100644 index 000000000..77485463e --- /dev/null +++ b/pyroute2/ipdb/rules.py @@ -0,0 +1,307 @@ +import logging +import threading +import traceback +from collections import namedtuple +from socket import AF_INET, AF_INET6 + +from pyroute2.ipdb.exceptions import CommitException +from pyroute2.ipdb.transactional import Transactional +from pyroute2.netlink import rtnl +from pyroute2.netlink.rtnl.fibmsg import FR_ACT_NAMES, fibmsg + +log = logging.getLogger(__name__) +groups = rtnl.RTMGRP_IPV4_RULE | rtnl.RTMGRP_IPV6_RULE + + +RuleKey = namedtuple( + 'RuleKey', + ( + 'action', + 'table', + 'priority', + 'iifname', + 'oifname', + 'fwmark', + 'fwmask', + 'family', + 'goto', + 'tun_id', + ), +) + + +class Rule(Transactional): + ''' + Persistent transactional rule object + ''' + + _fields = [fibmsg.nla2name(i[1]) for i in fibmsg.nla_map] + for key, _ in fibmsg.fields: + _fields.append(key) + _fields.append('removal') + _virtual_fields = ['ipdb_scope', 'ipdb_priority'] + _fields.extend(_virtual_fields) + cleanup = ( + 'attrs', + 'header', + 'event', + 'src_len', + 'dst_len', + 'res1', + 'res2', + ) + + @classmethod + def make_key(cls, msg): + values = [] + if isinstance(msg, fibmsg): + for field in RuleKey._fields: + v = msg.get_attr(msg.name2nla(field)) + if v is None: + v = msg.get(field, 0) + values.append(v) + elif isinstance(msg, dict): + for field in RuleKey._fields: + values.append(msg.get(field, 0)) + else: + raise TypeError('prime not supported: %s' % type(msg)) + return RuleKey(*values) + + def __init__(self, ipdb, mode=None, parent=None, uid=None): + Transactional.__init__(self, ipdb, mode, parent, uid) + with self._direct_state: + self['ipdb_priority'] = 0 + + def load_netlink(self, msg): + with self._direct_state: + if self['ipdb_scope'] == 'locked': + # do not touch locked interfaces + return + + self['ipdb_scope'] = 'system' + for key, value in msg.items(): + self[key] = value + + # merge NLA + for cell in msg['attrs']: + # + # Parse on demand + # + norm = fibmsg.nla2name(cell[0]) + if norm in self.cleanup: + continue + self[norm] = cell[1] + + if msg.get_attr('FRA_DST'): + dst = '%s/%s' % (msg.get_attr('FRA_DST'), msg['dst_len']) + self['dst'] = dst + if msg.get_attr('FRA_SRC'): + src = '%s/%s' % (msg.get_attr('FRA_SRC'), msg['src_len']) + self['src'] = src + + # finally, cleanup all not needed + for item in self.cleanup: + if item in self: + del self[item] + return self + + def commit( + self, tid=None, transaction=None, commit_phase=1, commit_mask=0xFF + ): + if not commit_phase & commit_mask: + return self + + error = None + drop = self.ipdb.txdrop + devop = 'set' + debug = {'traceback': None, 'next_stage': None} + notx = True + + if tid or transaction: + notx = False + if tid: + transaction = self.global_tx[tid] + else: + transaction = transaction or self.current_tx + + # create a new route + if self['ipdb_scope'] != 'system': + devop = 'add' + + # work on an existing route + snapshot = self.pick() + added, removed = transaction // snapshot + added.pop('ipdb_scope', None) + removed.pop('ipdb_scope', None) + + try: + # rule add/set + if any(added.values()) or devop == 'add': + old_key = self.make_key(self) + new_key = self.make_key(transaction) + + if new_key != old_key: + # check for the key conflict + if new_key in self.ipdb.rules: + raise CommitException('rule priority conflict') + else: + self.ipdb.rules[new_key] = self + self.nl.rule('del', **old_key._asdict()) + self.nl.rule('add', **transaction) + else: + if devop != 'add': + with self._direct_state: + self['ipdb_scope'] = 'locked' + wd = self.ipdb.watchdog( + 'RTM_DELRULE', **old_key._asdict() + ) + self.nl.rule('del', **old_key._asdict()) + wd.wait() + with self._direct_state: + self['ipdb_scope'] = 'reload' + self.nl.rule('add', **transaction) + transaction.wait_all_targets() + # rule removal + if (transaction['ipdb_scope'] in ('shadow', 'remove')) or ( + (transaction['ipdb_scope'] == 'create') and commit_phase == 2 + ): + if transaction['ipdb_scope'] == 'shadow': + with self._direct_state: + self['ipdb_scope'] = 'locked' + # create watchdog + key = self.make_key(snapshot) + wd = self.ipdb.watchdog('RTM_DELRULE', **key._asdict()) + self.nl.rule('del', **key._asdict()) + wd.wait() + if transaction['ipdb_scope'] == 'shadow': + with self._direct_state: + self['ipdb_scope'] = 'shadow' + # everything ok + drop = True + + except Exception as e: + error = e + # prepare postmortem + debug['traceback'] = traceback.format_exc() + debug['error_stack'] = [] + debug['next_stage'] = None + + if commit_phase == 1: + try: + self.commit( + transaction=snapshot, + commit_phase=2, + commit_mask=commit_mask, + ) + except Exception as i_e: + debug['next_stage'] = i_e + error = RuntimeError() + + if drop and notx: + self.drop(transaction.uid) + + if error is not None: + error.debug = debug + raise error + + return self + + def remove(self): + self['ipdb_scope'] = 'remove' + return self + + def shadow(self): + self['ipdb_scope'] = 'shadow' + return self + + +class RulesDict(dict): + def __init__(self, ipdb): + self.ipdb = ipdb + self.lock = threading.Lock() + self._event_map = { + 'RTM_NEWRULE': self.load_netlink, + 'RTM_DELRULE': self.load_netlink, + } + + def _register(self): + for msg in self.ipdb.nl.get_rules(family=AF_INET): + self.load_netlink(msg) + for msg in self.ipdb.nl.get_rules(family=AF_INET6): + self.load_netlink(msg) + + def __getitem__(self, key): + with self.lock: + if isinstance(key, RuleKey): + return super(RulesDict, self).__getitem__(key) + elif isinstance(key, tuple): + return super(RulesDict, self).__getitem__(RuleKey(*key)) + elif isinstance(key, int): + for k in self.keys(): + if key == k[2]: + return super(RulesDict, self).__getitem__(k) + elif isinstance(key, dict): + for v in self.values(): + for k in key: + if key[k] != v.get(k, None): + break + else: + return v + + def add(self, spec=None, **kwarg): + ''' + Create a rule from a dictionary + ''' + spec = dict(spec or kwarg) + # action and priority are parts of the key, so + # they must be specified + if 'priority' not in spec: + spec['priority'] = 32000 + if 'table' in spec: + spec['action'] = FR_ACT_NAMES['FR_ACT_TO_TBL'] + elif 'goto' in spec: + spec['action'] = FR_ACT_NAMES['FR_ACT_GOTO'] + if 'family' not in spec: + spec['family'] = AF_INET + + rule = Rule(self.ipdb) + rule.update(spec) + # setup the scope + with rule._direct_state: + rule['ipdb_scope'] = 'create' + # + rule.begin() + for key, value in spec.items(): + rule[key] = value + self[rule.make_key(spec)] = rule + return rule + + def load_netlink(self, msg): + if not isinstance(msg, fibmsg): + return + + key = Rule.make_key(msg) + + # RTM_DELRULE + if msg['event'] == 'RTM_DELRULE': + try: + # locate the record + record = self[key] + # delete the record + if record['ipdb_scope'] not in ('locked', 'shadow'): + del self[key] + with record._direct_state: + record['ipdb_scope'] = 'detached' + except Exception as e: + # just ignore this failure for now + log.debug("delrule failed for %s", e) + return + + # RTM_NEWRULE + if key not in self: + self[key] = Rule(self.ipdb) + self[key].load_netlink(msg) + return self[key] + + +spec = [{'name': 'rules', 'class': RulesDict, 'kwarg': {}}] diff --git a/pyroute2/ipdb/transactional.py b/pyroute2/ipdb/transactional.py new file mode 100644 index 000000000..e65757be4 --- /dev/null +++ b/pyroute2/ipdb/transactional.py @@ -0,0 +1,530 @@ +''' +''' + +import logging +import threading + +from pyroute2.common import Dotkeys, uuid32 +from pyroute2.ipdb.exceptions import CommitException +from pyroute2.ipdb.linkedset import LinkedSet + +# How long should we wait on EACH commit() checkpoint: for ipaddr, +# ports etc. That's not total commit() timeout. +SYNC_TIMEOUT = 5 +log = logging.getLogger(__name__) + + +class State(object): + def __init__(self, lock=None): + self.lock = lock or threading.Lock() + self.flag = 0 + + def acquire(self): + self.lock.acquire() + self.flag += 1 + + def release(self): + if self.flag < 1: + raise RuntimeError('release unlocked state') + self.flag -= 1 + self.lock.release() + + def is_set(self): + return self.flag + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.release() + + +def update(f): + def decorated(self, *argv, **kwarg): + if self._mode == 'snapshot': + # short-circuit + with self._write_lock: + return f(self, True, *argv, **kwarg) + elif self._mode == 'readonly': + raise RuntimeError('can not change readonly object') + + with self._write_lock: + direct = self._direct_state.is_set() + if not direct: + # 1. 'implicit': begin transaction, if there is none + if self._mode == 'implicit': + if not self.current_tx: + self.begin() + # 2. require open transaction for 'explicit' type + elif self._mode == 'explicit': + if not self.current_tx: + raise TypeError('start a transaction first') + # do not support other modes + else: + raise TypeError('transaction mode not supported') + # now that the transaction _is_ open + return f(self, direct, *argv, **kwarg) + + decorated.__doc__ = f.__doc__ + return decorated + + +def with_transaction(f): + def decorated(self, direct, *argv, **kwarg): + if direct: + f(self, *argv, **kwarg) + else: + transaction = self.current_tx + f(transaction, *argv, **kwarg) + return self + + return update(decorated) + + +class Transactional(Dotkeys): + ''' + Utility class that implements common transactional logic. + ''' + + _fields = [] + _virtual_fields = [] + _fields_cmp = {} + _linked_sets = [] + _nested = [] + + def __init__(self, ipdb=None, mode=None, parent=None, uid=None): + # + if ipdb is not None: + self.nl = ipdb.nl + self.ipdb = ipdb + else: + self.nl = None + self.ipdb = None + # + self._parent = None + if parent is not None: + self._mode = mode or parent._mode + self._parent = parent + elif ipdb is not None: + self._mode = mode or ipdb.mode + else: + self._mode = mode or 'implicit' + # + self.nlmsg = None + self.uid = uid or uuid32() + self.last_error = None + self._commit_hooks = [] + self._sids = [] + self._ts = threading.local() + self._snapshots = {} + self.global_tx = {} + self._targets = {} + self._local_targets = {} + self._write_lock = threading.RLock() + self._direct_state = State(self._write_lock) + self._linked_sets = self._linked_sets or set() + # + for i in self._fields: + Dotkeys.__setitem__(self, i, None) + + @property + def ro(self): + return self.pick(detached=False, readonly=True) + + def register_commit_hook(self, hook): + ''' ''' + self._commit_hooks.append(hook) + + def unregister_commit_hook(self, hook): + ''' ''' + with self._write_lock: + for cb in tuple(self._commit_hooks): + if hook == cb: + self._commit_hooks.pop(self._commit_hooks.index(cb)) + + ## + # Object serialization: dump, pick + def dump(self, not_none=True): + ''' ''' + with self._write_lock: + res = {} + for key in self: + if self[key] is not None and key[0] != '_': + if isinstance(self[key], Transactional): + res[key] = self[key].dump() + elif isinstance(self[key], LinkedSet): + res[key] = tuple(self[key]) + else: + res[key] = self[key] + return res + + def pick(self, detached=True, uid=None, parent=None, readonly=False): + ''' + Get a snapshot of the object. Can be of two + types: + * detached=True -- (default) "true" snapshot + * detached=False -- keep ip addr set updated from OS + + Please note, that "updated" doesn't mean "in sync". + The reason behind this logic is that snapshots can be + used as transactions. + ''' + with self._write_lock: + res = self.__class__( + ipdb=self.ipdb, mode='snapshot', parent=parent, uid=uid + ) + for key, value in self.items(): + if self[key] is not None: + if key in self._fields: + res[key] = self[key] + for key in self._linked_sets: + res[key] = type(self[key])(self[key]) + if not detached: + self[key].connect(res[key]) + if readonly: + res._mode = 'readonly' + + return res + + ## + # Context management: enter, exit + def __enter__(self): + if self._mode == 'readonly': + return self + elif self._mode not in ('implicit', 'explicit'): + raise TypeError('context managers require a transactional mode') + if not self.current_tx: + self.begin() + return self + + def __exit__(self, exc_type, exc_value, traceback): + # apply transaction only if there was no error + if self._mode == 'readonly': + return + elif exc_type is None: + try: + self.commit() + except Exception as e: + self.last_error = e + raise + + ## + # Implicit object transfomations + def __repr__(self): + res = {} + for i in tuple(self): + if self[i] is not None: + res[i] = self[i] + return res.__repr__() + + ## + # Object ops: +, -, /, ... + def __sub__(self, vs): + # create result + res = {} + + with self._direct_state: + # simple keys + for key in self: + if key in self._fields: + if (key not in vs) or (self[key] != vs[key]): + res[key] = self[key] + for key in self._linked_sets: + diff = type(self[key])(self[key] - vs[key]) + if diff: + res[key] = diff + else: + res[key] = set() + for key in self._nested: + res[key] = self[key] - vs[key] + return res + + def __floordiv__(self, vs): + left = {} + right = {} + with self._direct_state: + with vs._direct_state: + for key in set(tuple(self.keys()) + tuple(vs.keys())): + if self.get(key, None) != vs.get(key, None): + left[key] = self.get(key) + right[key] = vs.get(key) + continue + if key not in self: + right[key] = vs[key] + elif key not in vs: + left[key] = self[key] + for key in self._linked_sets: + ldiff = type(self[key])(self[key] - vs[key]) + rdiff = type(vs[key])(vs[key] - self[key]) + if ldiff: + left[key] = ldiff + else: + left[key] = set() + if rdiff: + right[key] = rdiff + else: + right[key] = set() + for key in self._nested: + left[key], right[key] = self[key] // vs[key] + return left, right + + ## + # Methods to be overloaded + def detach(self): + pass + + def load(self, data): + pass + + def commit(self, *args, **kwarg): + pass + + def last_snapshot_id(self): + return self._sids[-1] + + def invalidate(self): + # on failure, invalidate the interface and detach it + # from the parent + # 0. obtain lock on IPDB, to avoid deadlocks + # ... all the DB updates will wait + with self.ipdb.exclusive: + # 1. drop the IPRoute() link + self.nl = None + # 2. clean up ipdb + self.detach() + # 3. invalidate the interface + with self._direct_state: + for i in tuple(self.keys()): + del self[i] + self['ipdb_scope'] = 'invalid' + # 4. the rest + self._mode = 'invalid' + + ## + # Snapshot methods + def revert(self, sid): + with self._write_lock: + assert sid in self._snapshots + self.local_tx[sid] = self._snapshots[sid] + self.global_tx[sid] = self._snapshots[sid] + self.current_tx = self._snapshots[sid] + self._sids.remove(sid) + del self._snapshots[sid] + return self + + def snapshot(self, sid=None): + ''' + Create new snapshot + ''' + if self._parent: + raise RuntimeError("Can't init snapshot from a nested object") + if (self.ipdb is not None) and self.ipdb._stop: + raise RuntimeError("Can't create snapshots on released IPDB") + t = self.pick(detached=True, uid=sid) + self._snapshots[t.uid] = t + self._sids.append(t.uid) + for key, value in t.items(): + if isinstance(value, Transactional): + value.snapshot(sid=t.uid) + return t.uid + + def last_snapshot(self): + if not self._sids: + raise TypeError('create a snapshot first') + return self._snapshots[self._sids[-1]] + + ## + # Current tx + def _set_current_tx(self, tx): + with self._write_lock: + self._ts.current = tx + + def _get_current_tx(self): + ''' + The current active transaction (thread-local) + ''' + with self._write_lock: + if not hasattr(self._ts, 'current'): + self._ts.current = None + return self._ts.current + + current_tx = property(_get_current_tx, _set_current_tx) + + ## + # Local tx registry + def _get_local_tx(self): + with self._write_lock: + if not hasattr(self._ts, 'tx'): + self._ts.tx = {} + return self._ts.tx + + local_tx = property(_get_local_tx) + + ## + # Transaction ops: begin, review, drop + def begin(self): + ''' + Start new transaction + ''' + if self._parent is not None: + self._parent.begin() + else: + return self._begin() + + def _begin(self, tid=None): + if (self.ipdb is not None) and self.ipdb._stop: + raise RuntimeError("Can't start transaction on released IPDB") + t = self.pick(detached=False, uid=tid) + self.local_tx[t.uid] = t + self.global_tx[t.uid] = t + if self.current_tx is None: + self.current_tx = t + for key, value in t.items(): + if isinstance(value, Transactional): + # start transaction on a nested object + value._begin(tid=t.uid) + # link transaction to own one + t[key] = value.global_tx[t.uid] + return t.uid + + def review(self, tid=None): + ''' + Review the changes made in the transaction `tid` + or in the current active transaction (thread-local) + ''' + if self.current_tx is None: + raise TypeError('start a transaction first') + + tid = tid or self.current_tx.uid + + if self.get('ipdb_scope') == 'create': + if self.current_tx is not None: + prime = self.current_tx + else: + log.warning('the "create" scope without transaction') + prime = self + return dict( + [(x[0], x[1]) for x in prime.items() if x[1] is not None] + ) + + with self._write_lock: + added = self.global_tx[tid] - self + removed = self - self.global_tx[tid] + for key in self._linked_sets: + added['-%s' % (key)] = removed[key] + added['+%s' % (key)] = added[key] + del added[key] + return added + + def drop(self, tid=None): + ''' + Drop a transaction. If tid is not specified, drop + the current one. + ''' + with self._write_lock: + if tid is None: + tx = self.current_tx + if tx is None: + raise TypeError("no transaction") + else: + tx = self.global_tx[tid] + + if self.current_tx == tx: + self.current_tx = None + + # detach linked sets + for key in self._linked_sets: + if tx[key] in self[key].links: + self[key].disconnect(tx[key]) + for key, value in self.items(): + if isinstance(value, Transactional): + try: + value.drop(tx.uid) + except KeyError: + pass + # finally -- delete the transaction + del self.local_tx[tx.uid] + del self.global_tx[tx.uid] + + ## + # Property ops: set/get/delete + @update + def __setitem__(self, direct, key, value): + if not direct: + if self.get(key) == value: + return + # automatically set target on the active transaction, + # which must be started prior to that call + transaction = self.current_tx + transaction[key] = value + if value is not None: + transaction._targets[key] = threading.Event() + else: + # set the item + Dotkeys.__setitem__(self, key, value) + + # update on local targets + with self._write_lock: + if key in self._local_targets: + func = self._fields_cmp.get(key, lambda x, y: x == y) + if func(value, self._local_targets[key].value): + self._local_targets[key].set() + + # cascade update on nested targets + for tn in tuple(self.global_tx.values()): + if (key in tn._targets) and (key in tn): + if self._fields_cmp.get(key, lambda x, y: x == y)( + value, tn[key] + ): + tn._targets[key].set() + + @update + def __delitem__(self, direct, key): + # firstly set targets + self[key] = None + + # then continue with delete + if not direct: + transaction = self.current_tx + if key in transaction: + del transaction[key] + else: + Dotkeys.__delitem__(self, key) + + def option(self, key, value): + self[key] = value + return self + + def unset(self, key): + del self[key] + return self + + def wait_all_targets(self): + for key, target in self._targets.items(): + if key not in self._virtual_fields: + target.wait(SYNC_TIMEOUT) + if not target.is_set(): + raise CommitException('target %s is not set' % key) + + def wait_target(self, key, timeout=SYNC_TIMEOUT): + self._local_targets[key].wait(SYNC_TIMEOUT) + with self._write_lock: + return self._local_targets.pop(key).is_set() + + def set_target(self, key, value): + with self._write_lock: + self._local_targets[key] = threading.Event() + self._local_targets[key].value = value + if self.get(key) == value: + self._local_targets[key].set() + return self + + def mirror_target(self, key_from, key_to): + with self._write_lock: + self._local_targets[key_to] = self._local_targets[key_from] + return self + + def set(self, key, value): + self[key] = value + return self diff --git a/pyroute2/ipdb/utils.py b/pyroute2/ipdb/utils.py new file mode 100644 index 000000000..e2cc17d8f --- /dev/null +++ b/pyroute2/ipdb/utils.py @@ -0,0 +1,9 @@ +import os +import subprocess + + +def test_reachable_icmp(host): + with open(os.devnull, 'w') as devnull: + return subprocess.check_call( + ['ping', '-c', '1', host], stdout=devnull, stderr=devnull + ) diff --git a/pyroute2/iproute/__init__.py b/pyroute2/iproute/__init__.py index 904d8daef..0364cdfb1 100644 --- a/pyroute2/iproute/__init__.py +++ b/pyroute2/iproute/__init__.py @@ -1,4 +1,132 @@ # -*- coding: utf-8 -*- +''' +Classes +------- + +The RTNL API is provided by the class `RTNL_API`. It is a +mixin class that works on top of any RTNL-compatible socket, +so several classes with almost the same API are available: + +* `IPRoute` -- simple RTNL API +* `NetNS` -- RTNL API in a network namespace +* `IPBatch` -- RTNL packet compiler +* `RemoteIPRoute` -- run RTNL remotely (no deployment required) + +Responses as lists +------------------ + +The netlink socket implementation in the pyroute2 is +agnostic to particular netlink protocols, and always returns +a list of messages as the response to a request sent to the +kernel:: + + with IPRoute() as ipr: + + # this request returns one match + eth0 = ipr.link_lookup(ifname='eth0') + len(eth0) # -> 1, if exists, else 0 + + # but that one returns a set of + up = ipr.link_lookup(operstate='UP') + len(up) # -> k, where 0 <= k <= [interface count] + +Thus, always expect a list in the response, running any +`IPRoute()` netlink request. + +NLMSG_ERROR responses +~~~~~~~~~~~~~~~~~~~~~ + +Some kernel subsystems return `NLMSG_ERROR` in response to +any request. It is OK as long as `nlmsg["header"]["error"] is None`. +Otherwise an exception will be raised by the parser. + +So if instead of an exception you get a `NLMSG_ERROR` message, +it means `error == 0`, the same as `$? == 0` in bash. + +How to work with messages +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Every netlink message contains header, fields and NLAs +(netlink attributes). Every NLA is a netlink message... +(see "recursion"). + +And the library provides parsed messages according to +this scheme. Every RTNL message contains: + +* `nlmsg['header']` -- parsed header +* `nlmsg['attrs']` -- NLA chain (parsed on demand) +* 0 .. k data fields, e.g. `nlmsg['flags']` etc. +* `nlmsg.header` -- the header fields spec +* `nlmsg.fields` -- the data fields spec +* `nlmsg.nla_map` -- NLA spec + +An important parser feature is that NLAs are parsed +on demand, when someone tries to access them. Otherwise +the parser doesn't waste CPU cycles. + +The NLA chain is a list-like structure, not a dictionary. +The netlink standard doesn't require NLAs to be unique +within one message:: + + {'attrs': [('IFLA_IFNAME', 'lo'), # [1] + ('IFLA_TXQLEN', 1), + ('IFLA_OPERSTATE', 'UNKNOWN'), + ('IFLA_LINKMODE', 0), + ('IFLA_MTU', 65536), + ('IFLA_GROUP', 0), + ('IFLA_PROMISCUITY', 0), + ('IFLA_NUM_TX_QUEUES', 1), + ('IFLA_NUM_RX_QUEUES', 1), + ('IFLA_CARRIER', 1), + ...], + 'change': 0, + 'event': 'RTM_NEWLINK', # [2] + 'family': 0, + 'flags': 65609, + 'header': {'error': None, # [3] + 'flags': 2, + 'length': 1180, + 'pid': 28233, + 'sequence_number': 257, # [4] + 'type': 16}, # [5] + 'ifi_type': 772, + 'index': 1} + + # [1] every NLA is parsed upon access + # [2] this field is injected by the RTNL parser + # [3] if not None, an exception will be raised + # [4] more details in the netlink description + # [5] 16 == RTM_NEWLINK + +To access fields:: + + msg['index'] == 1 + +To access one NLA:: + + msg.get_attr('IFLA_CARRIER') == 1 + +When an NLA with the specified name is not present in the +chain, `get_attr()` returns `None`. To get the list of all +NLAs of that name, use `get_attrs()`. A real example with +NLA hierarchy, take notice of `get_attr()` and +`get_attrs()` usage:: + + # for macvlan interfaces there may be several + # IFLA_MACVLAN_MACADDR NLA provided, so use + # get_attrs() to get all the list, not only + # the first one + + (msg + .get_attr('IFLA_LINKINFO') # one NLA + .get_attr('IFLA_INFO_DATA') # one NLA + .get_attrs('IFLA_MACVLAN_MACADDR')) # a list of + +The protocol itself has no limit for number of NLAs of the +same type in one message, that's why we can not make a dictionary +from them -- unlike PF_ROUTE messages. + +''' import sys from pyroute2 import config @@ -14,41 +142,16 @@ RTM_NEWLINK, ) -AsyncIPRoute = None -if sys.platform.startswith('win'): - from pyroute2.iproute.windows import ( - ChaoticIPRoute, - IPRoute, - NetNS, - RawIPRoute, - ) -elif sys.platform.startswith('darwin'): - from pyroute2.iproute.darwin import ( - ChaoticIPRoute, - IPRoute, - NetNS, - RawIPRoute, - ) +if sys.platform.startswith('emscripten'): + from pyroute2.iproute.ipmock import ChaoticIPRoute, IPRoute, RawIPRoute +elif sys.platform.startswith('win'): + from pyroute2.iproute.windows import ChaoticIPRoute, IPRoute, RawIPRoute elif config.uname[0][-3:] == 'BSD': - from pyroute2.iproute.bsd import ChaoticIPRoute, IPRoute, NetNS, RawIPRoute + from pyroute2.iproute.bsd import ChaoticIPRoute, IPRoute, RawIPRoute else: - from pyroute2.iproute.linux import ( - AsyncIPRoute, - ChaoticIPRoute, - IPRoute, - NetNS, - RawIPRoute, - ) - -classes = [ - AsyncIPRoute, - RTNL_API, - IPBatch, - IPRoute, - RawIPRoute, - ChaoticIPRoute, - NetNS, -] + from pyroute2.iproute.linux import ChaoticIPRoute, IPRoute, RawIPRoute + +classes = [RTNL_API, IPBatch, IPRoute, RawIPRoute, ChaoticIPRoute] constants = [ RTM_GETLINK, diff --git a/pyroute2/iproute/bsd.py b/pyroute2/iproute/bsd.py index f01650f9d..52112f927 100644 --- a/pyroute2/iproute/bsd.py +++ b/pyroute2/iproute/bsd.py @@ -46,7 +46,6 @@ import errno import os -import queue import select import struct import threading @@ -55,7 +54,9 @@ from pyroute2.bsd.pf_route import IFF_VALUES from pyroute2.bsd.rtmsocket import RTMSocket from pyroute2.bsd.util import ARP, Ifconfig, Route +from pyroute2.common import AddrPool, Namespace from pyroute2.netlink import NLM_F_DUMP, NLM_F_MULTI, NLM_F_REQUEST, NLMSG_DONE +from pyroute2.netlink.proxy import NetlinkProxy from pyroute2.netlink.rtnl import ( RTM_GETADDR, RTM_GETLINK, @@ -72,6 +73,11 @@ from pyroute2.netlink.rtnl.ndmsg import ndmsg from pyroute2.netlink.rtnl.rtmsg import rtmsg +try: + import queue +except ImportError: + import Queue as queue + class IPRoute(object): def __init__(self, *argv, **kwarg): @@ -85,6 +91,10 @@ def __init__(self, *argv, **kwarg): self._route = Route(cmd=self._ssh + ['netstat', '-rn']) self.marshal = MarshalRtnl() self.target = kwarg.get('target') or 'localhost' + send_ns = Namespace( + self, {'addr_pool': AddrPool(0x10000, 0x1FFFF), 'monitor': False} + ) + self._sproxy = NetlinkProxy(policy='return', nl=send_ns) self._mon_th = None self._rtm = None self._brd_socket = None @@ -317,8 +327,3 @@ class RawIPRoute(IPRoute): class ChaoticIPRoute: def __init__(self, *argv, **kwarg): raise NotImplementedError() - - -class NetNS: - def __init__(self, *argv, **kwarg): - raise NotImplementedError() diff --git a/pyroute2/iproute/darwin.py b/pyroute2/iproute/darwin.py deleted file mode 100644 index 3c387e607..000000000 --- a/pyroute2/iproute/darwin.py +++ /dev/null @@ -1,478 +0,0 @@ -import os -import socket -import subprocess -from socket import AF_INET, AF_INET6 - -from pyroute2.common import dqn2int -from pyroute2.netlink import NLM_F_DUMP, NLM_F_MULTI, NLM_F_REQUEST, NLMSG_DONE -from pyroute2.netlink.rtnl import ( - RTM_GETADDR, - RTM_GETLINK, - RTM_GETNEIGH, - RTM_GETROUTE, - RTM_NEWADDR, - RTM_NEWLINK, - RTM_NEWNEIGH, - RTM_NEWROUTE, -) -from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg -from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg -from pyroute2.netlink.rtnl.marshal import MarshalRtnl - - -class IPRoute(object): - """ - macOS-specific IPRoute class (proof-of-concept). - Attempts to fetch interfaces, addresses, neighbors, and routes - by parsing macOS command output (ifconfig, netstat, arp, ndp, etc.). - """ - - def __init__(self, *argv, **kwarg): - self.marshal = MarshalRtnl() - self.target = kwarg.get('target') or 'localhost' - self._outq = kwarg.get('_outq', None) - self._pfdw = kwarg.get('_pfdw', None) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def clone(self): - return self - - def close(self, code=None): - pass - - def bind(self, *argv, **kwarg): - pass - - def getsockopt(self, *argv, **kwarg): - return 1024 * 1024 - - def sendto_gate(self, msg, addr): - """ - Handle incoming netlink requests (simulation for macOS). - """ - cmd = msg['header']['type'] - flags = msg['header']['flags'] - seq = msg['header']['sequence_number'] - - # Only handle dump requests - if flags != NLM_F_REQUEST | NLM_F_DUMP: - return - - if cmd == RTM_GETLINK: - rtype = RTM_NEWLINK - ret = self.get_links() - elif cmd == RTM_GETADDR: - rtype = RTM_NEWADDR - ret = self.get_addr() - elif cmd == RTM_GETROUTE: - rtype = RTM_NEWROUTE - ret = self.get_routes() - elif cmd == RTM_GETNEIGH: - rtype = RTM_NEWNEIGH - ret = self.get_neighbours() - else: - ret = [] - - # Set response type and finalize - for r in ret: - r['header']['type'] = rtype - r['header']['flags'] = NLM_F_MULTI - r['header']['sequence_number'] = seq - - done_msg = type(msg)() - done_msg['header']['type'] = NLMSG_DONE - done_msg['header']['sequence_number'] = seq - ret.append(done_msg) - - # If using an output queue (as done in Windows code), - # write data there - if self._outq and self._pfdw: - data = b'' - for r in ret: - r.encode() - data += r.data - self._outq.put(data) - os.write(self._pfdw, b'\0') - - def _parse_ifconfig(self): - """ - Parse `ifconfig` output to gather interface info, MAC, and addresses. - Returns a dictionary: - { - 'interfaces': [ifinfmsg, ifinfmsg, ...], - 'addresses': [ifaddrmsg, ifaddrmsg, ...] - } - """ - ret = {'interfaces': [], 'addresses': []} - - try: - output = subprocess.check_output(["ifconfig"]).decode( - "utf-8", errors="replace" - ) - except (subprocess.CalledProcessError, FileNotFoundError): - return ret - - blocks = output.strip().split("\n\n") - - for block in blocks: - lines = block.strip().splitlines() - if not lines: - continue - - # The first line usually has "en0: flags=..." - first_line = lines[0] - iface_name = first_line.split(":")[0].strip() - - # Attempt to fetch interface index and MAC address - try: - iface_index = socket.if_nametoindex(iface_name) - except OSError: - iface_index = 0 # fallback if something goes wrong - - mac_addr = "00:00:00:00:00:00" - for line in lines: - # Lines with "ether " on macOS contain MAC, e.g.: - # "ether 00:1c:42:aa:bb:cc" - if line.strip().startswith("ether "): - parts = line.strip().split() - if len(parts) >= 2: - mac_addr = parts[1] - break - - # Build ifinfmsg - spec_if = { - 'index': iface_index, - 'attrs': ( - ['IFLA_ADDRESS', mac_addr], - ['IFLA_IFNAME', iface_name], - ), - } - msg_if = ifinfmsg().load(spec_if) - msg_if['header']['target'] = self.target - msg_if['header']['type'] = RTM_NEWLINK - # remove 'value' if present - msg_if.pop('value', None) - ret['interfaces'].append(msg_if) - - # Look for IPv4 or IPv6 addresses - # "inet 192.168.1.10 netmask 0xffffff00 broadcast 192.168.1.255" - # "inet6 fe80::... prefixlen 64 ..." - for line in lines[1:]: - parts = line.strip().split() - if line.strip().startswith("inet "): - # IPv4 - try: - ip_idx = parts.index("inet") + 1 - ip_addr = parts[ip_idx] - mask_str = "255.255.255.0" # fallback - if "netmask" in parts: - mask_idx = parts.index("netmask") + 1 - mask_hex = parts[mask_idx] - if mask_hex.startswith("0x"): - nm_value = int(mask_hex, 16) - dotted_mask = [] - for _ in range(4): - dotted_mask.insert(0, str(nm_value & 0xFF)) - nm_value >>= 8 - mask_str = ".".join(dotted_mask) - - prefix_len = dqn2int(mask_str) - - spec_addr = { - 'index': iface_index, - 'family': AF_INET, - 'prefixlen': prefix_len, - 'attrs': ( - ['IFA_ADDRESS', ip_addr], - ['IFA_LOCAL', ip_addr], - ['IFA_LABEL', iface_name], - ), - } - msg_addr = ifaddrmsg().load(spec_addr) - msg_addr['header']['target'] = self.target - msg_addr['header']['type'] = RTM_NEWADDR - msg_addr.pop('value', None) - ret['addresses'].append(msg_addr) - except (ValueError, IndexError): - continue - elif line.strip().startswith("inet6 "): - # IPv6 - try: - ip_idx = parts.index("inet6") + 1 - ip_addr = parts[ip_idx] - prefix_len = 64 # default if not found - - # On macOS you'll often see - # "inet6 fe80::xxxx prefixlen 64 ..." - if "prefixlen" in parts: - pre_idx = parts.index("prefixlen") + 1 - prefix_len = int(parts[pre_idx]) - - spec_addr = { - 'index': iface_index, - 'family': AF_INET6, - 'prefixlen': prefix_len, - 'attrs': ( - ['IFA_ADDRESS', ip_addr], - ['IFA_LOCAL', ip_addr], - ['IFA_LABEL', iface_name], - ), - } - msg_addr = ifaddrmsg().load(spec_addr) - msg_addr['header']['target'] = self.target - msg_addr['header']['type'] = RTM_NEWADDR - msg_addr.pop('value', None) - ret['addresses'].append(msg_addr) - except (ValueError, IndexError): - continue - - return ret - - def _parse_routes(self): - """ - Parse `netstat -rn` to retrieve routing table entries. - Return list of ifinfmsg-like route objects, though strictly - they'd be `rtmsg` in a real Netlink environment. - For demonstration, we just return RTM_NEWROUTE messages - with minimal info: destination, gateway, interface index, etc. - """ - routes = [] - try: - output = subprocess.check_output(["netstat", "-rn"]).decode( - "utf-8", errors="replace" - ) - except (subprocess.CalledProcessError, FileNotFoundError): - return routes - - # On macOS, `netstat -rn` has headers like: - # Destination Gateway Flags Netif Expire - # default 192.168.1.1 UGSc en0 - # 127 127.0.0.1 UCS lo0 - # ... - lines = output.strip().splitlines() - # skip header lines - # find the header first - start_index = 0 - for i, line in enumerate(lines): - if "Destination" in line and "Gateway" in line: - start_index = i + 1 - break - - # parse from start_index onward - for line in lines[start_index:]: - parts = line.split() - if len(parts) < 4: - continue - destination, gateway, flags, netif = parts[:4] - # Attempt to get interface index - try: - idx = socket.if_nametoindex(netif) - except OSError: - idx = 0 - - # We'll store the route as ifinfmsg for - # demonstration, but real netlink - # code would use `rtmsg`. We'll keep it - # consistent with the rest of this - # POC approach. - spec_route = { - 'index': idx, - # not strictly correct, but placing - # them in attrs for demonstration - 'attrs': ( - ['ROUTE_DST', destination], - ['ROUTE_GATEWAY', gateway], - ['ROUTE_FLAGS', flags], - ['ROUTE_IFNAME', netif], - ), - } - # We'll build it as ifinfmsg with a 'type' = RTM_NEWROUTE - msg_route = ifinfmsg().load(spec_route) - msg_route['header']['target'] = self.target - msg_route['header']['type'] = RTM_NEWROUTE - msg_route.pop('value', None) - routes.append(msg_route) - - return routes - - def _parse_arp_neighbors(self): - """ - Parse `arp -an` for IPv4 neighbors and (optionally) - `ndp -an` for IPv6. - Returns a list of ifinfmsg (or ifaddrmsg) - objects simulating neighbor entries. - """ - neighbors = [] - # IPv4 neighbors (ARP) - try: - output = subprocess.check_output(["arp", "-an"]).decode( - "utf-8", errors="replace" - ) - # Lines look like: - # "? (192.168.1.10) at 00:1c:42:xx:yy:zz on en0 ifscope [ethernet]" - for line in output.strip().splitlines(): - line = line.strip() - if not line: - continue - # Quick parse: - # 1) IP in parentheses - # 2) MAC after "at" - # 3) interface name after "on" - parts = line.split() - # example parts: - # ["?", "(192.168.1.10)", - # "at", "00:1c:42:xx:yy:zz", - # "on", "en0", ...] - if len(parts) < 7: - continue - ip_str = parts[1].strip("()") - mac_str = parts[3] - iface_str = parts[5] - try: - idx = socket.if_nametoindex(iface_str) - except OSError: - idx = 0 - - spec_neigh = { - 'index': idx, - 'attrs': ( - ['NEIGH_IP', ip_str], - ['NEIGH_LLADDR', mac_str], - ['NEIGH_IFNAME', iface_str], - ), - } - msg_neigh = ifinfmsg().load(spec_neigh) - msg_neigh['header']['target'] = self.target - msg_neigh['header']['type'] = RTM_NEWNEIGH - msg_neigh.pop('value', None) - neighbors.append(msg_neigh) - except (subprocess.CalledProcessError, FileNotFoundError): - pass - - # Optional: IPv6 neighbors (NDP) - try: - output = subprocess.check_output(["ndp", "-an"]).decode( - "utf-8", errors="replace" - ) - # Lines look like: - # "fe80::1%lo0 lladdr 00:00:00:... router STALE" - for line in output.strip().splitlines(): - line = line.strip() - if not line: - continue - parts = line.split() - if len(parts) < 4: - continue - # typical parts: - # ["fe80::1%lo0", - # "lladdr", "00:00:00:..", - # "router", "STALE"] - # or ["2001:db8::1234%en0", - # "lladdr", - # "00:11:22:33:44:55", - # "REACHABLE"] - addr_part = parts[0] - # separate out the interface from the IP - if "%" in addr_part: - ip_str, iface_str = addr_part.split("%", 1) - else: - ip_str = addr_part - iface_str = "??" - mac_str = None - if "lladdr" in parts: - ll_idx = parts.index("lladdr") + 1 - if ll_idx < len(parts): - mac_str = parts[ll_idx] - if not mac_str: - continue - try: - idx = socket.if_nametoindex(iface_str) - except OSError: - idx = 0 - - spec_neigh = { - 'index': idx, - 'attrs': ( - ['NEIGH_IP', ip_str], - ['NEIGH_LLADDR', mac_str], - ['NEIGH_IFNAME', iface_str], - ), - } - msg_neigh = ifinfmsg().load(spec_neigh) - msg_neigh['header']['target'] = self.target - msg_neigh['header']['type'] = RTM_NEWNEIGH - msg_neigh.pop('value', None) - neighbors.append(msg_neigh) - except (subprocess.CalledProcessError, FileNotFoundError): - pass - - return neighbors - - def dump(self, groups=None): - # Return all info in sequence - for method in ( - self.get_links, - self.get_addr, - self.get_neighbours, - self.get_routes, - ): - for msg in method(): - yield msg - - def get_links(self, *argv, **kwarg): - ''' - { - 'attrs': (['IFLA_ADDRESS', '9a:9d:81:90:d2:8f'], - ['IFLA_IFNAME', 'lo0']), - 'header': {'target': 'localhost', 'type': 16}, - 'index': 1 - } - ''' - return self._parse_ifconfig()['interfaces'] - - def get_addr(self, *argv, **kwarg): - ''' - { - 'attrs': (['IFA_ADDRESS', '127.0.0.1'], - ['IFA_LOCAL', '127.0.0.1'], - ['IFA_LABEL', 'lo0']), - 'header': {'target': 'localhost', 'type': 20}, - 'index': 1, - 'family': , - 'prefixlen': 8 - } - ''' - return self._parse_ifconfig()['addresses'] - - def get_neighbours(self, *argv, **kwarg): - return self._parse_arp_neighbors() - - def get_routes(self, *argv, **kwarg): - return self._parse_routes() - - -class RawIPRoute(IPRoute): - pass - - -class ChaoticIPRoute: - """ - Placeholder, mirroring its Windows counterpart. - """ - - def __init__(self, *argv, **kwarg): - raise NotImplementedError() - - -class NetNS: - """ - Another placeholder class to match the Windows code structure. - """ - - def __init__(self, *argv, **kwarg): - raise NotImplementedError() diff --git a/pyroute2/iproute/ipmock.py b/pyroute2/iproute/ipmock.py index 6c366429c..507f1ced5 100644 --- a/pyroute2/iproute/ipmock.py +++ b/pyroute2/iproute/ipmock.py @@ -1,43 +1,29 @@ import copy import errno -import os +import queue import socket import struct -import threading from itertools import count -from pyroute2.config import AF_NETLINK -from pyroute2.netlink import ( - NLM_F_DUMP, - NLM_F_MULTI, - NLMSG_DONE, - nlmsg, - nlmsgerr, -) -from pyroute2.netlink.core import Stats -from pyroute2.netlink.rtnl import ( - RTM_DELADDR, - RTM_DELROUTE, - RTM_GETADDR, - RTM_GETLINK, - RTM_GETROUTE, - RTM_NEWADDR, - RTM_NEWLINK, - RTM_NEWROUTE, -) +from pyroute2.lab import LAB_API +from pyroute2.netlink.exceptions import NetlinkError +from pyroute2.netlink.nlsocket import NetlinkSocketBase, Stats from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg from pyroute2.netlink.rtnl.marshal import MarshalRtnl from pyroute2.netlink.rtnl.rtmsg import rtmsg +from pyroute2.requests.address import AddressFieldFilter, AddressIPRouteFilter +from pyroute2.requests.link import LinkFieldFilter +from pyroute2.requests.main import RequestProcessor +from pyroute2.requests.route import RouteFieldFilter interface_counter = count(3) -MAGIC_CLOSE = 0x42 class MockLink: def __init__( self, - index=0, + index, ifname='', address='00:00:00:00:00:00', broadcast='ff:ff:ff:ff:ff:ff', @@ -57,7 +43,7 @@ def __init__( br_forward_delay=0, alt_ifname_list=None, ): - self.index = index if index > 0 else next(interface_counter) + self.index = index self.ifname = ifname self.flags = flags self.address = address @@ -77,32 +63,6 @@ def __init__( self.br_forward_delay = br_forward_delay self.alt_ifname_list = alt_ifname_list or [] - def update_from_msg(self, msg): - [ - setattr(self, x, msg.get(x)) - for x in ['address', 'broadcast', 'mtu', 'ifname', 'master'] - if msg.get(x) is not None - ] - # - self.kind = msg.get(('linkinfo', 'kind')) - if msg.get('change') != 0: - self.flags = msg.get('flags') - # vlan - if self.kind == 'vlan': - self.vlan_id = msg.get(('linkinfo', 'data', 'vlan_id')) - self.link = msg.get('link') - elif self.kind == 'bridge': - self.br_max_age = msg.get(('linkinfo', 'data', 'br_max_age')) - self.br_forward_delay = msg.get( - ('linkinfo', 'data', 'br_forward_delay') - ) - - @classmethod - def load_from_msg(cls, msg): - ret = cls() - ret.update_from_msg(msg) - return ret - def export(self): ret = { 'attrs': [ @@ -282,9 +242,9 @@ def export(self): class MockAddress: def __init__( self, - index=0, - address=None, - prefixlen=None, + index, + address, + prefixlen, broadcast=None, label=None, family=2, @@ -299,19 +259,6 @@ def __init__( self.label = label self.family = family - def update_from_msg(self, msg): - [ - setattr(self, x, msg.get(x)) - for x in ['index', 'address', 'broadcast', 'prefixlen'] - if msg.get(x) is not None - ] - - @classmethod - def load_from_msg(cls, msg): - ret = cls() - ret.update_from_msg(msg) - return ret - def export(self): ret = { 'family': self.family, @@ -355,8 +302,8 @@ def export(self): class MockRoute: def __init__( self, - dst=None, - oif=0, + dst, + oif, gateway=None, prefsrc=None, family=2, @@ -381,27 +328,6 @@ def __init__( self.tos = kwarg.get('tos', 0) self._type = kwarg.get('type', 2) - def update_from_msg(self, msg): - [ - setattr(self, x, msg.get(x)) - for x in [ - 'dst', - 'dst_len', - 'gateway', - 'oif', - 'family', - 'table', - 'priority', - ] - if msg.get(x) is not None - ] - - @classmethod - def load_from_msg(cls, msg): - ret = cls() - ret.update_from_msg(msg) - return ret - def export(self): ret = { 'family': self.family, @@ -563,325 +489,211 @@ def export(self): } -class IPEngine: - '''Mock network objects database with the socket API. - - WIP: work in progress. - - A drop-in replacement to use instead of a low level RTNL socket. - Implements all the required socket properties and provides a - network objects database with RTNL protocol. - - Example:: - - >>> ipe = IPEngine() - >>> ipr = IPRoute(use_socket=ipe) - >>> [ x.get('ifname') for x in ipr.link('dump') ] - ['lo', 'eth0'] - - ''' - - def __init__( - self, - sfamily=AF_NETLINK, - stype=socket.SOCK_DGRAM, - sproto=0, - netns='default', - flags=os.O_CREAT, - ): +class IPRoute(LAB_API, NetlinkSocketBase): + def __init__(self, *argv, **kwarg): + super().__init__() self.marshal = MarshalRtnl() - self.netns = netns - self.flags = flags - self.magic = 0 - self._stype = stype - self._sfamily = sfamily - self._sproto = sproto - self._local = threading.local() - self._lock = threading.Lock() - self._broadcast = set() - self.processors = { - RTM_GETADDR: self.RTM_GETADDR, - RTM_GETLINK: self.RTM_GETLINK, - RTM_NEWADDR: self.RTM_NEWADDR, - RTM_DELADDR: self.RTM_DELADDR, - RTM_NEWLINK: self.RTM_NEWLINK, - RTM_DELROUTE: self.RTM_DELROUTE, - RTM_NEWROUTE: self.RTM_NEWROUTE, - RTM_GETROUTE: self.RTM_GETROUTE, - } - self.initdb() - - @property - def loopback_r(self): - if not hasattr(self._local, 'loopback_r'): - self._local.loopback_r, self._local.loopback_w = socket.socketpair( - socket.AF_UNIX, socket.SOCK_DGRAM - ) - return self._local.loopback_r - - @property - def loopback_w(self): - if not hasattr(self._local, 'loopback_w'): - self._local.loopback_r, self._local.loopback_w = socket.socketpair( - socket.AF_UNIX, socket.SOCK_DGRAM - ) - return self._local.loopback_w - - def initdb(self): - if self.netns not in presets: - if not self.flags & os.O_CREAT: - raise FileNotFoundError() - presets[self.netns] = copy.deepcopy(presets['netns']) - self.database = copy.deepcopy(presets[self.netns]) - - def close(self): - if self.magic == MAGIC_CLOSE: - self.loopback_r.close() - self.loopback_w.close() - - def bind(self, address=None): - self._broadcast.add(self.loopback_w) - msg = rtmsg() - msg.load(self.database['routes'][0].export()) - msg.encode() - self.loopback_w.send(msg.data) - - def fileno(self): - return self.loopback_r.fileno() - - def recv(self, bufsize, flags=0): - return self.loopback_r.recv(bufsize, flags) - - def recvfrom(self, bufsize, flags=0): - return self.loopback_r.recvfrom(bufsize, flags) - - def recvmsg(self, bufsize, ancbufsize=0, flags=0): - return self.loopback_r.recvmsg(bufsize, ancbufsize, flags) - - def recv_into(self, buffer, nbytes=0, flags=0): - return self.loopback_r.recv_into(buffer, nbytes, flags) - - def recvfrom_into(self, buffer, nbytes=0, flags=0): - return self.loopback_r.recvfrom_into(buffer, nbytes, flags) - - def recvmsg_into(self, buffers, ancbufsize=0, flags=0): - return self.loopback_r.recvmsg_into(buffers, ancbufsize, flags) - - def send(self, data, flags=0): - return self.nl_handle(data) - - def sendall(self, data, flags=0): - return self.nl_handle(data) - - def sendto(self, data, flags, address=0): - return self.nl_handle(data) - - def sendmsg(self, buffers, ancdata=None, flags=None, address=None): - raise NotImplementedError() - - def setblocking(self, flag): - return self.loopback_r.setblocking(flag) - - def getblocking(self): - return self.loopback_r.getblocking() - - def getsockname(self): - return self.loopback_r.getsockname() - - def getpeername(self): - return self.loopback_r.getpeername() - - @property - def type(self): - return self._stype - - @property - def family(self): - return self._sfamily - - @property - def proto(self): - return self._sproto - - def nl_handle(self, data): - with self._lock: - for msg in self.marshal.parse(data): - key = msg['header']['type'] - tag = msg['header']['sequence_number'] - if key in self.processors: - self.processors[key](msg) - else: - self.nl_done(tag) - return len(data) - - def nl_dump(self, registry, msg_class, tag): - for item in registry: + self.target = kwarg.get('target') + self.preset = copy.deepcopy( + presets[kwarg['preset'] if 'preset' in kwarg else 'default'] + ) + self.buffer_queue = queue.Queue(maxsize=512) + self.input_from_buffer_queue = True + + def bind(self, async_cache=True, clone_socket=True): + pass + + def dump(self, groups=None): + for method in (self.get_links, self.get_addr, self.get_routes): + for msg in method(): + yield msg + + def _get_dump(self, dump, msg_class): + for data in dump: + loader = msg_class() + loader.load(data.export()) + loader.encode() msg = msg_class() - msg.load(item.export()) - msg['header']['flags'] = NLM_F_MULTI - msg['header']['sequence_number'] = tag - msg.encode() + msg.data = loader.data + msg.decode() + if self.target is not None: + msg['header']['target'] = self.target yield msg - def nl_broadcast(self, msg): - for sock in self._broadcast: - msg['header']['sequence_number'] = 0 - msg.reset() - msg.encode() - sock.send(msg.data) - - def nl_done(self, tag): - msg = nlmsg() - msg['header']['type'] = NLMSG_DONE - msg['header']['sequence_number'] = tag - msg.encode() - self.loopback_w.send(msg.data) - - def nl_error(self, tag, code): - msg = nlmsgerr() - msg['header']['sequence_number'] = tag - msg['error'] = code - msg.encode() - self.loopback_w.send(msg.data) - - def RTM_GETROUTE(self, req): - tag = req['header']['sequence_number'] - for msg in self.nl_dump(self.database['routes'], rtmsg, tag): - self.loopback_w.send(msg.data) - self.nl_done(tag) - - def RTM_GETADDR(self, msg_in): - tag = msg_in['header']['sequence_number'] - for msg_out in self.nl_dump(self.database['addr'], ifaddrmsg, tag): - self.loopback_w.send(msg_out.data) - self.nl_done(tag) - - def RTM_GETLINK(self, msg_in): - tag = msg_in['header']['sequence_number'] - database = self.database['links'] - if msg_in.get('index') > 0: - database = [ - x - for x in self.database['links'] - if x.index == msg_in.get('index') - ] - elif msg_in.get('ifname'): - database = [ - x - for x in self.database['links'] - if x.ifname == msg_in.get('ifname') - ] - for msg_out in self.nl_dump(database, ifinfmsg, tag): - self.loopback_w.send(msg_out.data) - if msg_in.get(('header', 'flags')) & NLM_F_DUMP: - return self.nl_done(tag) - self.nl_error(tag, 0) - - def RTM_NEWROUTE(self, req): - tag = req['header']['sequence_number'] - if not req.get('oif'): - (gateway,) = struct.unpack( - '>I', socket.inet_aton(req.get('gateway')) - ) - for route in self.database['routes']: - if route.dst is None: - continue - (dst,) = struct.unpack('>I', socket.inet_aton(route.dst)) - if (gateway & (0xFFFFFFFF << (32 - route.dst_len))) == dst: - req['attrs'].append(('RTA_OIF', route.oif)) + def _match(self, mode, obj, spec): + keys = { + 'address': ['address', 'prefixlen', 'index', 'family'], + 'link': ['index', 'ifname'], + 'route': ['dst', 'dst_len', 'oif', 'priority'], + } + check = False + for key in keys[mode]: + if key in spec: + check = True + if spec[key] != getattr(obj, key): + return False + if not check: + return False + return True + + def addr(self, command, **spec): + if command == 'dump': + return self.get_addr() + request = RequestProcessor(context=spec, prime=spec) + request.apply_filter(AddressFieldFilter()) + request.apply_filter(AddressIPRouteFilter(command)) + request.finalize() + address = None + + for address in self.preset['addr']: + if self._match('address', address, request): + if command == 'add': + raise NetlinkError(errno.EEXIST, 'address exists') + break + else: + if command == 'del': + raise NetlinkError(errno.ENOENT, 'address does not exist') + address = MockAddress(**request) + + if command == 'add': + for link in self.preset['links']: + if link.index == request['index']: break else: - return self.nl_error(tag, errno.ENOENT) - idx = { - (x.dst, x.dst_len, x.oif, x.priority, x.gateway, x.table): x - for x in self.database['routes'] - } - req_index = ( - req.get('dst'), - req.get('dst_len'), - req.get('oif'), - req.get('priority'), - req.get('gateway'), - req.get('IFLA_TABLE') or req.get('table'), - ) - if req_index in idx: - return self.nl_error(tag, errno.EEXIST) - route = MockRoute.load_from_msg(req) - self.database['routes'].append(route) - self.nl_error(tag, 0) - msg = rtmsg() - msg.load(route.export()) - self.nl_broadcast(msg) - - def RTM_DELROUTE(self, req): - idx = { - (x.dst, x.dst_len, x.oif, x.priority, x.table): x - for x in self.database['routes'] - } - req_index = ( - req.get('dst'), - req.get('dst_len'), - req.get('oif'), - req.get('priority'), - req.get('IFLA_TABLE') or req.get('table'), - ) - tag = req['header']['sequence_number'] - if req_index not in idx: - return self.nl_error(tag, errno.ENOENT) - self.database['routes'].remove(idx[req_index]) - self.nl_error(tag, 0) - self.nl_broadcast(req) - - def RTM_DELADDR(self, req): - idx = { - (x.index, x.address, x.prefixlen): x for x in self.database['addr'] - } - req_index = ( - req.get("index"), - req.get("address"), - req.get("prefixlen"), - ) - tag = req['header']['sequence_number'] - if req_index not in idx: - return self.nl_error(tag, errno.ENOENT) - self.database['addr'].remove(idx[req_index]) - self.nl_error(tag, 0) - self.nl_broadcast(req) - - def RTM_NEWADDR(self, req): - idx = { - (x.index, x.address, x.prefixlen) for x in self.database['addr'] - } - req_index = ( - req.get("index"), - req.get("address"), - req.get("prefixlen"), - ) - tag = req['header']['sequence_number'] - if req_index in idx: - return self.nl_error(tag, errno.EEXIST) - addr = MockAddress.load_from_msg(req) - self.database['addr'].append(addr) - self.nl_error(tag, 0) - msg = ifaddrmsg() - msg.load(addr.export()) - self.nl_broadcast(msg) - - def RTM_NEWLINK(self, req): - idx = {x.index: x for x in self.database['links']} - nmx = {x.ifname: x for x in self.database['links']} - tag = req['header']['sequence_number'] - if req.get('index') in idx: - link = idx[req.get('index')] - if link.ifname == req.get('ifname'): - return self.nl_error(tag, errno.EEXIST) - link.update_from_msg(req) - elif req.get('ifname') in nmx: - return self.nl_error(tag, errno.EEXIST) + raise NetlinkError(errno.ENOENT, 'link not found') + address.label = link.ifname + self.preset['addr'].append(address) + for msg in self._get_dump([address], ifaddrmsg): + msg.encode() + self.buffer_queue.put(msg.data) + elif command == 'del': + self.preset['addr'].remove(address) + for msg in self._get_dump([address], ifaddrmsg): + msg['header']['type'] = 21 + msg['event'] = 'RTM_DELADDR' + msg.encode() + self.buffer_queue.put(msg.data) + + return self._get_dump([address], ifaddrmsg) + + def link(self, command, **spec): + if command == 'dump': + return self.get_links() + if 'state' in spec: + spec['flags'] = 1 if spec.pop('state') == 'up' else 0 + request = RequestProcessor(context=spec, prime=spec) + request.apply_filter(LinkFieldFilter()) + request.finalize() + + for interface in self.preset['links']: + if self._match('link', interface, request): + if command == 'add': + raise NetlinkError(errno.EEXIST, 'interface exists') + break + else: + index = next(interface_counter) + if 'address' not in request: + request['address'] = f'00:11:22:33:44:{index:02}' + if 'index' not in request: + request['index'] = index + if 'tflags' in request: + del request['tflags'] + if 'target' in request: + del request['target'] + interface = MockLink(**request) + + if command == 'add': + self.preset['links'].append(interface) + for msg in self._get_dump([interface], ifinfmsg): + msg.encode() + self.buffer_queue.put(msg.data) + elif command == 'set': + for key, value in request.items(): + if hasattr(interface, key): + setattr(interface, key, value) + for msg in self._get_dump([interface], ifinfmsg): + msg.encode() + self.buffer_queue.put(msg.data) + + return self._get_dump([interface], ifinfmsg) + + def route(self, command, **spec): + if command == 'dump': + return self.get_routes() + request = RequestProcessor(context=spec, prime=spec) + request.apply_filter(RouteFieldFilter()) + request.finalize() + + for route in self.preset['routes']: + if self._match('route', route, request): + if command == 'add': + raise NetlinkError(errno.EEXIST, 'route exists') + break else: - link = MockLink.load_from_msg(req) - self.database['links'].append(link) - self.nl_error(tag, 0) - msg = ifinfmsg() - msg.load(link.export()) - self.nl_broadcast(msg) + if command == 'del': + raise NetlinkError(errno.ENOENT, 'route does not exist') + if 'tflags' in request: + del request['tflags'] + if 'target' in request: + del request['target'] + if 'multipath' in request: + del request['multipath'] + if 'metrics' in request: + del request['metrics'] + if 'deps' in request: + del request['deps'] + if 'oif' not in request: + (gateway,) = struct.unpack( + '>I', socket.inet_aton(request['gateway']) + ) + for route in self.preset['routes']: + if route.dst is None: + continue + (dst,) = struct.unpack('>I', socket.inet_aton(route.dst)) + if (gateway & (0xFFFFFFFF << (32 - route.dst_len))) == dst: + request['oif'] = route.oif + break + else: + raise NetlinkError(errno.ENOENT, 'no route to the gateway') + route = MockRoute(**request) + + if command == 'add': + self.preset['routes'].append(route) + for msg in self._get_dump([route], rtmsg): + msg.encode() + self.buffer_queue.put(msg.data) + elif command == 'set': + for key, value in request.items(): + if hasattr(route, key): + setattr(route, key, value) + for msg in self._get_dump([route], rtmsg): + msg.encode() + self.buffer_queue.put(msg.data) + elif command == 'del': + self.preset['routes'].remove(route) + for msg in self._get_dump([route], rtmsg): + msg['header']['type'] = 25 + msg['event'] = 'RTM_DELROUTE' + msg.encode() + self.buffer_queue.put(msg.data) + + return self._get_dump([route], rtmsg) + + def get_addr(self): + return self._get_dump(self.preset['addr'], ifaddrmsg) + + def get_links(self): + return self._get_dump(self.preset['links'], ifinfmsg) + + def get_routes(self): + return self._get_dump(self.preset['routes'], rtmsg) + + +class ChaoticIPRoute: + def __init__(self, *argv, **kwarg): + raise NotImplementedError() + + +class RawIPRoute: + def __init__(self, *argv, **kwarg): + raise NotImplementedError() diff --git a/pyroute2/iproute/linux.py b/pyroute2/iproute/linux.py index ffb6bb1ff..a1f7e698d 100644 --- a/pyroute2/iproute/linux.py +++ b/pyroute2/iproute/linux.py @@ -1,24 +1,34 @@ # -*- coding: utf-8 -*- -import asyncio -import errno -import io import logging import os -import struct import time import warnings from functools import partial +from itertools import chain from socket import AF_INET, AF_INET6, AF_UNSPEC -from pyroute2 import config, netns -from pyroute2.common import AF_MPLS, basestring, get_time -from pyroute2.netlink import NLM_F_ACK, NLM_F_DUMP, NLM_F_REQUEST, NLMSG_ERROR +from pyroute2 import config +from pyroute2.common import AF_MPLS, basestring +from pyroute2.config import AF_BRIDGE +from pyroute2.lab import LAB_API +from pyroute2.netlink import ( + NLM_F_ACK, + NLM_F_APPEND, + NLM_F_ATOMIC, + NLM_F_CREATE, + NLM_F_DUMP, + NLM_F_ECHO, + NLM_F_EXCL, + NLM_F_REPLACE, + NLM_F_REQUEST, + NLM_F_ROOT, + NLMSG_ERROR, +) from pyroute2.netlink.exceptions import ( NetlinkDumpInterrupted, NetlinkError, SkipInode, ) -from pyroute2.netlink.nlsocket import NetlinkRequest, NetlinkSocket from pyroute2.netlink.rtnl import ( RTM_DELADDR, RTM_DELLINK, @@ -46,14 +56,12 @@ RTM_NEWNEIGH, RTM_NEWNETNS, RTM_NEWNSID, - RTM_NEWPROBE, RTM_NEWQDISC, RTM_NEWROUTE, RTM_NEWRULE, RTM_NEWTCLASS, RTM_NEWTFILTER, RTM_SETLINK, - RTMGRP_DEFAULTS, RTMGRP_IPV4_IFADDR, RTMGRP_IPV4_ROUTE, RTMGRP_IPV4_RULE, @@ -63,17 +71,25 @@ RTMGRP_LINK, RTMGRP_MPLS_ROUTE, RTMGRP_NEIGH, + TC_H_ROOT, ndmsg, + rt_proto, + rt_scope, + rt_type, ) from pyroute2.netlink.rtnl.fibmsg import fibmsg from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg from pyroute2.netlink.rtnl.ifstatsmsg import ifstatsmsg -from pyroute2.netlink.rtnl.iprsocket import AsyncIPRSocket, ChaoticIPRSocket +from pyroute2.netlink.rtnl.iprsocket import ( + ChaoticIPRSocket, + IPBatchSocket, + IPRSocket, +) from pyroute2.netlink.rtnl.ndtmsg import ndtmsg from pyroute2.netlink.rtnl.nsidmsg import nsidmsg from pyroute2.netlink.rtnl.nsinfmsg import nsinfmsg -from pyroute2.netlink.rtnl.probe_msg import probe_msg +from pyroute2.netlink.rtnl.riprsocket import RawIPRSocket from pyroute2.netlink.rtnl.rtmsg import rtmsg from pyroute2.netlink.rtnl.tcmsg import plugins as tc_plugins from pyroute2.netlink.rtnl.tcmsg import tcmsg @@ -89,75 +105,23 @@ NeighbourFieldFilter, NeighbourIPRouteFilter, ) -from pyroute2.requests.probe import ProbeFieldFilter from pyroute2.requests.route import RouteFieldFilter, RouteIPRouteFilter from pyroute2.requests.rule import RuleFieldFilter, RuleIPRouteFilter -from pyroute2.requests.tc import TcIPRouteFilter, TcRequestFilter -from .parsers import default_routes, export_routes +from .parsers import default_routes DEFAULT_TABLE = 254 -IPROUTE2_DUMP_MAGIC = 0x45311224 log = logging.getLogger(__name__) -def get_default_request_filters(mode, command): - filters = { - 'link': [LinkFieldFilter(), LinkIPRouteFilter(command)], - 'addr': [AddressFieldFilter(), AddressIPRouteFilter(command)], - 'neigh': [NeighbourFieldFilter(), NeighbourIPRouteFilter(command)], - 'route': [ - RouteFieldFilter(add_defaults=(command not in ('dump', 'show'))), - RouteIPRouteFilter(command), - ], - 'rule': [RuleFieldFilter(), RuleIPRouteFilter(command)], - 'tc': [TcRequestFilter(), TcIPRouteFilter(command)], - 'brport': [BridgePortFieldFilter(command)], - 'vlan_filter': [BridgeFieldFilter(), BridgeIPRouteFilter(command)], - 'probe': [ProbeFieldFilter()], - 'stats': [], - } - return filters[mode] - - -def get_dump_filter(mode, command, query, parameters=None): - if 'dump_filter' in query: - return query.pop('dump_filter'), query - if command not in ('dump', 'show'): - return RequestProcessor(parameters=parameters), query - new_query = {} - if 'family' in query: - new_query['family'] = query.pop('family') - if 'ext_mask' in query: - new_query['ext_mask'] = query.pop('ext_mask') - if 'match' in query: - query = query['match'] - if callable(query): - return query, {} - dump_filter = RequestProcessor( - context=query, prime=query, parameters=parameters - ) - for rf in query.pop( - 'dump_filter', get_default_request_filters(mode, command) - ): - dump_filter.add_filter(rf) - dump_filter.finalize() - return dump_filter, new_query - - -def get_arguments_processor(mode, command, query, parameters=None): - if 'request_filter' in query: - return query['request_filter'] - parameters = parameters or {} - # if not query: - # return RequestProcessor() - processor = RequestProcessor(context=query, prime=query) - for pname, pvalue in parameters.items(): - processor.set_parameter(pname, pvalue) - for rf in get_default_request_filters(mode, command): - processor.add_filter(rf) - processor.finalize() - return processor +def get_dump_filter(kwarg): + if 'match' in kwarg: + return kwarg.pop('match'), kwarg + else: + new_kwarg = {} + if 'family' in kwarg: + new_kwarg['family'] = kwarg.pop('family') + return kwarg, new_kwarg def transform_handle(handle): @@ -168,81 +132,85 @@ def transform_handle(handle): class RTNL_API: - '''A mixin RTNL API class. - - `RTNL_API` should not be instantiated by itself, it is intended + ''' + `RTNL_API` should not be instantiated by itself. It is intended to be used as a mixin class. Following classes use `RTNL_API`: - * `AsyncIPRoute` -- Asynchronous RTNL API - * `IPRoute` -- Synchronous RTNL API - * `NetNS` -- Legace netns-enabled RTNL API + * `IPRoute` -- RTNL API to the current network namespace + * `NetNS` -- RTNL API to another network namespace + * `IPBatch` -- RTNL compiler + * `ShellIPR` -- RTNL via standard I/O, runs IPRoute in a shell - This class was started as iproute2 ip/tc equivalent, but as a - Python API. It does not provide any complicated logic, but instead - runs simple RTNL queries: It is an old-school API, that provides access to rtnetlink as is. It helps you to retrieve and change almost all the data, available - through rtnetlink: - - .. testcode:: cls01 + through rtnetlink:: from pyroute2 import IPRoute - ipr = IPRoute() - # create an interface - ipr.link("add", ifname="brx", kind="bridge") - + ipr.link('add', ifname='brx', kind='bridge') # lookup the index - dev = ipr.link_lookup(ifname="brx")[0] - - # bring it up - ipr.link("set", index=dev, state="up") - + dev = ipr.link_lookup(ifname='brx')[0] + # bring it down + ipr.link('set', index=dev, state='down') # change the interface MAC address and rename it just for fun - ipr.link( - "set", - index=dev, - address="00:11:22:33:44:55", - ifname="br-ctrl", - ) - + ipr.link('set', index=dev, + address='00:11:22:33:44:55', + ifname='br-ctrl') # add primary IP address - ipr.addr( - "add", - index=dev, - address="10.0.0.1", - prefixlen=24, - broadcast="10.0.0.255", - ) + ipr.addr('add', index=dev, + address='10.0.0.1', mask=24, + broadcast='10.0.0.255') # add secondary IP address - ipr.addr( - "add", - index=dev, - address="10.0.0.2", - prefixlen=24, - broadcast="10.0.0.255", - ) + ipr.addr('add', index=dev, + address='10.0.0.2', mask=24, + broadcast='10.0.0.255') + # bring it up + ipr.link('set', index=dev, state='up') + ''' - .. testcode:: cls01 - :hide: + def __init__(self, *argv, **kwarg): + if 'netns_path' in kwarg: + self.netns_path = kwarg['netns_path'] + else: + self.netns_path = config.netns_path + super().__init__(*argv, **kwarg) + if not self.nlm_generator: - br = ipr.link("get", index=dev)[0] - assert br.get("flags") & 1 - try: - assert br.get("flags") & 2 - except AssertionError: - pass - assert br.get("ifname") == "br-ctrl" - assert br.get("address") == "00:11:22:33:44:55" - addr1, addr2 = ipr.addr("dump", index=dev) - assert addr1.get("address") == "10.0.0.1" - assert addr2.get("address") == "10.0.0.2" - assert addr1.get("broadcast") == "10.0.0.255" - assert addr2.get("broadcast") == "10.0.0.255" - assert addr1.get("prefixlen") == 24 - assert addr2.get("prefixlen") == 24 - ''' + def filter_messages(*argv, **kwarg): + return tuple(self._genmatch(*argv, **kwarg)) + + self._genmatch = self.filter_messages + self.filter_messages = filter_messages + + def make_request_type(self, command, command_map): + if isinstance(command, basestring): + return (lambda x: (x[0], self.make_request_flags(x[1])))( + command_map[command] + ) + elif isinstance(command, int): + return command, self.make_request_flags('create') + elif isinstance(command, (list, tuple)): + return command + else: + raise TypeError('allowed command types: int, str, list, tuple') + + def make_request_flags(self, mode): + flags = { + 'dump': NLM_F_REQUEST | NLM_F_DUMP, + 'get': NLM_F_REQUEST | NLM_F_ACK, + 'req': NLM_F_REQUEST | NLM_F_ACK, + } + flags['create'] = flags['req'] | NLM_F_CREATE | NLM_F_EXCL + flags['append'] = flags['req'] | NLM_F_CREATE | NLM_F_APPEND + flags['change'] = flags['req'] | NLM_F_REPLACE + flags['replace'] = flags['change'] | NLM_F_CREATE + + return flags[mode] | ( + NLM_F_ECHO + if (self.config['nlm_echo'] and mode not in ('get', 'dump')) + else 0 + ) def filter_messages(self, dump_filter, msgs): ''' @@ -257,72 +225,76 @@ def filter_messages(self, dump_filter, msgs): A callable `dump_filter` must return True or False: - - .. testcode:: fm01 + .. code-block:: python # get all links with names starting with eth: # - for link in ipr.filter_messages( - lambda x: x.get("ifname").startswith("eth"), - ipr.link("dump"), - ): - print(link.get("ifname")) - - .. testoutput:: fm01 - - eth0 + ipr.filter_messages( + lambda x: x.get_attr('IFLA_IFNAME').startswith('eth'), + ipr.link('dump') + ) A dict `dump_filter` can have callables as values: - .. testcode:: fm02 + .. code-block:: python # get all links with names starting with eth, and # MAC address in a database: # - database = [ - "52:54:00:72:58:b2", - ] - - for link in ipr.filter_messages( + ipr.filter_messages( { - "ifname": lambda x: x.startswith("eth"), - "address": lambda x: x in database, + 'ifname': lambda x: x.startswith('eth'), + 'address': lambda x: x in database, }, - ipr.link("dump"), - ): - print(link.get("ifname")) - - .. testoutput:: fm02 - - eth0 + ipr.link('dump') + ) ... or constants to compare with: - .. testcode:: fm03 + .. code-block:: python # get all links in state up: # - for link in ipr.filter_messages( - {"state": "up"}, ipr.link("dump"), - ): - print(link.get("ifname")) - - .. testoutput:: fm03 - - lo - eth0 + ipr.filter_message({'state': 'up'}, ipr.link('dump')) ''' # filtered results, the generator version for msg in msgs: - if NetlinkRequest.match_one_message(dump_filter, msg): - yield msg + if hasattr(dump_filter, '__call__'): + if dump_filter(msg): + yield msg + elif isinstance(dump_filter, dict): + matches = [] + for key in dump_filter: + # get the attribute + if isinstance(key, str): + nkey = (key,) + elif isinstance(key, tuple): + nkey = key + else: + continue + value = msg.get_nested(*nkey) + if value is not None and callable(dump_filter[key]): + matches.append(dump_filter[key](value)) + else: + matches.append(dump_filter[key] == value) + if all(matches): + yield msg # 8<--------------------------------------------------------------- # - async def dump(self, groups=None): + def dump(self, groups=None): ''' Dump network objects. + On OpenBSD: + + * get_links() + * get_addr() + * get_neighbours() + * get_routes() + + On Linux: + * get_links() * get_addr() * get_neighbours() @@ -330,100 +302,46 @@ async def dump(self, groups=None): * dump FDB * IPv4 and IPv6 rules ''' - groups_map = { - RTMGRP_LINK: [ - self.get_links, - self.get_vlans, - partial(self.fdb, 'dump'), - ], - RTMGRP_IPV4_IFADDR: [partial(self.get_addr, family=AF_INET)], - RTMGRP_IPV6_IFADDR: [partial(self.get_addr, family=AF_INET6)], - RTMGRP_NEIGH: [self.get_neighbours], - RTMGRP_IPV4_ROUTE: [partial(self.get_routes, family=AF_INET)], - RTMGRP_IPV6_ROUTE: [partial(self.get_routes, family=AF_INET6)], - RTMGRP_MPLS_ROUTE: [partial(self.get_routes, family=AF_MPLS)], - RTMGRP_IPV4_RULE: [partial(self.get_rules, family=AF_INET)], - RTMGRP_IPV6_RULE: [partial(self.get_rules, family=AF_INET6)], - } - - async def ret(): - for group, methods in groups_map.items(): - if group & (groups if groups is not None else self.groups): - for method in methods: - async for msg in await method(): - yield msg - - return ret() - - async def ensure( - self, method, present=True, timeout=10, interval=0.2, **spec - ): - '''Ensure object's state. - - The issue with RTNL calls is that they are not synchronous: - even if the kernel returns success for a call, the changes - may become visible after some short time. Because of that - adding dependant object immediately one after another may fail. - Say, adding a route directly after the required interface - address. - - Since pyroute2 RTNL API is more or less one to one mapping - of the kernel RTNL, it has the same problem. - - This method aims to mitigate this issue. - - * if `present == True`, try to add/set an object by the spec. - * if `present == False`, try to remove - * and finally wait up to `timeout` for the changes to be applied. - - Example: - - .. testcode:: - - interface = ipr.ensure(ipr.link, - present=True, - ifname='test0', - kind='dummy', - state='up', - ) - ipr.ensure(ipr.addr, - present=True, - index=interface, - address='192.168.0.2/24', - ) - ''' - state = [x async for x in await method('dump', **spec)] - if present: - if state: - return state - try: - await method('add', **spec) - except NetlinkError as e: - if e.code != errno.EEXIST: - raise - await method('set', **spec) + ## + # Well, it's the Linux API, why OpenBSD / FreeBSD here? + # + # 'Cause when you run RemoteIPRoute, it uses this class, + # and the code may be run on BSD systems as well, though + # BSD systems have only subset of the API + # + if self.uname[0] == 'OpenBSD': + groups_map = { + 1: [ + self.get_links, + self.get_addr, + self.get_neighbours, + self.get_routes, + ] + } else: - if not state: - return state - try: - await method('del', **spec) - except NetlinkError as e: - if e.code not in ( - errno.ENODEV, - errno.ENOENT, - errno.ENONET, - errno.EADDRNOTAVAIL, - ): - raise - return await self.poll( - method, 'dump', present, timeout, interval, **spec - ) - - async def poll( - self, method, command, present=True, timeout=10, interval=0.2, **spec - ): - '''Wait for a method to succeed. + groups_map = { + RTMGRP_LINK: [ + self.get_links, + self.get_vlans, + partial(self.fdb, 'dump'), + ], + RTMGRP_IPV4_IFADDR: [partial(self.get_addr, family=AF_INET)], + RTMGRP_IPV6_IFADDR: [partial(self.get_addr, family=AF_INET6)], + RTMGRP_NEIGH: [self.get_neighbours], + RTMGRP_IPV4_ROUTE: [partial(self.get_routes, family=AF_INET)], + RTMGRP_IPV6_ROUTE: [partial(self.get_routes, family=AF_INET6)], + RTMGRP_MPLS_ROUTE: [partial(self.get_routes, family=AF_MPLS)], + RTMGRP_IPV4_RULE: [partial(self.get_rules, family=AF_INET)], + RTMGRP_IPV6_RULE: [partial(self.get_rules, family=AF_INET6)], + } + for group, methods in groups_map.items(): + if group & (groups if groups is not None else self.groups): + for method in methods: + for msg in method(): + yield msg + def poll(self, method, command, timeout=10, interval=0.2, **spec): + ''' Run `method` with a positional argument `command` and keyword arguments `**spec` every `interval` seconds, but not more than `timeout`, until it returns a result which doesn't evaluate to @@ -431,7 +349,7 @@ async def poll( Example: - .. testcode:: p0 + .. code-block:: python # create a bridge interface and wait for it: # @@ -439,200 +357,45 @@ async def poll( 'ifname': 'br0', 'kind': 'bridge', 'state': 'up', + 'br_stp_state': 1, } ipr.link('add', **spec) ret = ipr.poll(ipr.link, 'dump', **spec) assert ret[0].get('ifname') == 'br0' - assert ret[0].get('flags') & 1 assert ret[0].get('state') == 'up' - assert ret[0].get(('linkinfo', 'kind')) == 'bridge' - - .. testcode:: p1 - :hide: - - try: - ipr.poll(ipr.link, 'dump', ifname='br1') - except TimeoutError: - pass + assert ret[0].get(('linkinfo', 'data', 'br_stp_state')) == 1 ''' - ctime = get_time() + ctime = time.time() ret = tuple() - while ctime + timeout > get_time(): + while ctime + timeout > time.time(): try: - ret = await method(command, **spec) - if not isinstance(ret, list): - ret = [x async for x in ret] - if (ret and present) or (not ret and not present): + ret = method(command, **spec) + if ret: return ret - await asyncio.sleep(interval) + time.sleep(interval) except NetlinkDumpInterrupted: pass - raise asyncio.TimeoutError() - - # 8<--------------------------------------------------------------- - # - # Diagnostics - # - async def probe(self, command, **kwarg): - '''Run a network probe. - - The API will trigger a network probe from the environment it - works in. For NetNS it will be the network namespace, for - remote IPRoute instances it will be the host it runs on. - - Running probes via API allows to test network connectivity - between the environments in a simple uniform way. - - Supported arguments: - - * kind -- probe type, for now only ping is supported - * dst -- target to run the probe against - * num -- number of probes to run - * timeout -- timeout for the whole request - - Examples:: - - ipr.probe("add", kind="ping", dst="10.0.0.1") - - By default ping probe will send one ICMP request towards - the target. To change this, use num argument:: - - ipr.probe( - "add", - kind="ping", - dst="10.0.0.1", - num=4, - timeout=10 - ) - - Timeout for the ping probe by default is 1 second, which - may not be enough to run multiple requests. - - In the next release more probe types are planned, like TCP - port probe. - ''' - msg = probe_msg() - arguments = get_arguments_processor('probe', command, kwarg) - request = NetlinkRequest( - self, - msg, - msg_type=RTM_NEWPROBE, - msg_flags=1, - request_filter=arguments, - ) - await request.send() - return [x async for x in request.response()] - - # 8<--------------------------------------------------------------- - # - # Binary streams methods - # - async def route_dump(self, fd, family=AF_UNSPEC, fmt='iproute2'): - '''Save routes as a binary dump into a file object. - - fd -- an open file object, must support `write()` - family -- AF_UNSPEC, AF_INET, etc. -- filter routes by family - fmt -- dump format, "iproute2" (default) or "raw" - - The binary dump is just a set of unparsed netlink messages. - The `iproute2` prepends the dump with a magic uint32, so - `IPRoute` does the same for compatibility. If you want a raw - dump without any additional magic data, use `fmt="raw"`. - - This routine neither close the file object, nor uses `seek()` - to rewind, it's up to the user. - ''' - - if fmt == 'iproute2': - fd.write(struct.pack('I', IPROUTE2_DUMP_MAGIC)) - elif fmt != 'raw': - raise TypeError('dump format not supported') - msg = rtmsg() - msg['family'] = family - request = NetlinkRequest( - self, - msg, - msg_type=RTM_GETROUTE, - msg_flags=NLM_F_DUMP | NLM_F_REQUEST, - parser=export_routes(fd), - ) - await request.send() - return [x async for x in request.response()] - - async def route_dumps(self, family=AF_UNSPEC, fmt='iproute2'): - '''Save routes and returns as a `bytes` object. - - The same as `.route_dump()`, but returns `bytes`. - ''' - fd = io.BytesIO() - await self.route_dump(fd, family, fmt) - return fd.getvalue() - - async def route_load(self, fd, fmt='iproute2'): - '''Load routes from a binary dump. - - fd -- an open file object, must support `read()` - fmt -- dump format, "iproute2" (default) or "raw" - - The current version parses the dump and loads routes one - by one. This behavior will be changed in the future to - optimize the performance, but the result will be the same. - - If `fmt == "iproute2"`, then the loader checks the magic iproute2 - prefix in the dump. Otherwise it parses the data from byte 0. - ''' - if fmt == 'iproute2': - if ( - not struct.unpack('I', fd.read(struct.calcsize('I')))[0] - == IPROUTE2_DUMP_MAGIC - ): - raise TypeError('wrong dump magic') - elif fmt != 'raw': - raise TypeError('dump format not supported') - ret = [] - for msg in self.marshal.parse(fd.read()): - request = NetlinkRequest( - self, - msg, - command='replace', - command_map={'replace': (RTM_NEWROUTE, 'replace')}, - ) - await request.send() - ret.extend( - [ - x['header']['error'] is None - async for x in request.response() - ] - ) - if not all(ret): - raise NetlinkError('error loading route dump') - return [] - - async def route_loads(self, data, fmt='iproute2'): - '''Load routes from a `bytes` object. - - Like `.route_load()`, but accepts `bytes` instead of an file file. - ''' - fd = io.BytesIO() - fd.write(data) - fd.seek(0) - return await self.route_load(fd, fmt) + raise TimeoutError() # 8<--------------------------------------------------------------- # # Listing methods # - async def get_qdiscs(self, index=None): + def get_qdiscs(self, index=None): ''' - Get all queue disciplines for all interfaces or for - the selected one. - - A compatibility method, == .tc("dump") + Get all queue disciplines for all interfaces or for specified + one. ''' - return await self.tc('dump', index=index) + msg = tcmsg() + msg['family'] = AF_UNSPEC + ret = self.nlm_request(msg, RTM_GETQDISC) + if index is None: + return tuple(ret) + else: + return [x for x in ret if x['index'] == index] - async def get_filters(self, index=0, handle=0, parent=0): + def get_filters(self, index=0, handle=0, parent=0): ''' Get filters for specified interface, handle and parent. ''' @@ -641,22 +404,18 @@ async def get_filters(self, index=0, handle=0, parent=0): msg['index'] = index msg['handle'] = transform_handle(handle) msg['parent'] = transform_handle(parent) - request = NetlinkRequest(self, msg, msg_type=RTM_GETTFILTER) - await request.send() - return request.response() + return tuple(self.nlm_request(msg, RTM_GETTFILTER)) - async def get_classes(self, index=0): + def get_classes(self, index=0): ''' Get classes for specified interface. ''' msg = tcmsg() msg['family'] = AF_UNSPEC msg['index'] = index - request = NetlinkRequest(self, msg, msg_type=RTM_GETTCLASS) - await request.send() - return request.response() + return tuple(self.nlm_request(msg, RTM_GETTCLASS)) - async def get_vlans(self, **kwarg): + def get_vlans(self, **kwarg): ''' Dump available vlan info on bridge ports ''' @@ -670,11 +429,12 @@ async def get_vlans(self, **kwarg): # # maybe place it as mapping into ifinfomsg.py? # - return await self.link( - 'dump', family=config.AF_BRIDGE, ext_mask=2, match=kwarg + dump_filter, kwarg = get_dump_filter(kwarg) + return self.link( + 'dump', family=AF_BRIDGE, ext_mask=2, match=dump_filter ) - async def get_links(self, *argv, **kwarg): + def get_links(self, *argv, **kwarg): ''' Get network interfaces. @@ -689,21 +449,23 @@ async def get_links(self, *argv, **kwarg): interfaces = [1, 2, 3] ip.get_links(*interfaces) ''' + result = [] links = argv or [0] if links[0] == 'all': # compat syntax links = [0] if links[0] == 0: - return await self.link('dump', **kwarg) - - async def dump(): - for index in links: - for link in await self.link('get', index=index, **kwarg): - yield link + cmd = 'dump' + else: + cmd = 'get' - return dump() + for index in links: + if index > 0: + kwarg['index'] = index + result.extend(self.link(cmd, **kwarg)) + return result - async def get_neighbours(self, family=AF_UNSPEC, match=None, **kwarg): + def get_neighbours(self, family=AF_UNSPEC, match=None, **kwarg): ''' Dump ARP cache records. @@ -729,19 +491,17 @@ async def get_neighbours(self, family=AF_UNSPEC, match=None, **kwarg): # and filter them by a function: ip.get_neighbours(AF_BRIDGE, match=lambda x: x['state'] == 2) ''' - return await self.neigh('dump', family=family, match=match or kwarg) + return self.neigh('dump', family=family, match=match or kwarg) - async def get_ntables(self, family=AF_UNSPEC): + def get_ntables(self, family=AF_UNSPEC): ''' Get neighbour tables ''' msg = ndtmsg() msg['family'] = family - request = NetlinkRequest(self, msg, msg_type=RTM_GETNEIGHTBL) - await request.send() - return request.response() + return tuple(self.nlm_request(msg, RTM_GETNEIGHTBL)) - async def get_addr(self, family=AF_UNSPEC, match=None, **kwarg): + def get_addr(self, family=AF_UNSPEC, match=None, **kwarg): ''' Dump addresses. @@ -768,9 +528,9 @@ async def get_addr(self, family=AF_UNSPEC, match=None, **kwarg): ip.get_addr(match=lambda x: x['index'] == 1) ''' - return await self.addr('dump', family=family, match=match or kwarg) + return self.addr('dump', family=family, match=match or kwarg) - async def get_rules(self, family=AF_UNSPEC, match=None, **kwarg): + def get_rules(self, family=AF_UNSPEC, match=None, **kwarg): ''' Get all rules. By default return all rules. To explicitly request the IPv4 rules use `family=AF_INET`. @@ -779,9 +539,13 @@ async def get_rules(self, family=AF_UNSPEC, match=None, **kwarg): ip.get_rules() # get all the rules for all families ip.get_rules(family=AF_INET6) # get only IPv6 rules ''' - return await self.rule('dump', family=family, match=match or kwarg) + return self.rule( + (RTM_GETRULE, NLM_F_REQUEST | NLM_F_ROOT | NLM_F_ATOMIC), + family=family, + match=match or kwarg, + ) - async def get_routes(self, family=255, match=None, **kwarg): + def get_routes(self, family=255, match=None, **kwarg): ''' Get all routes. You can specify the table. There are up to 4294967295 routing classes (tables), and the kernel @@ -801,18 +565,11 @@ async def get_routes(self, family=255, match=None, **kwarg): uses an invalid value here. Hack but true. And let's hope the kernel team will not fix this bug. ''' - # get a particular route? - async def dump(dst): - for route in await self.route('get', dst=dst): - yield route - if isinstance(kwarg.get('dst'), str): - return dump(kwarg['dst']) + return self.route('get', dst=kwarg['dst']) else: - return await self.route( - 'dump', family=family, match=match or kwarg - ) + return self.route('dump', family=family, match=match or kwarg) # 8<--------------------------------------------------------------- @@ -846,7 +603,7 @@ def list_link_kind(self): # # List NetNS info # - async def _dump_one_ns(self, path, registry): + def _dump_one_ns(self, path, registry): item = nsinfmsg() item['netnsid'] = 0xFFFFFFFF # default netnsid "unknown" nsfd = 0 @@ -867,12 +624,8 @@ async def _dump_one_ns(self, path, registry): # may not work on older kernels ( <4.20 ?) # msg['attrs'] = [('NETNSA_FD', nsfd)] - request = NetlinkRequest( - self, msg, msg_type=RTM_GETNSID, msg_flags=NLM_F_REQUEST - ) - await request.send() try: - async for info in request.response(): + for info in self.nlm_request(msg, RTM_GETNSID, NLM_F_REQUEST): # response to nlm_request() is a list or a generator, # that's why loop item['netnsid'] = info.get_attr('NETNSA_NSID') @@ -890,17 +643,17 @@ async def _dump_one_ns(self, path, registry): item['event'] = 'RTM_NEWNETNS' return item - async def _dump_dir(self, path, registry): + def _dump_dir(self, path, registry): for name in os.listdir(path): # strictly speaking, there is no need to use os.sep, # since the code is not portable outside of Linux nspath = '%s%s%s' % (path, os.sep, name) try: - yield await self._dump_one_ns(nspath, registry) + yield self._dump_one_ns(nspath, registry) except SkipInode: pass - async def _dump_proc(self, registry): + def _dump_proc(self, registry): for name in os.listdir('/proc'): try: int(name) @@ -908,15 +661,11 @@ async def _dump_proc(self, registry): continue try: - yield await self._dump_one_ns( - '/proc/%s/ns/net' % name, registry - ) + yield self._dump_one_ns('/proc/%s/ns/net' % name, registry) except SkipInode: pass - async def get_netnsid( - self, nsid=None, pid=None, fd=None, target_nsid=None - ): + def get_netnsid(self, nsid=None, pid=None, fd=None, target_nsid=None): '''Return a dict containing the result of a RTM_GETNSID query. This loosely corresponds to the "ip netns list-id" command. ''' @@ -934,11 +683,8 @@ async def get_netnsid( if target_nsid is not None: msg['attrs'].append(('NETNSA_TARGET_NSID', target_nsid)) - request = NetlinkRequest( - self, msg, msg_type=RTM_GETNSID, msg_flags=NLM_F_REQUEST - ) - await request.send() - async for r in request.response(): + response = self.nlm_request(msg, RTM_GETNSID, NLM_F_REQUEST) + for r in response: return { 'nsid': r.get_attr('NETNSA_NSID'), 'current_nsid': r.get_attr('NETNSA_CURRENT_NSID'), @@ -946,7 +692,7 @@ async def get_netnsid( return None - async def get_netns_info(self, list_proc=False): + def get_netns_info(self, list_proc=False): ''' A prototype method to list available netns and associated interfaces. A bit weird to have it here and not under @@ -960,7 +706,7 @@ async def get_netns_info(self, list_proc=False): # fetch veth peers # peers = {} - async for peer in await self.link('dump'): + for peer in self.get_links(): netnsid = peer.get_attr('IFLA_LINK_NETNSID') if netnsid is not None: if netnsid not in peers: @@ -973,32 +719,28 @@ async def get_netns_info(self, list_proc=False): # * one iterator for /proc//ns/net # views = [] - for path in self.status['netns_path']: + for path in self.netns_path: views.append(self._dump_dir(path, registry)) if list_proc: views.append(self._dump_proc(registry)) - # # iterate all the items # - async def ret(): - for view in views: - try: - async for item in view: - # - # remove uninitialized 'value' field - # - del item['value'] - # - # fetch peers for that ns - # - for peer in peers.get(item['netnsid'], []): - item['attrs'].append(('NSINFO_PEER', peer)) - yield item - except OSError: - pass - - return ret() + for view in views: + try: + for item in view: + # + # remove uninitialized 'value' field + # + del item['value'] + # + # fetch peers for that ns + # + for peer in peers.get(item['netnsid'], []): + item['attrs'].append(('NSINFO_PEER', peer)) + yield item + except OSError: + pass def set_netnsid(self, nsid=None, pid=None, fd=None): '''Assigns an id to a peer netns using RTM_NEWNSID query. @@ -1027,26 +769,26 @@ def set_netnsid(self, nsid=None, pid=None, fd=None): # # Shortcuts # - async def get_default_routes(self, family=AF_UNSPEC, table=DEFAULT_TABLE): + def get_default_routes(self, family=AF_UNSPEC, table=DEFAULT_TABLE): ''' Get default routes ''' msg = rtmsg() msg['family'] = family - dump_filter, _ = get_dump_filter( - 'route', 'dump', {'table': table} if table is not None else {} - ) - request = NetlinkRequest( - self, + + routes = self.nlm_request( msg, msg_type=RTM_GETROUTE, msg_flags=NLM_F_DUMP | NLM_F_REQUEST, parser=default_routes, ) - await request.send() - return request.response() - async def link_lookup(self, match=None, **kwarg): + if table is None: + return routes + else: + return self.filter_messages({'table': table}, routes) + + def link_lookup(self, match=None, **kwarg): ''' Lookup interface index (indeces) by first level NLA value. @@ -1063,15 +805,14 @@ async def link_lookup(self, match=None, **kwarg): if kwarg and set(kwarg) < {'index', 'ifname', 'altname'}: # shortcut for index and ifname try: - for link in await self.link('get', **kwarg): + for link in self.link('get', **kwarg): return [link['index']] - except (NetlinkError, KeyError): + except NetlinkError: return [] else: # otherwise fallback to the userspace filter return [ - link['index'] - async for link in await self.get_links(match=match or kwarg) + link['index'] for link in self.get_links(match=match or kwarg) ] # 8<--------------------------------------------------------------- @@ -1080,7 +821,7 @@ async def link_lookup(self, match=None, **kwarg): # # Shortcuts to flush RTNL objects # - async def flush_routes(self, *argv, **kwarg): + def flush_routes(self, *argv, **kwarg): ''' Flush routes -- purge route records from a table. Arguments are the same as for `get_routes()` @@ -1088,18 +829,12 @@ async def flush_routes(self, *argv, **kwarg): `get_routes()` to `nlm_request()`. ''' ret = [] - async for route in await self.get_routes(*argv, **kwarg): - request = NetlinkRequest( - self, - route, - msg_type=RTM_DELROUTE, - msg_flags=NLM_F_REQUEST | NLM_F_ACK, - ) - await request.send() - ret.extend([y async for y in request.response()]) + for route in self.get_routes(*argv, **kwarg): + self.put(route, msg_type=RTM_DELROUTE, msg_flags=NLM_F_REQUEST) + ret.append(route) return ret - async def flush_addr(self, *argv, **kwarg): + def flush_addr(self, *argv, **kwarg): ''' Flush IP addresses. @@ -1111,25 +846,14 @@ async def flush_addr(self, *argv, **kwarg): # flush all addresses with IFA_LABEL='eth0': ipr.flush_addr(label='eth0') ''' + flags = NLM_F_CREATE | NLM_F_REQUEST ret = [] - work = [] - async for addr in await self.get_addr(*argv, **kwarg): - work.append( - { - 'index': addr.get('index'), - 'address': addr.get('address'), - 'prefixlen': addr.get('prefixlen'), - } - ) - for addr in work: - try: - ret.extend(await self.addr('del', **addr)) - except NetlinkError: - if not ret: - raise + for addr in self.get_addr(*argv, **kwarg): + self.put(addr, msg_type=RTM_DELADDR, msg_flags=flags) + ret.append(addr) return ret - async def flush_rules(self, *argv, **kwarg): + def flush_rules(self, *argv, **kwarg): ''' Flush rules. Please keep in mind, that by default the function operates on **all** rules of **all** families. To work only on @@ -1143,16 +867,11 @@ async def flush_rules(self, *argv, **kwarg): # flush all IPv6 rules that point to table 250: ipr.flush_rules(family=socket.AF_INET6, table=250) ''' + flags = NLM_F_CREATE | NLM_F_REQUEST ret = [] - for rule in tuple([x async for x in await self.rule('dump', **kwarg)]): - request = NetlinkRequest( - self, - rule, - msg_type=RTM_DELRULE, - msg_flags=NLM_F_REQUEST | NLM_F_ACK, - ) - await request.send() - ret.extend([y async for y in request.response()]) + for rule in self.get_rules(*argv, **kwarg): + self.put(rule, msg_type=RTM_DELRULE, msg_flags=flags) + ret.append(rule) return ret # 8<--------------------------------------------------------------- @@ -1161,7 +880,7 @@ async def flush_rules(self, *argv, **kwarg): # # Extensions to low-level functions # - async def brport(self, command, **kwarg): + def brport(self, command, **kwarg): ''' Set bridge port parameters. Example:: @@ -1172,38 +891,50 @@ async def brport(self, command, **kwarg): Possible keywords are NLA names for the `protinfo_bridge` class, without the prefix and in lower letters. ''' - - # forward the set command to link() if command == 'set': linkkwarg = dict() linkkwarg['index'] = kwarg.pop('index', 0) linkkwarg['kind'] = 'bridge_slave' for key in kwarg: linkkwarg[key] = kwarg[key] - return await self.link(command, **linkkwarg) + return self.link(command, **linkkwarg) + if (command in ('dump', 'show')) and ('match' not in kwarg): + match = kwarg + else: + match = kwarg.pop('match', None) command_map = { 'dump': (RTM_GETLINK, 'dump'), 'show': (RTM_GETLINK, 'dump'), } - if command not in command_map: - raise TypeError('command not supported') - dump_filter, kwarg = get_dump_filter('brport', command, kwarg) - arguments = get_arguments_processor('brport', command, kwarg) - request = NetlinkRequest( - self, ifinfmsg(), command, command_map, dump_filter, arguments + (command, msg_flags) = self.make_request_type(command, command_map) + + msg = ifinfmsg() + msg['index'] = kwarg.get('index', 0) + msg['family'] = AF_BRIDGE + protinfo = ( + RequestProcessor(context=match, prime=match) + .apply_filter(BridgePortFieldFilter(command)) + .finalize() + ) + msg['attrs'].append( + ('IFLA_PROTINFO', {'attrs': protinfo['attrs']}, 0x8000) ) - await request.send() - return request.response() + ret = self.nlm_request(msg, msg_type=command, msg_flags=msg_flags) + if match is not None: + ret = self.filter_messages(match, ret) + + if self.nlm_generator and not msg_flags & NLM_F_DUMP == NLM_F_DUMP: + ret = tuple(ret) + + return ret - async def vlan_filter(self, command, **kwarg): + def vlan_filter(self, command, **kwarg): ''' Vlan filters is another approach to support vlans in Linux. Before vlan filters were introduced, there was only one way to bridge vlans: one had to create vlan interfaces and - then add them as ports: - - .. aafig:: + then add them as ports:: +------+ +----------+ net --> | eth0 | <--> | eth0.500 | <---+ @@ -1219,9 +950,7 @@ async def vlan_filter(self, command, **kwarg): It means that one has to create as many bridges, as there were vlans. Vlan filters allow to bridge together underlying interfaces - and create vlans already on the bridge: - - .. aafig:: + and create vlans already on the bridge:: # v500 label shows which interfaces have vlan filter @@ -1367,15 +1096,17 @@ async def vlan_filter(self, command, **kwarg): 'add': (RTM_SETLINK, 'req'), 'del': (RTM_DELLINK, 'req'), } - kwarg['family'] = config.AF_BRIDGE - kwarg['command_map'] = command_map - kwarg['dump_filter'] = None - kwarg['request_filter'] = get_arguments_processor( - 'vlan_filter', command, kwarg - ) - return await self.link(command, **kwarg) - async def fdb(self, command, **kwarg): + kwarg['family'] = AF_BRIDGE + kwarg['kwarg_filter'] = [ + BridgeFieldFilter(), + BridgeIPRouteFilter(command), + ] + + (command, flags) = self.make_request_type(command, command_map) + return tuple(self.link((command, flags), **kwarg)) + + def fdb(self, command, **kwarg): ''' Bridge forwarding database management. @@ -1452,7 +1183,11 @@ async def fdb(self, command, **kwarg): ip.fdb('dump', vlan=200) ''' - kwarg['family'] = config.AF_BRIDGE + dump_filter = None + if command == 'dump': + dump_filter, kwarg = get_dump_filter(kwarg) + + kwarg['family'] = AF_BRIDGE # nud -> state if 'nud' in kwarg: kwarg['state'] = kwarg.pop('nud') @@ -1473,13 +1208,15 @@ async def fdb(self, command, **kwarg): # self (default) or master kwarg['flags'] = kwarg.get('flags', 0) | ndmsg.flags['self'] # - return await self.neigh(command, **kwarg) + if dump_filter is not None: + kwarg['match'] = dump_filter + return self.neigh(command, **kwarg) # 8<--------------------------------------------------------------- # # General low-level configuration methods # - async def neigh(self, command, **kwarg): + def neigh(self, command, **kwarg): ''' Neighbours operations, same as `ip neigh` or `bridge fdb` @@ -1545,19 +1282,46 @@ async def neigh(self, command, **kwarg): 'get': (RTM_GETNEIGH, 'get'), 'append': (RTM_NEWNEIGH, 'append'), } - if isinstance(kwarg.get('match'), str): - kwarg['match'] = {'ifname': kwarg['match']} - dump_filter, kwarg = get_dump_filter('neigh', command, kwarg) - arguments = get_arguments_processor('neigh', command, kwarg) - request = NetlinkRequest( - self, ndmsg.ndmsg(), command, command_map, dump_filter, arguments - ) - await request.send() + dump_filter = None + msg = ndmsg.ndmsg() if command == 'dump': - return request.response() - return [x async for x in request.response()] + dump_filter, kwarg = get_dump_filter(kwarg) + + request = ( + RequestProcessor(context=kwarg, prime=kwarg) + .apply_filter(NeighbourFieldFilter()) + .apply_filter(NeighbourIPRouteFilter(command)) + .finalize() + ) + msg_type, msg_flags = self.make_request_type(command, command_map) - async def link(self, command, **kwarg): + # fill the fields + for field in msg.fields: + if ( + command == "dump" + and self.strict_check + and field[0] == "ifindex" + ): + # is dump & strict_check, leave ifindex for NLA + continue + msg[field[0]] = request.pop(field[0], 0) + + for key, value in request.items(): + nla = ndmsg.ndmsg.name2nla(key) + if msg.valid_nla(nla) and value is not None: + msg['attrs'].append([nla, value]) + + ret = self.nlm_request(msg, msg_type=msg_type, msg_flags=msg_flags) + + if command == 'dump' and dump_filter: + ret = self.filter_messages(dump_filter, ret) + + if self.nlm_generator and not msg_flags & NLM_F_DUMP == NLM_F_DUMP: + ret = tuple(ret) + + return ret + + def link(self, command, **kwarg): ''' Link operations. @@ -1837,8 +1601,8 @@ async def link(self, command, **kwarg): Keyword "state" is reserved. State can be "up" or "down", it is a shortcut:: - state="up": flags=1, change=1 - state="down": flags=0, change=1 + state="up": flags=1, mask=1 + state="down": flags=0, mask=0 SR-IOV virtual function setup:: @@ -1901,21 +1665,52 @@ async def link(self, command, **kwarg): 'dump': (RTM_GETLINK, 'dump'), 'get': (RTM_GETLINK, 'get'), } - if isinstance(kwarg.get('match'), str): - kwarg['match'] = {'ifname': kwarg['match']} - if 'command_map' in kwarg: - command_map = kwarg.pop('command_map') - dump_filter, kwarg = get_dump_filter('link', command, kwarg) - arguments = get_arguments_processor('link', command, kwarg) - request = NetlinkRequest( - self, ifinfmsg(), command, command_map, dump_filter, arguments - ) - await request.send() + dump_filter = None + request = {} + msg = ifinfmsg() + if command == 'dump': - return request.response() - return [x async for x in request.response()] + dump_filter, kwarg = get_dump_filter(kwarg) + + if kwarg: + if kwarg.get('kwarg_filter'): + filters = kwarg['kwarg_filter'] + else: + filters = [LinkFieldFilter(), LinkIPRouteFilter(command)] + request = RequestProcessor(context=kwarg, prime=kwarg) + for rfilter in filters: + request.apply_filter(rfilter) + request.finalize() + + msg_type, msg_flags = self.make_request_type(command, command_map) + + for field in msg.fields: + msg[field[0]] = request.pop(field[0], 0) + + # attach NLA + for key, value in request.items(): + nla = type(msg).name2nla(key) + if msg.valid_nla(nla) and value is not None: + msg['attrs'].append([nla, value]) + + ret = self.nlm_request(msg, msg_type=msg_type, msg_flags=msg_flags) + + if command == 'dump' and dump_filter is not None: + if isinstance(dump_filter, dict): + dump_filter = ( + RequestProcessor(context=dump_filter, prime=dump_filter) + .apply_filter(LinkFieldFilter()) + .apply_filter(LinkIPRouteFilter('dump')) + .finalize() + ) + ret = self.filter_messages(dump_filter, ret) - async def addr(self, command, **kwarg): + if self.nlm_generator and not msg_flags & NLM_F_DUMP == NLM_F_DUMP: + ret = tuple(ret) + + return ret + + def addr(self, command, *argv, **kwarg): ''' Address operations @@ -1964,7 +1759,20 @@ def addr(self, command, match=None, **kwarg): local='10.1.1.1') ''' if command in ('get', 'set'): - return + return [] + ## + # This block will be deprecated in a short term + if argv: + warnings.warn( + 'positional arguments for IPRoute.addr() are deprecated, ' + 'use keyword arguments', + DeprecationWarning, + ) + converted_argv = zip( + ('index', 'address', 'prefixlen', 'family', 'scope', 'match'), + argv, + ) + kwarg.update(converted_argv) if 'mask' in kwarg: warnings.warn( 'usage of mask is deprecated, use prefixlen instead', @@ -1978,27 +1786,47 @@ def addr(self, command, match=None, **kwarg): 'replace': (RTM_NEWADDR, 'replace'), 'dump': (RTM_GETADDR, 'dump'), } - # quirks in the filter: flags are supplied as NLA, not as a field - dump_filter, kwarg = get_dump_filter('addr', command, kwarg) - arguments = get_arguments_processor('addr', command, kwarg) - request = NetlinkRequest( - self, - ifaddrmsg(), - command, - command_map, - dump_filter, - arguments, + dump_filter = None + msg = ifaddrmsg() + if command == 'dump': + dump_filter, kwarg = get_dump_filter(kwarg) + + request = ( + RequestProcessor(context=kwarg, prime=kwarg) + .apply_filter(AddressFieldFilter()) + .apply_filter(AddressIPRouteFilter(command)) + .finalize() + ) + msg_type, msg_flags = self.make_request_type(command, command_map) + + for field in msg.fields: + if field[0] != 'flags': # Flags are supplied as NLA + msg[field[0]] = request.pop(field[0], 0) + + # work on NLA + for key, value in request.items(): + nla = ifaddrmsg.name2nla(key) + if msg.valid_nla(nla) and value is not None: + msg['attrs'].append([nla, value]) + + ret = self.nlm_request( + msg, + msg_type=msg_type, + msg_flags=msg_flags, terminate=lambda x: x['header']['type'] == NLMSG_ERROR, ) - await request.send() - if command == 'dump': - return request.response() - return [x async for x in request.response()] + if command == 'dump' and dump_filter is not None: + ret = self.filter_messages(dump_filter, ret) + + if self.nlm_generator and not msg_flags & NLM_F_DUMP == NLM_F_DUMP: + ret = tuple(ret) + + return ret - async def tc(self, command, kind=None, index=None, handle=None, **kwarg): + def tc(self, command, kind=None, index=0, handle=0, **kwarg): ''' "Swiss knife" for traffic control. With the method you can - dump, add, delete or modify qdiscs, classes and filters. + add, delete or modify qdiscs, classes and filters. * command -- add or delete qdisc, class, filter. * kind -- a string identifier -- "sfq", "htb", "u32" and so on. @@ -2070,52 +1898,63 @@ async def tc(self, command, kind=None, index=None, handle=None, **kwarg): return 'No help available' command_map = { - 'dump': (RTM_GETQDISC, 'dump'), - 'get': (RTM_GETQDISC, 'req'), 'add': (RTM_NEWQDISC, 'create'), 'del': (RTM_DELQDISC, 'req'), 'remove': (RTM_DELQDISC, 'req'), 'delete': (RTM_DELQDISC, 'req'), 'change': (RTM_NEWQDISC, 'change'), 'replace': (RTM_NEWQDISC, 'replace'), - 'dump-class': (RTM_GETTCLASS, 'dump'), - 'get-class': (RTM_GETTCLASS, 'dump'), 'add-class': (RTM_NEWTCLASS, 'create'), 'del-class': (RTM_DELTCLASS, 'req'), 'change-class': (RTM_NEWTCLASS, 'change'), 'replace-class': (RTM_NEWTCLASS, 'replace'), - 'dump-filter': (RTM_GETTFILTER, 'dump'), - 'get-filter': (RTM_GETTFILTER, 'dump'), 'add-filter': (RTM_NEWTFILTER, 'create'), 'del-filter': (RTM_DELTFILTER, 'req'), 'change-filter': (RTM_NEWTFILTER, 'change'), 'replace-filter': (RTM_NEWTFILTER, 'replace'), } + if command == 'del': + if index == 0: + index = [ + x['index'] for x in self.get_links() if x['index'] != 1 + ] + if isinstance(index, (list, tuple, set)): + return list(chain(*(self.tc('del', index=x) for x in index))) + command, flags = self.make_request_type(command, command_map) + msg = tcmsg() + # transform handle, parent and target, if needed: + handle = transform_handle(handle) + for item in ('parent', 'target', 'default'): + if item in kwarg and kwarg[item] is not None: + kwarg[item] = transform_handle(kwarg[item]) + msg['index'] = index + msg['handle'] = handle + if 'info' in kwarg: + msg['info'] = kwarg['info'] + opts = kwarg.get('opts', None) + ## + # + # + if kind in tc_plugins: + p = tc_plugins[kind] + msg['parent'] = kwarg.pop('parent', getattr(p, 'parent', 0)) + if hasattr(p, 'fix_msg'): + p.fix_msg(msg, kwarg) + if kwarg: + if command in (RTM_NEWTCLASS, RTM_DELTCLASS): + opts = p.get_class_parameters(kwarg) + else: + opts = p.get_parameters(kwarg) + else: + msg['parent'] = kwarg.get('parent', TC_H_ROOT) - if command[:3] in ('add', 'cha'): - if kind is None: - raise ValueError('must specify kind for add/change commands') if kind is not None: - kwarg['kind'] = kind - # 8<----------------------------------------------- - # compatibility section, to be cleaned up? - if index is not None: - kwarg['index'] = index - if handle is not None: - kwarg['handle'] = handle - # 8<----------------------------------------------- - dump_filter, kwarg = get_dump_filter('tc', command, kwarg) - arguments = get_arguments_processor('tc', command, kwarg) - - request = NetlinkRequest( - self, tcmsg(), command, command_map, dump_filter, arguments - ) - await request.send() - if command.startswith('dump'): - return request.response() - return [x async for x in request.response()] + msg['attrs'].append(['TCA_KIND', kind]) + if opts is not None: + msg['attrs'].append(['TCA_OPTIONS', opts]) + return tuple(self.nlm_request(msg, msg_type=command, msg_flags=flags)) - async def route(self, command, **kwarg): + def route(self, command, **kwarg): ''' Route operations. @@ -2421,6 +2260,18 @@ async def route(self, command, **kwarg): if command in ('add', 'set', 'replace', 'change', 'append'): kwarg['proto'] = kwarg.get('proto', 'static') or 'static' kwarg['type'] = kwarg.get('type', 'unicast') or 'unicast' + if 'match' not in kwarg and command in ('dump', 'show'): + match = kwarg + else: + match = kwarg.pop('match', None) + callback = kwarg.pop('callback', None) + request = ( + RequestProcessor(context=kwarg, prime=kwarg) + .apply_filter(RouteFieldFilter()) + .apply_filter(RouteIPRouteFilter(command)) + .finalize() + ) + kwarg = request command_map = { 'add': (RTM_NEWROUTE, 'create'), @@ -2435,25 +2286,70 @@ async def route(self, command, **kwarg): 'show': (RTM_GETROUTE, 'dump'), 'dump': (RTM_GETROUTE, 'dump'), } + (command, flags) = self.make_request_type(command, command_map) msg = rtmsg() - parameters = {'strict_check': self.status['strict_check']} - dump_filter, kwarg = get_dump_filter( - 'route', command, kwarg, parameters - ) - arguments = get_arguments_processor( - 'route', command, kwarg, parameters + # table is mandatory without strict_check; by default == 254 + # if table is not defined in kwarg, save it there + # also for nla_attr. Do not set it in strict_check, use + # NLA instead + if not self.strict_check: + table = kwarg.get('table', 254) + msg['table'] = table if table <= 255 else 252 + msg['family'] = kwarg.pop('family', AF_INET) + msg['scope'] = kwarg.pop('scope', rt_scope['universe']) + msg['dst_len'] = kwarg.pop('dst_len', None) or kwarg.pop('mask', 0) + msg['src_len'] = kwarg.pop('src_len', 0) + msg['tos'] = kwarg.pop('tos', 0) + msg['flags'] = kwarg.pop('flags', 0) + msg['type'] = kwarg.pop('type', rt_type['unspec']) + msg['proto'] = kwarg.pop('proto', rt_proto['unspec']) + msg['attrs'] = [] + + if msg['family'] == AF_MPLS: + for key in tuple(kwarg): + if key not in ('dst', 'newdst', 'via', 'multipath', 'oif'): + kwarg.pop(key) + + for key in kwarg: + nla = rtmsg.name2nla(key) + if nla == 'RTA_DST' and not kwarg[key]: + continue + if kwarg[key] is not None: + msg['attrs'].append([nla, kwarg[key]]) + # fix IP family, if needed + if msg['family'] in (AF_UNSPEC, 255): + if key == 'multipath' and len(kwarg[key]) > 0: + hop = kwarg[key][0] + attrs = hop.get('attrs', []) + for attr in attrs: + if attr[0] == 'RTA_GATEWAY': + msg['family'] = ( + AF_INET6 + if attr[1].find(':') >= 0 + else AF_INET + ) + break + + ret = self.nlm_request( + msg, msg_type=command, msg_flags=flags, callback=callback ) + if match: + if isinstance(match, dict): + match = ( + RequestProcessor(context=match, prime=match) + .apply_filter(RouteFieldFilter(add_defaults=False)) + .apply_filter(RouteIPRouteFilter('dump')) + .finalize() + ) + ret = self.filter_messages(match, ret) - request = NetlinkRequest( - self, msg, command, command_map, dump_filter, arguments - ) - await request.send() - if command in ('dump', 'show'): - return request.response() - return [x async for x in request.response()] + if self.nlm_generator and not flags & NLM_F_DUMP == NLM_F_DUMP: + ret = tuple(ret) - async def rule(self, command, **kwarg): + return ret + + def rule(self, command, **kwarg): ''' Rule operations @@ -2524,283 +2420,92 @@ async def rule(self, command, **kwarg): fwmark=10) ''' if command == 'set': - return [] + return + + if 'match' not in kwarg and command == 'dump': + match = kwarg + else: + match = kwarg.pop('match', None) + request = ( + RequestProcessor(context=kwarg, prime=kwarg) + .apply_filter(RuleFieldFilter()) + .apply_filter(RuleIPRouteFilter(command)) + .finalize() + ) command_map = { 'add': (RTM_NEWRULE, 'create'), 'del': (RTM_DELRULE, 'req'), 'remove': (RTM_DELRULE, 'req'), 'delete': (RTM_DELRULE, 'req'), - 'dump': (RTM_GETRULE, 'root'), + 'dump': (RTM_GETRULE, 'dump'), } - if isinstance(kwarg.get('match'), str): - kwarg['match'] = {'ifname': kwarg['match']} + command, flags = self.make_request_type(command, command_map) msg = fibmsg() - dump_filter, kwarg = get_dump_filter('rule', command, kwarg) - arguments = get_arguments_processor('rule', command, kwarg) - request = NetlinkRequest( - self, msg, command, command_map, dump_filter, arguments - ) - await request.send() - if command == 'dump': - return request.response() - return [x async for x in request.response()] + table = request.get('table', 0) + msg['table'] = table if table <= 255 else 252 + for key in ('family', 'src_len', 'dst_len', 'action', 'tos', 'flags'): + msg[key] = request.pop(key, 0) + msg['attrs'] = [] + + for key in request: + if command == RTM_GETRULE and self.strict_check: + if key in ("match", "priority"): + continue + nla = fibmsg.name2nla(key) + if request[key] is not None: + msg['attrs'].append([nla, request[key]]) + + ret = self.nlm_request(msg, msg_type=command, msg_flags=flags) + + if match: + if isinstance(match, dict): + match = ( + RequestProcessor(context=match, prime=match) + .apply_filter(RuleFieldFilter()) + .apply_filter(RuleIPRouteFilter('dump')) + .finalize() + ) + ret = self.filter_messages(match, ret) - async def stats(self, command, **kwarg): + if self.nlm_generator and not flags & NLM_F_DUMP == NLM_F_DUMP: + ret = tuple(ret) + + return ret + + def stats(self, command, **kwarg): ''' Stats prototype. ''' + if (command == 'dump') and ('match' not in kwarg): + match = kwarg + else: + match = kwarg.pop('match', None) + command_map = { 'dump': (RTM_GETSTATS, 'dump'), 'get': (RTM_GETSTATS, 'get'), } + command, flags = self.make_request_type(command, command_map) msg = ifstatsmsg() msg['filter_mask'] = kwarg.get('filter_mask', 31) msg['ifindex'] = kwarg.get('ifindex', 0) - dump_filter, kwarg = get_dump_filter('stats', command, kwarg) - request = NetlinkRequest(self, msg, command, command_map, dump_filter) - await request.send() - if command == 'dump': - return request.response() - return [x async for x in request.response()] - - # 8<--------------------------------------------------------------- - - -class AsyncIPRoute(AsyncIPRSocket, RTNL_API): - ''' - Regular ordinary async utility class, provides RTNL API using - AsyncIPRSocket as the transport level. - - .. warning:: - The project core is currently undergoing refactoring, so - some methods may still use the old synchronous API. This - will be addressed in future updates. - - The main RTNL API class is built on an asyncio core. All methods - that send netlink requests are asynchronous and return awaitables. - Dump requests return asynchronous generators, while other requests - return iterables, such as tuples or lists. - - This design choice addresses the fact that RTNL dumps, such as - routes or neighbors, can return an extremely large number of objects. - Buffering the entire response in memory could lead to performance - issues. - - .. testcode:: - - import asyncio - - from pyroute2 import AsyncIPRoute - - - async def main(): - async with AsyncIPRoute() as ipr: - # create a link: immediate evaluation - await ipr.link("add", ifname="test0", kind="dummy") - - # dump links: lazy evaluation - async for link in await ipr.link("dump"): - print(link.get("ifname")) - - asyncio.run(main()) - - .. testoutput:: - - lo - eth0 - test0 - ''' - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - self.close() - - -class IPRoute(NetlinkSocket): - ''' - A synchronous version of AsyncIPRoute. All the same API, but - sync. Provides a legacy API for the old code that is not using - asyncio. - - This API is designed to be compatible with the old synchronous `IPRoute` - from version 0.8.x and earlier: - - .. testcode:: - - from pyroute2 import IPRoute - - with IPRoute() as ipr: - for msg in ipr.addr("dump"): - addr = msg.get("address") - mask = msg.get("prefixlen") - print(f"{addr}/{mask}") - - .. testoutput:: - - 127.0.0.1/8 - 192.168.122.28/24 - - .. testcode:: - - from pyroute2 import IPRoute - with IPRoute() as ipr: + ret = self.nlm_request(msg, msg_type=command, msg_flags=flags) + if match is not None: + ret = self.filter_messages(match, ret) - # this request returns one match, one interface index - eth0 = ipr.link_lookup(ifname="eth0") - assert len(eth0) == 1 # 1 if exists else 0 + if self.nlm_generator and not flags & NLM_F_DUMP == NLM_F_DUMP: + ret = tuple(ret) - # this requests uses a lambda to filter interfaces - # and returns all interfaces that are up - nics_up = set(ipr.link_lookup(lambda x: x.get("flags") & 1)) - assert len(nics_up) == 2 - assert nics_up == {1, 2} - ''' - - def __init__( - self, - port=None, - pid=None, - fileno=None, - sndbuf=1048576, - rcvbuf=1048576, - rcvsize=16384, - all_ns=False, - async_qsize=None, - nlm_generator=None, - target='localhost', - ext_ack=False, - strict_check=False, - groups=RTMGRP_DEFAULTS, - nlm_echo=False, - netns=None, - flags=os.O_CREAT, - libc=None, - use_socket=None, - use_event_loop=None, - telemetry=None, - ): - self.asyncore = AsyncIPRoute( - port=port, - pid=pid, - fileno=fileno, - sndbuf=sndbuf, - rcvbuf=rcvbuf, - rcvsize=rcvsize, - all_ns=all_ns, - async_qsize=async_qsize, - nlm_generator=nlm_generator, - target=target, - ext_ack=ext_ack, - strict_check=strict_check, - groups=groups, - nlm_echo=nlm_echo, - use_socket=use_socket, - netns=netns, - flags=flags, - libc=libc, - use_event_loop=use_event_loop, - telemetry=telemetry, - ) - self.asyncore.status['event_loop'] = 'new' - self.asyncore.local.keep_event_loop = True - self.asyncore.event_loop.run_until_complete( - self.asyncore.setup_endpoint() - ) - if self.asyncore.socket.fileno() == -1: - raise OSError(9, 'Bad file descriptor') - - @classmethod - def from_asyncore(cls, iproute): - ret = cls() - ret.asyncore = iproute return ret - def ensure(self, method, present=True, timeout=10, interval=0.2, **spec): - # method points to the sync API, and is a partial() wrapper - # extract async method from the wrapper's arguments - method = method.args[0] - return self._run_with_cleanup( - self.asyncore.ensure, method, present, timeout, interval, **spec - ) - - def poll(self, method, command, timeout=10, interval=0.2, **spec): - ctime = get_time() - ret = tuple() - while ctime + timeout > get_time(): - try: - ret = [x for x in method(command, **spec)] - if ret: - return ret - time.sleep(interval) - except NetlinkDumpInterrupted: - pass - raise TimeoutError() - - def _run_force_sync(self, func, *argv, **kwarg): - return tuple(self._generate_with_cleanup(func, *argv, **kwarg)) - - def _run_generic_rtnl(self, func, *argv, **kwarg): - if len(argv) and argv[0] in ('dump', 'show'): - if not config.nlm_generator: - return tuple(self._generate_with_cleanup(func, *argv, **kwarg)) - return self._generate_with_cleanup(func, *argv, **kwarg) - return self._run_with_cleanup(func, *argv, **kwarg) - - def __getattr__(self, name): - generic_methods = set( - ( - 'addr', - 'link', - 'neigh', - 'route', - 'rule', - 'tc', - 'fdb', - 'brport', - 'probe', - 'stats', - 'link_lookup', - 'vlan_filter', - 'flush_addr', - 'flush_rules', - 'flush_routes', - 'get_netnsid', - 'route_dump', - 'route_dumps', - 'route_load', - 'route_loads', - ) - ) - sync_methods = set( - ( - 'list_link_kind', - 'unregister_link_kind', - 'register_link_kind', - 'get_pid', - 'close_file', - 'open_file', - 'filter_messages', - 'set_netnsid', - ) - ) - - symbol = getattr(self.asyncore, name) - if name in set(RTNL_API.__dict__.keys()) - sync_methods: - if name in generic_methods: - return partial(self._run_generic_rtnl, symbol) - if not config.nlm_generator: - return partial(self._run_force_sync, symbol) - return partial(self._generate_with_cleanup, symbol) - return symbol + # 8<--------------------------------------------------------------- -class IPBatch(IPRoute): +class IPBatch(RTNL_API, IPBatchSocket): ''' Netlink requests compiler. Does not send any requests, but instead stores them in the internal binary buffer. The @@ -2826,63 +2531,24 @@ class IPBatch(IPRoute): ''' - def __init__(self): - super().__init__() - self.reset() - - def reset(self): - self.asyncore.batch = bytearray() - - -class RawIPRoute(IPRoute): - def __init__(self): - super().__init__() - self.asyncore.request_proxy = None + pass -class NetNS(IPRoute): +class IPRoute(LAB_API, RTNL_API, IPRSocket): + ''' + Regular ordinary utility class, see RTNL API for the list of methods. ''' - The `NetNS` class, prior to version 0.9.1, was used to run the RTNL API - in a network namespace. Starting with pyroute2 version 0.9.1, the network - namespace functionality has been integrated into the library core. To run - an `IPRoute` or `AsyncIPRoute` instance in a network namespace, simply use - the `netns` argument: - - .. testcode:: - - from pyroute2 import IPRoute - - with IPRoute(netns="test") as ipr: - assert ipr.status["netns"] == "test" - - After initialization, the netns name is available as `.status["netns"]`. - - The old synchronous `NetNS` class is still available for compatibility - but now serves as a wrapper around `IPRoute`. - .. testcode:: + pass - from pyroute2 import NetNS - with NetNS("test") as ns: - assert ns.status["netns"] == "test" +class RawIPRoute(RTNL_API, RawIPRSocket): + ''' + The same as `IPRoute`, but does not use the netlink proxy. + Thus it can not manage e.g. tun/tap interfaces. ''' - def __init__( - self, - netns=None, - flags=os.O_CREAT, - target='localhost', - libc=None, - groups=RTMGRP_DEFAULTS, - ): - super().__init__( - target=target, netns=netns, flags=flags, libc=libc, groups=groups - ) - - def remove(self): - self.close() - netns.remove(self.status['netns']) + pass class ChaoticIPRoute(RTNL_API, ChaoticIPRSocket): diff --git a/pyroute2/iproute/parsers.py b/pyroute2/iproute/parsers.py index faaf499c4..5e0f1a7a3 100644 --- a/pyroute2/iproute/parsers.py +++ b/pyroute2/iproute/parsers.py @@ -1,49 +1,9 @@ import struct -from functools import partial from pyroute2.netlink import NLMSG_DONE, nlmsg -from pyroute2.netlink.exceptions import NetlinkError -from pyroute2.netlink.rtnl import RTM_NEWROUTE from pyroute2.netlink.rtnl.rtmsg import rtmsg -def get_header(data, offset): - # get message header - header = dict( - zip( - ('length', 'type', 'flags', 'sequence_number'), - struct.unpack_from('IHHI', data, offset), - ) - ) - header['error'] = None - return header - - -def msg_done(header): - msg = nlmsg() - msg['header'] = header - msg.length = msg['header']['length'] - return msg - - -def _export_routes(fd, data, offset, length): - '''Export RTM_NEWROUTE messages binary data. - - Otherwise return NLMSG_DONE. - ''' - header = get_header(data, offset) - if header['type'] == NLMSG_DONE: - return msg_done(header) - elif header['type'] == RTM_NEWROUTE: - fd.write(data[offset : offset + length]) - return - raise NetlinkError() - - -def export_routes(fd): - return partial(_export_routes, fd) - - def default_routes(data, offset, length): ''' Only for RTM_NEWROUTE. @@ -54,9 +14,19 @@ def default_routes(data, offset, length): * nlmsg() -- NLMSG_DONE * None for any other messages ''' - header = get_header(data, offset) + # get message header + header = dict( + zip( + ('length', 'type', 'flags', 'sequence_number'), + struct.unpack_from('IHHI', data, offset), + ) + ) + header['error'] = None if header['type'] == NLMSG_DONE: - return msg_done(header) + msg = nlmsg() + msg['header'] = header + msg.length = msg['header']['length'] + return msg # skip to NLA: offset + nlmsg header + rtmsg data cursor = offset + 28 diff --git a/pyroute2/iproute/windows.py b/pyroute2/iproute/windows.py index 42f32f40d..65cc327e7 100644 --- a/pyroute2/iproute/windows.py +++ b/pyroute2/iproute/windows.py @@ -15,8 +15,9 @@ import os from socket import AF_INET -from pyroute2.common import dqn2int +from pyroute2.common import AddrPool, Namespace, dqn2int from pyroute2.netlink import NLM_F_DUMP, NLM_F_MULTI, NLM_F_REQUEST, NLMSG_DONE +from pyroute2.netlink.proxy import NetlinkProxy from pyroute2.netlink.rtnl import ( RTM_GETADDR, RTM_GETLINK, @@ -79,6 +80,10 @@ class IP_ADAPTER_INFO(ctypes.Structure): class IPRoute(object): def __init__(self, *argv, **kwarg): self.marshal = MarshalRtnl() + send_ns = Namespace( + self, {'addr_pool': AddrPool(0x10000, 0x1FFFF), 'monitor': False} + ) + self._sproxy = NetlinkProxy(policy='return', nl=send_ns) self.target = kwarg.get('target') or 'localhost' def __enter__(self): @@ -270,8 +275,3 @@ class RawIPRoute(IPRoute): class ChaoticIPRoute: def __init__(self, *argv, **kwarg): raise NotImplementedError() - - -class NetNS: - def __init__(self, *argv, **kwarg): - raise NotImplementedError() diff --git a/pyroute2/ipset.py b/pyroute2/ipset.py index e26d06859..750c50e65 100644 --- a/pyroute2/ipset.py +++ b/pyroute2/ipset.py @@ -459,7 +459,7 @@ def add( skbprio=None, skbqueue=None, wildcard=False, - **kwargs, + **kwargs ): ''' Add a member to the ipset. @@ -513,7 +513,7 @@ def add( skbprio=skbprio, skbqueue=skbqueue, wildcard=wildcard, - **kwargs, + **kwargs ) def delete( diff --git a/pyroute2/ipvs.py b/pyroute2/ipvs.py deleted file mode 100644 index 71233f7d5..000000000 --- a/pyroute2/ipvs.py +++ /dev/null @@ -1,204 +0,0 @@ -''' -IPVS -- IP Virtual Server -------------------------- - -IPVS configuration is done via generic netlink protocol. -At the low level one can use it with a GenericNetlinkSocket, -binding it to "IPVS" generic netlink family. - -But for the convenience the library provides utility classes: - - * IPVS -- a socket class to access the API - * IPVSService -- a class to define IPVS service records - * IPVSDest -- a class to define real server records - -Dump all the records:: - - from pyroute2 import IPVS, IPVSDest, IPVSService - - # run the socket - ipvs = IPVS() - - # iterate all the IPVS services - for s in ipvs.service("dump"): - - # create a utility object from a netlink message - service = IPVSService.from_message(s) - print("Service: ", service) - - # iterate all the real servers for this service - for d in ipvs.dest("dump", service=service): - - # create and print a utility object - dest = IPVSDest.from_message(d) - print(" Real server: ", dest) - -Create a service and a real server record:: - - from socket import IPPROTO_TCP - from pyroute2 import IPVS, IPVSDest, IPVSService - - ipvs = IPVS() - - service = IPVSService(addr="192.168.122.1", port=80, protocol=IPPROTO_TCP) - real_server = IPVSDest(addr="10.0.2.20", port=80) - - ipvs.service("add", service=service) - ipvs.dest("add", service=service, dest=real_server) - -Delete a service:: - - from pyroute2 import IPVS, IPVSService - - ipvs = IPVS() - ipvs.service("del", - service=IPVSService( - addr="192.168.122.1", - port=80, - protocol=IPPROTO_TCP - ) - ) - -''' - -from socket import AF_INET - -from pyroute2.common import get_address_family -from pyroute2.netlink.generic import ipvs -from pyroute2.netlink.nlsocket import NetlinkRequest -from pyroute2.requests.common import NLAKeyTransform -from pyroute2.requests.main import RequestProcessor - - -class ServiceFieldFilter(NLAKeyTransform): - _nla_prefix = 'IPVS_SVC_ATTR_' - - def set_addr(self, context, value): - ret = {"addr": value} - if "af" in context.keys(): - family = context["af"] - else: - family = ret["af"] = get_address_family(value) - if family == AF_INET and "netmask" not in context.keys(): - ret["netmask"] = "255.255.255.255" - return ret - - -class DestFieldFilter(NLAKeyTransform): - _nla_prefix = 'IPVS_DEST_ATTR_' - - def set_addr(self, context, value): - ret = {"addr": value} - if "addr_family" not in context.keys(): - ret["addr_family"] = get_address_family(value) - return ret - - -class NLAFilter(RequestProcessor): - msg = None - keys = tuple() - field_filters = None - nla = None - default_values = {} - - def __init__(self, **kwarg): - dict.update(self, self.default_values) - # save field filters - flt = self.field_filters - # init resets the filters, not fixed yet - super().__init__(prime=kwarg) - # restore filters - for f in flt: - self.add_filter(f) - - @classmethod - def from_message(cls, msg): - obj = cls() - for key, value in msg.get(cls.nla)["attrs"]: - obj[key] = value - obj.pop("stats", None) - obj.pop("stats64", None) - return obj - - def dump_nla(self, items=None): - if items is None: - items = self.items() - self.update(self) - self.finalize() - return { - "attrs": list( - map(lambda x: (self.msg.name2nla(x[0]), x[1]), items) - ) - } - - def dump_key(self): - return self.dump_nla( - items=filter(lambda x: x[0] in self.key_fields, self.items()) - ) - - -class IPVSService(NLAFilter): - field_filters = [ServiceFieldFilter()] - msg = ipvs.ipvsmsg.service - key_fields = ("af", "protocol", "addr", "port") - nla = "IPVS_CMD_ATTR_SERVICE" - default_values = { - "timeout": 0, - "sched_name": "wlc", - "flags": {"flags": 0, "mask": 0xFFFF}, - } - - -class IPVSDest(NLAFilter): - field_filters = [DestFieldFilter()] - msg = ipvs.ipvsmsg.dest - nla = "IPVS_CMD_ATTR_DEST" - default_values = { - "fwd_method": 3, - "weight": 1, - "tun_type": 0, - "tun_port": 0, - "tun_flags": 0, - "u_thresh": 0, - "l_thresh": 0, - } - - -class IPVS(ipvs.IPVSSocket): - - def service(self, command, service=None): - command_map = { - "add": (ipvs.IPVS_CMD_NEW_SERVICE, "create"), - "set": (ipvs.IPVS_CMD_SET_SERVICE, "change"), - "update": (ipvs.IPVS_CMD_DEL_SERVICE, "change"), - "del": (ipvs.IPVS_CMD_DEL_SERVICE, "req"), - "get": (ipvs.IPVS_CMD_GET_SERVICE, "get"), - "dump": (ipvs.IPVS_CMD_GET_SERVICE, "dump"), - } - cmd, flags = NetlinkRequest.calculate_request_type( - command, command_map - ) - msg = ipvs.ipvsmsg() - msg["cmd"] = cmd - msg["version"] = ipvs.GENL_VERSION - if service is not None: - msg["attrs"] = [("IPVS_CMD_ATTR_SERVICE", service.dump_nla())] - return self.nlm_request(msg, msg_type=self.prid, msg_flags=flags) - - def dest(self, command, service, dest=None): - command_map = { - "add": (ipvs.IPVS_CMD_NEW_DEST, "create"), - "set": (ipvs.IPVS_CMD_SET_DEST, "change"), - "update": (ipvs.IPVS_CMD_DEL_DEST, "change"), - "del": (ipvs.IPVS_CMD_DEL_DEST, "req"), - "get": (ipvs.IPVS_CMD_GET_DEST, "get"), - "dump": (ipvs.IPVS_CMD_GET_DEST, "dump"), - } - cmd, flags = self.make_request_type(command, command_map) - msg = ipvs.ipvsmsg() - msg["cmd"] = cmd - msg["version"] = 0x1 - msg["attrs"] = [("IPVS_CMD_ATTR_SERVICE", service.dump_key())] - if dest is not None: - msg["attrs"].append(("IPVS_CMD_ATTR_DEST", dest.dump_nla())) - return self.nlm_request(msg, msg_type=self.prid, msg_flags=flags) diff --git a/pyroute2/iwutil.py b/pyroute2/iwutil.py index f03128324..35b444531 100644 --- a/pyroute2/iwutil.py +++ b/pyroute2/iwutil.py @@ -132,7 +132,7 @@ ''' import logging -from pyroute2.netlink import NLM_F_DUMP, NLM_F_REQUEST +from pyroute2.netlink import NLM_F_ACK, NLM_F_DUMP, NLM_F_REQUEST from pyroute2.netlink.nl80211 import ( BSS_STATUS_NAMES, CHAN_WIDTH, @@ -140,22 +140,49 @@ NL80211, NL80211_NAMES, SCAN_FLAGS_NAMES, - AsyncNL80211, nl80211cmd, ) log = logging.getLogger(__name__) -class AsyncIW(AsyncNL80211): +class IW(NL80211): + def __init__(self, *argv, **kwarg): + # get specific groups kwarg + if 'groups' in kwarg: + groups = kwarg['groups'] + del kwarg['groups'] + else: + groups = None + + # get specific async kwarg + if 'async' in kwarg: + # FIXME + # raise deprecation error after 0.5.3 + # + log.warning( + 'use "async_cache" instead of "async", ' + '"async" is a keyword from Python 3.7' + ) + kwarg['async_cache'] = kwarg.pop('async') + + if 'async_cache' in kwarg: + async_cache = kwarg.pop('async_cache') + else: + async_cache = False + + # align groups with async_cache + if groups is None: + groups = ~0 if async_cache else 0 + + # continue with init + super(IW, self).__init__(*argv, **kwarg) - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - await super().setup_endpoint() - await self.bind() + # do automatic bind + # FIXME: unfortunately we can not omit it here + self.bind(groups, async_cache=async_cache) - async def del_interface(self, dev): + def del_interface(self, dev): ''' Delete a virtual interface @@ -164,9 +191,11 @@ async def del_interface(self, dev): msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_DEL_INTERFACE'] msg['attrs'] = [['NL80211_ATTR_IFINDEX', dev]] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def add_interface(self, ifname, iftype, dev=None, phy=0): + def add_interface(self, ifname, iftype, dev=None, phy=0): ''' Create a virtual interface @@ -209,21 +238,25 @@ async def add_interface(self, ifname, iftype, dev=None, phy=0): msg['attrs'].append(['NL80211_ATTR_WIPHY', phy]) else: raise TypeError('no device specified') - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def list_dev(self): + def list_dev(self): ''' Get list of all wifi network interfaces ''' - return await self.get_interfaces_dump() + return self.get_interfaces_dump() - async def list_wiphy(self): + def list_wiphy(self): ''' Get list of all phy devices ''' msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_WIPHY'] - return await self._do_dump(msg) + return self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) def _get_phy_name(self, attr): return 'phy%i' % attr.get_attr('NL80211_ATTR_WIPHY') @@ -231,12 +264,12 @@ def _get_phy_name(self, attr): def _get_frequency(self, attr): return attr.get_attr('NL80211_ATTR_WIPHY_FREQ') or 0 - async def get_interfaces_dict(self): + def get_interfaces_dict(self): ''' Get interfaces dictionary ''' ret = {} - async for wif in await self.get_interfaces_dump(): + for wif in self.get_interfaces_dump(): chan_width = wif.get_attr('NL80211_ATTR_CHANNEL_WIDTH') freq = self._get_frequency(wif) if chan_width is not None else 0 wifname = wif.get_attr('NL80211_ATTR_IFNAME') @@ -249,44 +282,50 @@ async def get_interfaces_dict(self): ] return ret - async def get_interfaces_dump(self): + def get_interfaces_dump(self): ''' Get interfaces dump ''' msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_INTERFACE'] - return await self._do_dump(msg) + return self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) - async def get_interface_by_phy(self, attr): + def get_interface_by_phy(self, attr): ''' Get interface by phy ( use x.get_attr('NL80211_ATTR_WIPHY') ) ''' msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_INTERFACE'] msg['attrs'] = [['NL80211_ATTR_WIPHY', attr]] - return await self._do_request( - msg, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + return self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP ) - async def get_interface_by_ifindex(self, ifindex): + def get_interface_by_ifindex(self, ifindex): ''' Get interface by ifindex ( use x.get_attr('NL80211_ATTR_IFINDEX') ''' msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_INTERFACE'] msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]] - return await self._do_request(msg) + return self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST + ) - async def get_stations(self, ifindex): + def get_stations(self, ifindex): ''' Get stations by ifindex ''' msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_STATION'] msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]] - return await self._do_dump(msg) + return self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) - async def join_ibss( + def join_ibss( self, ifindex, ssid, @@ -351,9 +390,11 @@ async def join_ibss( if bssid is not None: msg['attrs'].append(['NL80211_ATTR_MAC', bssid]) - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def leave_ibss(self, ifindex): + def leave_ibss(self, ifindex): ''' Leave the IBSS -- the IBSS is determined by the network interface ''' @@ -361,9 +402,11 @@ async def leave_ibss(self, ifindex): msg['cmd'] = NL80211_NAMES['NL80211_CMD_LEAVE_IBSS'] msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def authenticate(self, ifindex, bssid, ssid, freq, auth_type=0): + def authenticate(self, ifindex, bssid, ssid, freq, auth_type=0): ''' Send an Authentication management frame. ''' @@ -378,9 +421,11 @@ async def authenticate(self, ifindex, bssid, ssid, freq, auth_type=0): ['NL80211_ATTR_AUTH_TYPE', auth_type], ] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def deauthenticate(self, ifindex, bssid, reason_code=0x01): + def deauthenticate(self, ifindex, bssid, reason_code=0x01): ''' Send a Deauthentication management frame. ''' @@ -393,9 +438,11 @@ async def deauthenticate(self, ifindex, bssid, reason_code=0x01): ['NL80211_ATTR_REASON_CODE', reason_code], ] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def associate(self, ifindex, bssid, ssid, freq, info_elements=None): + def associate(self, ifindex, bssid, ssid, freq, info_elements=None): ''' Send an Association request frame. ''' @@ -412,9 +459,11 @@ async def associate(self, ifindex, bssid, ssid, freq, info_elements=None): if info_elements is not None: msg['attrs'].append(['NL80211_ATTR_IE', info_elements]) - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def disassociate(self, ifindex, bssid, reason_code=0x03): + def disassociate(self, ifindex, bssid, reason_code=0x03): ''' Send a Disassociation management frame. ''' @@ -427,9 +476,11 @@ async def disassociate(self, ifindex, bssid, reason_code=0x03): ['NL80211_ATTR_REASON_CODE', reason_code], ] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def connect(self, ifindex, ssid, bssid=None): + def connect(self, ifindex, ssid, bssid=None): ''' Connect to the ap with ssid and bssid ''' @@ -442,28 +493,33 @@ async def connect(self, ifindex, ssid, bssid=None): if bssid is not None: msg['attrs'].append(['NL80211_ATTR_MAC', bssid]) - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def disconnect(self, ifindex): + def disconnect(self, ifindex): ''' Disconnect the device ''' msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_DISCONNECT'] msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]] + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - return await self._do_request(msg) - - async def survey(self, ifindex): + def survey(self, ifindex): ''' Return the survey info. ''' msg = nl80211cmd() msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_SURVEY'] msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]] - return await self._do_dump(msg) + return self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) - async def scan(self, ifindex, ssids=None, flush_cache=False): + def scan(self, ifindex, ssids=None, flush_cache=False): ''' Trigger scan and get results. @@ -493,19 +549,18 @@ async def scan(self, ifindex, ssids=None, flush_cache=False): scan_flags |= SCAN_FLAGS_NAMES['NL80211_SCAN_FLAG_FLUSH'] msg['attrs'].append(['NL80211_ATTR_SCAN_FLAGS', scan_flags]) - await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) # monitor the results notification on the secondary socket scanResultNotFound = True while scanResultNotFound: - listMsg = await nsock.get() - try: - async for msg in listMsg: - if msg["event"] == "NL80211_CMD_NEW_SCAN_RESULTS": - scanResultNotFound = False - break - finally: - await listMsg.aclose() + listMsg = nsock.get() + for msg in listMsg: + if msg["event"] == "NL80211_CMD_NEW_SCAN_RESULTS": + scanResultNotFound = False + break # close the secondary socket nsock.close() @@ -513,9 +568,11 @@ async def scan(self, ifindex, ssids=None, flush_cache=False): msg2 = nl80211cmd() msg2['cmd'] = NL80211_NAMES['NL80211_CMD_GET_SCAN'] msg2['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]] - return await self._do_dump(msg2) + return self.nlm_request( + msg2, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) - async def get_associated_bss(self, ifindex): + def get_associated_bss(self, ifindex): ''' Returns the same info like scan() does, but only about the currently associated BSS. @@ -532,23 +589,23 @@ async def get_associated_bss(self, ifindex): msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_SCAN'] msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]] - coro = self._do_dump(msg) - try: - async for x in coro: - attr_bss = x.get_attr('NL80211_ATTR_BSS') - if attr_bss is not None: - status = attr_bss.get_attr('NL80211_BSS_STATUS') - if status in ( - BSS_STATUS_NAMES['associated'], - BSS_STATUS_NAMES['ibss_joined'], - ): - return x - finally: - await coro.aclose() + res = self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) + + for x in res: + attr_bss = x.get_attr('NL80211_ATTR_BSS') + if attr_bss is not None: + status = attr_bss.get_attr('NL80211_BSS_STATUS') + if status in ( + BSS_STATUS_NAMES['associated'], + BSS_STATUS_NAMES['ibss_joined'], + ): + return x return None - async def get_regulatory_domain(self, attr=None): + def get_regulatory_domain(self, attr=None): ''' Get regulatory domain information. If attr specified, get regulatory domain information for this device @@ -562,9 +619,9 @@ async def get_regulatory_domain(self, attr=None): else: msg['attrs'] = [['NL80211_ATTR_WIPHY', attr]] - return await self._do_request(msg) + return self.nlm_request(msg, msg_type=self.prid, msg_flags=flags) - async def set_regulatory_domain(self, alpha2): + def set_regulatory_domain(self, alpha2): ''' Set regulatory domain. ''' @@ -572,9 +629,11 @@ async def set_regulatory_domain(self, alpha2): msg['cmd'] = NL80211_NAMES['NL80211_CMD_REQ_SET_REG'] msg['attrs'] = [['NL80211_ATTR_REG_ALPHA2', alpha2]] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def set_tx_power(self, dev, mode, mbm=None): + def set_tx_power(self, dev, mode, mbm=None): ''' Set TX power of interface. @@ -591,9 +650,11 @@ async def set_tx_power(self, dev, mode, mbm=None): if mbm is not None: msg['attrs'].append(['NL80211_ATTR_WIPHY_TX_POWER_LEVEL', mbm]) - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def set_wiphy_netns_by_pid(self, wiphy, pid): + def set_wiphy_netns_by_pid(self, wiphy, pid): ''' Set wiphy network namespace to process network namespace. ''' @@ -604,9 +665,11 @@ async def set_wiphy_netns_by_pid(self, wiphy, pid): ['NL80211_ATTR_PID', pid], ] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def set_wiphy_netns_by_fd(self, wiphy, netns_fd): + def set_wiphy_netns_by_fd(self, wiphy, netns_fd): ''' Set wiphy network namespace to namespace referenced by fd. ''' @@ -617,9 +680,11 @@ async def set_wiphy_netns_by_fd(self, wiphy, netns_fd): ['NL80211_ATTR_NETNS_FD', netns_fd], ] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def set_interface_type(self, ifindex, iftype): + def set_interface_type(self, ifindex, iftype): ''' Set interface type - ifindex — device index @@ -651,13 +716,15 @@ async def set_interface_type(self, ifindex, iftype): ['NL80211_ATTR_IFTYPE', iftype], ] - return await self._do_request(msg) + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) - async def get_interface_type(self, ifindex) -> str: + def get_interface_type(self, ifindex) -> str: ''' return interface type name ''' - dump = await self.get_interface_by_ifindex(ifindex) + dump = self.get_interface_by_ifindex(ifindex) type = None for d in dump: type = d.get_attr('NL80211_ATTR_IFTYPE') @@ -670,140 +737,3 @@ async def get_interface_type(self, ifindex) -> str: res = 'Not Found Type' return res - - -class IW(NL80211): - async_class = AsyncIW - - def del_interface(self, dev): - return self._run_with_cleanup(self.asyncore.del_interface, dev) - - def add_interface(self, ifname, iftype, dev=None, phy=0): - return self._run_with_cleanup( - self.asyncore.add_interface, ifname, iftype, dev, phy - ) - - def list_dev(self): - return self._generate_with_cleanup(self.asyncore.list_dev) - - def list_wiphy(self): - return self._generate_with_cleanup(self.asyncore.list_wiphy) - - def get_interfaces_dict(self): - return self._run_with_cleanup(self.asyncore.get_interfaces_dict) - - def get_interfaces_dump(self): - return self._generate_with_cleanup(self.asyncore.get_interfaces_dump) - - def get_interface_by_phy(self, attr): - return self._run_with_cleanup(self.asyncore.get_interface_by_phy, attr) - - def get_interface_by_ifindex(self, ifindex): - return self._run_with_cleanup( - self.asyncore.get_interface_by_ifindex, ifindex - ) - - def get_stations(self, ifindex): - return self._generate_with_cleanup(self.asyncore.get_stations, ifindex) - - def join_ibss( - self, - ifindex, - ssid, - freq, - bssid=None, - channel_fixed=False, - width=None, - center=None, - center2=None, - ): - return self._run_with_cleanup( - self.asyncore.join_ibss, - ifindex, - ssid, - freq, - bssid, - channel_fixed, - width, - center, - center2, - ) - - def leave_ibss(self, ifindex): - return self._run_with_cleanup(self.asyncore.leave_ibss, ifindex) - - def authenticate(self, ifindex, bssid, ssid, freq, auth_type=0): - return self._run_with_cleanup( - self.asyncore.authenticate, ifindex, bssid, ssid, freq, auth_type - ) - - def deauthenticate(self, ifindex, bssid, reason_code=0x01): - return self._run_with_cleanup( - self.asyncore.deauthenticate, ifindex, bssid, reason_code - ) - - def associate(self, ifindex, bssid, ssid, freq, info_elements=None): - return self._run_with_cleanup( - self.asyncore.associate, ifindex, bssid, ssid, freq, info_elements - ) - - def disassociate(self, ifindex, bssid, reason_code=0x03): - return self._run_with_cleanup( - self.asyncore.disassociate, ifindex, bssid, reason_code - ) - - def connect(self, ifindex, ssid, bssid=None): - return self._run_with_cleanup( - self.asyncore.connect, ifindex, ssid, bssid - ) - - def disconnect(self, ifindex): - return self._run_with_cleanup(self.asyncore.disconnect, ifindex) - - def survey(self, ifindex): - return self._generate_with_cleanup(self.asyncore.survey, ifindex) - - def scan(self, ifindex, ssids=None, flush_cache=False): - return self._generate_with_cleanup( - self.asyncore.scan, ifindex, ssids, flush_cache - ) - - def get_associated_bss(self, ifindex): - return self._run_with_cleanup( - self.asyncore.get_associated_bss, ifindex - ) - - def get_regulatory_domain(self, attr=None): - return self._run_with_cleanup( - self.asyncore.get_regulatory_domain, attr - ) - - def set_regulatory_domain(self, alpha2): - return self._run_with_cleanup( - self.asyncore.set_regulatory_domain, alpha2 - ) - - def set_tx_power(self, dev, mode, mbm=None): - return self._run_with_cleanup( - self.asyncore.set_tx_power, dev, mode, mbm - ) - - def set_wiphy_netns_by_pid(self, wiphy, pid): - return self._run_with_cleanup( - self.asyncore.set_wiphy_netns_by_pid, wiphy, pid - ) - - def set_wiphy_netns_by_fd(self, wiphy, netns_fd): - return self._run_with_cleanup( - self.asyncore.set_wiphy_netns_by_fd, wiphy, netns_fd - ) - - def set_interface_type(self, ifindex, iftype): - return self._run_with_cleanup( - self.asyncore.set_interface_type, ifindex, iftype - ) - - def get_interface_type(self, ifindex) -> str: - return self._run_with_cleanup( - self.asyncore.get_interface_type, ifindex - ) diff --git a/pyroute2/lab.py b/pyroute2/lab.py new file mode 100644 index 000000000..34d5e82fe --- /dev/null +++ b/pyroute2/lab.py @@ -0,0 +1,30 @@ +import inspect + +try: + from unittest import mock +except ImportError: + mock = None + +registry = [] +use_mock = False + + +class LAB_API: + def __init__(self, *argv, **kwarg): + super().__init__(*argv, **kwarg) + if use_mock: + if mock is None: + # postpone ImportError + # + # unittest may not be available on embedded platforms, + # but it is still used by IPRoute class; it is safe + # to leave it in the minimal for now, just raise an + # exception when being used + # + # Bug-Url: https://github.com/svinota/pyroute2/pull/1096 + raise ImportError('unittest.mock not available') + registry.append(self) + for name, method in inspect.getmembers( + self, predicate=inspect.ismethod + ): + setattr(self, name, mock.MagicMock(name=name, wraps=method)) diff --git a/pyroute2/loader.py b/pyroute2/loader.py new file mode 100644 index 000000000..57fa5fe58 --- /dev/null +++ b/pyroute2/loader.py @@ -0,0 +1,43 @@ +import struct +import sys + +## +# +# Logging setup +# +# See the history: +# * https://github.com/svinota/pyroute2/issues/246 +# * https://github.com/svinota/pyroute2/issues/255 +# * https://github.com/svinota/pyroute2/issues/270 +# * https://github.com/svinota/pyroute2/issues/573 +# * https://github.com/svinota/pyroute2/issues/601 +# +from pyroute2.config import log + +## +# +# Windows platform specific: socket module monkey patching +# +# To use the library on Windows, run:: +# pip install win-inet-pton +# +if sys.platform.startswith('win'): # noqa: E402 + import win_inet_pton # noqa: F401 + + +def init(): + try: + # probe, if the bytearray can be used in struct.unpack_from() + struct.unpack_from('I', bytearray((1, 0, 0, 0)), 0) + except Exception: + if sys.version_info[0] < 3: + # monkeypatch for old Python versions + log.warning('patching struct.unpack_from()') + + def wrapped(fmt, buf, offset=0): + return struct._u_f_orig(fmt, str(buf), offset) + + struct._u_f_orig = struct.unpack_from + struct.unpack_from = wrapped + else: + raise diff --git a/pyroute2/minimal.py b/pyroute2/minimal.py index 5ab73572f..630185c47 100644 --- a/pyroute2/minimal.py +++ b/pyroute2/minimal.py @@ -12,135 +12,61 @@ except ImportError: __version__ = 'unknown' -## -# -# Logging setup -# -# See the history: -# * https://github.com/svinota/pyroute2/issues/246 -# * https://github.com/svinota/pyroute2/issues/255 -# * https://github.com/svinota/pyroute2/issues/270 -# * https://github.com/svinota/pyroute2/issues/573 -# * https://github.com/svinota/pyroute2/issues/601 -# -from pyroute2.config import log -from pyroute2.conntrack import AsyncConntrack, Conntrack, ConntrackEntry -from pyroute2.devlink import DL, AsyncDL -from pyroute2.ethtool.ethtool import Ethtool -from pyroute2.iproute import ( - AsyncIPRoute, - ChaoticIPRoute, - IPBatch, - IPRoute, - NetNS, - RawIPRoute, -) -from pyroute2.ipset import IPSet -from pyroute2.ipvs import IPVS, IPVSDest, IPVSService -from pyroute2.iwutil import IW, AsyncIW -from pyroute2.netlink.connector.cn_proc import ProcEventSocket -from pyroute2.netlink.devlink import AsyncDevlinkSocket, DevlinkSocket -from pyroute2.netlink.diag import DiagSocket, ss2 -from pyroute2.netlink.event import AsyncEventSocket, EventSocket -from pyroute2.netlink.event.acpi_event import ( - AcpiEventSocket, - AsyncAcpiEventSocket, -) -from pyroute2.netlink.event.dquot import AsyncDQuotSocket, DQuotSocket -from pyroute2.netlink.event.thermal import ( - AsyncThermalEventSocket, - ThermalEventSocket, -) + +from pyroute2.conntrack import Conntrack, ConntrackEntry +from pyroute2.iproute import ChaoticIPRoute, IPBatch, IPRoute, RawIPRoute +from pyroute2.iproute.ipmock import IPRoute as IPMock +from pyroute2.iwutil import IW +from pyroute2.netlink.devlink import DevlinkSocket +from pyroute2.netlink.diag import DiagSocket +from pyroute2.netlink.event.acpi_event import AcpiEventSocket +from pyroute2.netlink.event.dquot import DQuotSocket from pyroute2.netlink.exceptions import ( ChaoticException, NetlinkDecodeError, NetlinkDumpInterrupted, NetlinkError, ) -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) -from pyroute2.netlink.generic.ethtool import AsyncNlEthtool, NlEthtool -from pyroute2.netlink.generic.ipvs import AsyncIPVSSocket, IPVSSocket -from pyroute2.netlink.generic.l2tp import AsyncL2tp, L2tp -from pyroute2.netlink.generic.mptcp import MPTCP, AsyncMPTCP -from pyroute2.netlink.generic.wireguard import AsyncWireGuard, WireGuard +from pyroute2.netlink.generic import GenericNetlinkSocket +from pyroute2.netlink.generic.l2tp import L2tp +from pyroute2.netlink.generic.mptcp import MPTCP +from pyroute2.netlink.generic.wireguard import WireGuard from pyroute2.netlink.ipq import IPQSocket -from pyroute2.netlink.nfnetlink.nfctsocket import AsyncNFCTSocket, NFCTSocket -from pyroute2.netlink.nfnetlink.nftsocket import AsyncNFTSocket, NFTSocket -from pyroute2.netlink.nl80211 import NL80211, AsyncNL80211 -from pyroute2.netlink.rtnl.iprsocket import AsyncIPRSocket, IPRSocket -from pyroute2.netlink.taskstats import AsyncTaskStats, TaskStats +from pyroute2.netlink.nfnetlink.nfctsocket import NFCTSocket +from pyroute2.netlink.nfnetlink.nftsocket import NFTSocket +from pyroute2.netlink.nl80211 import NL80211 +from pyroute2.netlink.rtnl.iprsocket import IPRSocket +from pyroute2.netlink.taskstats import TaskStats from pyroute2.netlink.uevent import UeventSocket -from pyroute2.plan9.client import Plan9ClientSocket -from pyroute2.plan9.server import Plan9ServerSocket -from pyroute2.wiset import WiSet modules = [ AcpiEventSocket, - AsyncAcpiEventSocket, - AsyncConntrack, - AsyncDL, - AsyncDQuotSocket, - AsyncDevlinkSocket, - AsyncEventSocket, - AsyncGenericNetlinkSocket, - AsyncIPRSocket, - AsyncIPRoute, - AsyncIPVSSocket, - AsyncIW, - AsyncL2tp, - AsyncMPTCP, - AsyncNFCTSocket, - AsyncNFTSocket, - AsyncNL80211, - AsyncNlEthtool, - AsyncTaskStats, - AsyncThermalEventSocket, - AsyncWireGuard, ChaoticException, ChaoticIPRoute, Conntrack, ConntrackEntry, - DL, - DQuotSocket, DevlinkSocket, DiagSocket, - Ethtool, - EventSocket, - GenericNetlinkSocket, + DQuotSocket, IPBatch, + IPMock, IPQSocket, - IPRSocket, IPRoute, - IPSet, - IPVS, - IPVSDest, - IPVSService, - IPVSSocket, + IPRSocket, IW, + GenericNetlinkSocket, L2tp, MPTCP, + NetlinkError, + NetlinkDecodeError, + NetlinkDumpInterrupted, NFCTSocket, NFTSocket, NL80211, - NetNS, - NetlinkDecodeError, - NetlinkDumpInterrupted, - NetlinkError, - NlEthtool, - Plan9ClientSocket, - Plan9ServerSocket, - ProcEventSocket, RawIPRoute, TaskStats, - ThermalEventSocket, UeventSocket, - WiSet, WireGuard, - log, - ss2, ] __all__ = [] diff --git a/pyroute2/ndb/cli.py b/pyroute2/ndb/cli.py new file mode 100755 index 000000000..9c8d7f88f --- /dev/null +++ b/pyroute2/ndb/cli.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +import argparse +import json +import sys + +from pyroute2.cli.console import Console +from pyroute2.cli.server import Server + +try: + from pyroute2.cli.auth.auth_keystone import OSAuthManager +except ImportError: + OSAuthManager = None +try: + from pyroute2.cli.auth.auth_radius import RadiusAuthManager +except ImportError: + RadiusAuthManager = None +try: + import readline +except ImportError: + readline = None + + +def run(): + argp = argparse.ArgumentParser() + for spec in ( + ('-a', '[S] IP address to listen on'), + ('-c', '[C] Command line to run'), + ('-l', '[C,S] Log spec'), + ('-m', 'Set mode (C,S)'), + ('-p', '[S] Port to listen on'), + ('-r', '[C] Load rc file'), + ('-s', '[C,S] Load sources from a json file'), + ('-x', '[S] Strict auth'), + ): + argp.add_argument(spec[0], help=spec[1]) + argp.add_argument('script', nargs='*', help='script to run') + args = argp.parse_args() + commands = [] + sources = None + if args.s: + with open(args.s, 'r') as f: + sources = json.loads(f.read()) + + if args.m in ('S', 'server'): + if args.p: + port = int(args.p) + else: + port = 8080 + + auth_plugins = {} + if OSAuthManager is not None: + auth_plugins['keystone'] = OSAuthManager + if RadiusAuthManager is not None: + auth_plugins['radius:cleartext'] = RadiusAuthManager + + server = Server( + address=args.a or 'localhost', + port=port, + log=args.l, + sources=sources, + auth_strict=args.x, + auth_plugins=auth_plugins, + ) + server.serve_forever() + return 0 + else: + console = Console(log=args.l, sources=sources) + if readline is not None: + console.set_completer(readline) + if args.r: + console.loadrc(args.r) + + for script in args.script: + console.loadrc(script) + + if args.c: + commands.append(args.c) + console.interact(readfunc=lambda x: commands.pop(0)) + elif not args.script: + console.interact() + return 1 if console.session.errors > 0 else 0 + + +if __name__ == '__main__': + rcode = run() + sys.exit(rcode) diff --git a/pyroute2/ndb/cluster.py b/pyroute2/ndb/cluster.py new file mode 100644 index 000000000..4a7bedbd2 --- /dev/null +++ b/pyroute2/ndb/cluster.py @@ -0,0 +1,39 @@ +import json +import socket + +from pyroute2.common import basestring + +from .main import NDB +from .transport import Messenger, Transport + + +def init(config): + if isinstance(config, basestring): + config = json.loads(config) + else: + config = json.load(config) + hostname = config['local'].get('hostname', socket.gethostname()) + messenger = Messenger( + config['local']['id'], + Transport(config['local']['address'], config['local']['port']), + ) + + for target in config['local'].get('targets', []): + messenger.targets.add(target) + + if not messenger.targets: + messenger.targets.add(hostname) + + for peer in config.get('peers', []): + messenger.add_peer(*peer) + + sources = config['local'].get('sources') + if sources is None: + sources = [{'target': hostname, 'kind': 'local'}] + + return NDB( + log=config.get('log', 'debug'), + sources=sources, + localhost=sources[0]['target'], + messenger=messenger, + ) diff --git a/pyroute2/ndb/main.py b/pyroute2/ndb/main.py index e274b84df..2ab039509 100644 --- a/pyroute2/ndb/main.py +++ b/pyroute2/ndb/main.py @@ -1,11 +1,22 @@ ''' -.. testsetup:: * +.. testsetup:: - from pyroute2 import config from pyroute2 import NDB + ndb = NDB(sources=[{'target': 'localhost', 'kind': 'IPMock'}]) - config.mock_netlink = True - ndb = NDB() +.. testsetup:: netns + + from types import MethodType + + from pyroute2 import NDB + + ndb = NDB(sources=[{'target': 'localhost', 'kind': 'IPMock'}]) + + def add_mock_netns(self, netns): + return self.add_orig(target=netns, kind='IPMock', preset='netns') + + ndb.sources.add_orig = ndb.sources.add + ndb.sources.add = MethodType(add_mock_netns, ndb.sources) .. testcleanup:: * @@ -271,31 +282,49 @@ import atexit import ctypes import ctypes.util -import inspect import logging import logging.handlers -import queue +import sys import threading -from functools import reduce -from urllib.parse import urlparse +from pyroute2 import config from pyroute2.common import basestring ## # NDB stuff +from .auth_manager import AuthManager from .events import ShutdownException -from .objects import RTNL_Object -from .objects.interface import SyncInterface -from .objects.route import SyncRoute -from .source import SyncSource -from .sync_api import Flags, SyncDB, SyncSources, SyncView +from .messages import cmsg +from .schema import DBProvider from .task_manager import TaskManager from .transaction import Transaction from .view import SourcesView, View +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + +try: + import queue +except ImportError: + import Queue as queue + log = logging.getLogger(__name__) +NDB_VIEWS_SPECS = ( + ('interfaces', 'interfaces'), + ('addresses', 'addresses'), + ('routes', 'routes'), + ('neighbours', 'neighbours'), + ('af_bridge_fdb', 'fdb'), + ('rules', 'rules'), + ('netns', 'netns'), + ('af_bridge_vlans', 'vlans'), +) + + class Log: def __init__(self, log_id=None): self.logger = None @@ -374,24 +403,6 @@ def close(self): def channel(self, name): return logging.getLogger('pyroute2.ndb.%s.%s' % (self.log_id, name)) - def callstack(self, *argv, **kwarg): - # frame 0: self.debug() - # frame 1: the function that sends the logging - # frame 2: the caller function - self.main.debug( - 'call stack: %s:%s %s() -> %s:%s %s()', - *reduce( - lambda x, y: x + y, - reversed( - [ - (x.filename, x.lineno, x.function) - for x in inspect.stack()[1:3] - ] - ), - ), - ) - return self.debug(*argv, **kwarg) - def debug(self, *argv, **kwarg): return self.main.debug(*argv, **kwarg) @@ -433,8 +444,17 @@ def qsize(self): return self._bypass.qsize() -class NDB: +class AuthProxy: + def __init__(self, ndb, auth_managers): + self._ndb = ndb + self._auth_managers = auth_managers + + for vtable, vname in NDB_VIEWS_SPECS: + view = View(self._ndb, vtable, auth_managers=self._auth_managers) + setattr(self, vname, view) + +class NDB: @property def nsmanager(self): return '%s/nsmanager' % self.localhost @@ -443,7 +463,7 @@ def __init__( self, sources=None, localhost='localhost', - db_provider=None, + db_provider='sqlite3', db_spec=':memory:', db_cleanup=True, rtnl_debug=False, @@ -451,6 +471,9 @@ def __init__( auto_netns=False, libc=None, ): + if db_provider == 'postgres': + db_provider = 'psycopg2' + self.localhost = localhost self.schema = None self.libc = libc or ctypes.CDLL( @@ -477,9 +500,20 @@ def __init__( # # fix sources prime if sources is None: - sources = [ - {'target': self.localhost, 'kind': 'local', 'nlm_generator': 1} - ] + if config.mock_iproute: + sources = [{'target': 'localhost', 'kind': 'IPMock'}] + else: + sources = [ + { + 'target': self.localhost, + 'kind': 'local', + 'nlm_generator': 1, + } + ] + if sys.platform.startswith('linux'): + sources.append( + {'target': self.nsmanager, 'kind': 'nsmanager'} + ) elif not isinstance(sources, (list, tuple)): raise ValueError('sources format not supported') @@ -488,67 +522,40 @@ def __init__( spec['target'] = self.localhost break + am = AuthManager( + {'obj:list': True, 'obj:read': True, 'obj:modify': True}, + self.log.channel('auth'), + ) + self.sources = SourcesView(self, auth_managers=[am]) self._call_registry = {} self._nl = sources atexit.register(self.close) self._dbm_ready.clear() self._dbm_error = None self.config = { + 'provider': str(DBProvider(db_provider)), 'spec': db_spec, 'rtnl_debug': rtnl_debug, 'db_cleanup': db_cleanup, 'auto_netns': auto_netns, 'recordset_pipe': 'false', } - # self.task_manager = TaskManager(self) - # self._dbm_thread = threading.Thread( - target=self.task_manager.main, name='NDB main loop' + target=self.task_manager.run, name='NDB main loop' ) self._dbm_thread.daemon = True self._dbm_thread.start() self._dbm_ready.wait() - for vname, view in self._create_views(): - setattr(self, vname, view) - self.db = SyncDB(self.task_manager.event_loop, self) - for spec in self._nl: - spec['event'] = None - self.sources.add(**spec) if self._dbm_error is not None: raise self._dbm_error + for vtable, vname in NDB_VIEWS_SPECS: + view = View(self, vtable, auth_managers=[am]) + setattr(self, vname, view) # self.query = Query(self.schema) - def _create_views(self, flags=Flags.UNSPEC): - views_map = ( - ('interfaces', 'interfaces', View, SyncView), - ('addresses', 'addresses', View, SyncView), - ('routes', 'routes', View, SyncView), - ('neighbours', 'neighbours', View, SyncView), - ('af_bridge_fdb', 'fdb', View, SyncView), - ('rules', 'rules', View, SyncView), - ('netns', 'netns', View, SyncView), - ('probes', 'probes', View, SyncView), - ('af_bridge_vlans', 'vlans', View, SyncView), - ('sources', 'sources', SourcesView, SyncSources), - ) - class_map = { - 'interfaces': SyncInterface, - 'routes': SyncRoute, - 'sources': SyncSource, - 'default': RTNL_Object, - } - ret = {} - for vtable, vname, vclass, sync_vclass in views_map: - view = vclass(self, vtable) - sview = sync_vclass( - self.task_manager.event_loop, view, class_map, flags=flags - ) - ret[vname] = sview - return iter(ret.items()) - - def _get_view(self, table, chain=None): - return View(self, table, chain) + def _get_view(self, table, chain=None, auth_managers=None): + return View(self, table, chain, auth_managers) def __enter__(self): return self @@ -560,16 +567,15 @@ def begin(self): return Transaction(self.log.channel('transaction')) def readonly(self): - class AuthProxy: - pass - - ap = AuthProxy() - for vname, view in self._create_views(flags=Flags.RO): - setattr(ap, vname, view) - return ap + return self.auth_proxy( + AuthManager( + {'obj:list': True, 'obj:read': True, 'obj:modify': False}, + self.log.channel('auth'), + ) + ) def auth_proxy(self, auth_manager): - raise NotImplementedError() + return AuthProxy(self, [auth_manager]) def close(self): with self._global_lock: @@ -585,10 +591,8 @@ def close(self): except ValueError: pass # shutdown the _dbm_thread - self.task_manager.event_loop.call_soon_threadsafe( - self.task_manager.stop_event.set - ) - self._dbm_shutdown.wait() + self._event_queue.shutdown() + self._event_queue.bypass((cmsg(None, ShutdownException()),)) self._dbm_thread.join() # shutdown the logger -- free the resources self.log.close() diff --git a/pyroute2/ndb/noipdb.py b/pyroute2/ndb/noipdb.py new file mode 100644 index 000000000..ac2f0825e --- /dev/null +++ b/pyroute2/ndb/noipdb.py @@ -0,0 +1,198 @@ +import logging + +from pyroute2.ndb.main import NDB + +log = logging.getLogger(__name__) + + +class ObjectProxy(dict): + def __init__(self, obj): + self._obj = obj + + def __getattribute__(self, key): + if key[:4] == 'set_': + + def set_value(value): + self[key[4:]] = value + return self + + return set_value + try: + return self[key] + except KeyError: + return super(ObjectProxy, self).__getattribute__(key) + + def __setattr__(self, key, value): + if key == '_obj': + super(ObjectProxy, self).__setattr__(key, value) + else: + super(ObjectProxy, self).__getattribute__('_obj')[key] = value + + def __getitem__(self, key): + return super(ObjectProxy, self).__getattribute__('_obj')[key] + + def __setitem__(self, key, value): + super(ObjectProxy, self).__getattribute__('_obj')[key] = value + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if hasattr(self, 'commit'): + self.commit() + + def __repr__(self): + return repr(super(ObjectProxy, self).__getattribute__('_obj')) + + def __contains__(self, key): + return key in super(ObjectProxy, self).__getattribute__('_obj') + + def get_ndb_object(self): + return self._obj + + def keys(self): + return self._obj.keys() + + def items(self): + return self._obj.items() + + def values(self): + return self._obj.values() + + def __iter__(self): + return self._obj.__iter__() + + @property + def _mode(self): + return 'implicit' + + +class Interface(ObjectProxy): + def add_ip(self, *argv, **kwarg): + self._obj.add_ip(*argv, **kwarg) + return self + + def del_ip(self, *argv, **kwarg): + self._obj.del_ip(*argv, **kwarg) + return self + + def add_port(self, *argv, **kwarg): + self._obj.add_port(*argv, **kwarg) + return self + + def del_port(self, *argv, **kwarg): + self._obj.del_port(*argv, **kwarg) + return self + + def commit(self, *argv, **kwarg): + self._obj.commit(*argv, **kwarg) + return self + + def up(self): + self._obj.set('state', 'up') + return self + + def down(self): + self._obj.set('state', 'down') + return self + + def remove(self): + self._obj.remove() + return self + + @property + def if_master(self): + return self._obj.get('master', None) + + @property + def ipaddr(self): + return tuple(self._obj.ipaddr.dump().select('address', 'prefixlen')) + + +class Interfaces(ObjectProxy): + text_create = ''' +When `create().commit()` fails, the failed interface object behaves +differently in IPDB and NDB. IPDB saves the failed object in the database, +while the NDB database contains only the system reflection, and the failed +object may stay only being referenced by a variable. + +`KeyError: 'object exists'` vs. `CreateException` +''' + + def __getitem__(self, key): + return Interface(super(Interfaces, self).__getitem__(key)) + + def __iter__(self): + return iter(self.keys()) + + def add(self, *argv, **kwarg): + return self.create(*argv, **kwarg) + + def create(self, *argv, **kwarg): + log.warning(self.text_create) + return Interface(self._obj.create(*argv, **kwarg)) + + def keys(self): + ret = [] + for record in self._obj.dump(): + ret += [record.ifname, record.index] + return ret + + def has_key(self, key): + return key in self.keys() + + +class NoIPDB(object): + text_create = ''' +IPDB has a shortcut method to create interfaces: `ipdb.create(...)`. + +NDB has `create()` methods only under respective views: +`ndb.interfaces.create(...)`, `ndb.addresses.create(...)` etc. +''' + + text_nl = ''' +Unlike IPDB, NDB can work with many netlink sources. The default one +referenced as `localhost`:: + + # + # these two statements are equivalent: + # + ndb.sources['localhost'].nl.get_links() + ipdb.nl.get_links() + +''' + + def __init__(self, *argv, **kwarg): + if argv or kwarg: + log.warning( + '%s does not support IPDB parameters, ignoring', + self.__class__.__name__, + ) + if len(argv) > 0 or 'nl' in kwarg: + log.warning( + '%s does not support shared netlink sources,' + ' ignoring `nl` and starting with local IPRoute', + self.__class__.__name__, + ) + + self._ndb = NDB() + self.interfaces = Interfaces(self._ndb.interfaces) + + @property + def nl(self): + log.warning(self.text_nl) + return self._ndb.sources['localhost'].nl + + @property + def ipaddr(self): + ret = dict([(x.index, []) for x in self._ndb.interfaces.dump()]) + for record in self._ndb.addresses.dump(): + ret[record.index].append((record.address, record.prefixlen)) + return ret + + def create(self, *argv, **kwarg): + log.warning(self.text_create) + return self.interfaces.create(*argv, **kwarg) + + def release(self): + self._ndb.close() diff --git a/pyroute2/ndb/objects/__init__.py b/pyroute2/ndb/objects/__init__.py index 39017d597..4d9c8e62e 100644 --- a/pyroute2/ndb/objects/__init__.py +++ b/pyroute2/ndb/objects/__init__.py @@ -27,10 +27,11 @@ .. testsetup:: - from pyroute2 import IPRoute, NDB + from pyroute2 import IPMock as IPRoute + from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True .. testcode:: @@ -61,7 +62,6 @@ === ''' -import asyncio import collections import errno import json @@ -69,16 +69,16 @@ import time import traceback import weakref -from enum import IntFlag from functools import partial -from typing import Awaitable, Callable, Union +from pyroute2 import cli from pyroute2.netlink.exceptions import NetlinkError from pyroute2.requests.main import RequestProcessor +from ..auth_manager import AuthManager, check_auth from ..events import InvalidateHandlerException, State +from ..messages import cmsg_event from ..report import Record -from ..sync_api import SyncBase RSLV_IGNORE = 0 RSLV_RAISE = 1 @@ -86,30 +86,30 @@ RSLV_DELETE = 3 -# FIXME:3.9: TypeAlias -# FIXME 3.9: Union -Req = dict[str, Union[str, int]] - - -async def fallback_add(self, idx_req, req): +def fallback_add(self, idx_req, req): # ignore all set/get for objects with incomplete idx_req if set(idx_req.keys()) != set(self.kspec): self.log.debug('ignore incomplete idx_req in the fallback') return # try to set the object - self.sources[self['target']].api(self.api, 'set', **req) + ( + self.ndb._event_queue.put( + self.sources[self['target']].api(self.api, 'set', **req), + source=self['target'], + ) + ) # try to get the object - self.sources[self['target']].api(self.api, 'get', **idx_req) + ( + self.ndb._event_queue.put( + self.sources[self['target']].api(self.api, 'get', **idx_req), + source=self['target'], + ) + ) # reload the collected data self.load_sql() -class ObjectFlags(IntFlag): - UNSPEC = 0x0 - SNAPSHOT = 0x1 - - -class AsyncObject(dict): +class RTNL_Object(dict): ''' The common base class for NDB objects -- interfaces, routes, rules addresses etc. Implements common logic for all the classes, like @@ -215,7 +215,9 @@ def key(self, k): # @classmethod def _count(cls, view): - return view.ndb.schema.fetchone('SELECT count(*) FROM %s' % view.table) + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s' % view.table + ) @classmethod def _dump_where(cls, view): @@ -231,7 +233,7 @@ def _sdump(cls, view, names, fnames): ) yield names where, values = cls._dump_where(view) - for record in view.ndb.schema.fetch(req + where, values): + for record in view.ndb.task_manager.db_fetch(req + where, values): yield record @classmethod @@ -251,8 +253,8 @@ def dump(cls, view): ) @classmethod - def spec_normalize(cls, spec): - return spec + def spec_normalize(cls, processed, spec): + return processed @staticmethod def key_load_context(key, context): @@ -267,11 +269,11 @@ def __init__( load=True, master=None, check=True, - flags=ObjectFlags.UNSPEC, + auth_managers=None, ): self.view = view self.ndb = view.ndb - self.sources = view.ndb.sources.asyncore + self.sources = view.ndb.sources self.master = master self.ctxid = ctxid self.schema = view.ndb.schema @@ -280,20 +282,22 @@ def __init__( self.iclass = iclass self.utable = self.utable or self.table self.errors = [] - self.flags = flags self.atime = time.time() self.log = self.ndb.log.channel('rtnl_object') self.log.debug('init') + if auth_managers is None: + auth_managers = [AuthManager(None, self.ndb.log.channel('auth'))] + self.auth_managers = auth_managers self.state = State() self.state.set('invalid') self.snapshot_deps = [] - self.load_event = asyncio.Event() - self.evq = self.ndb.task_manager.event_queue + self.load_event = threading.Event() self.load_event.set() self.load_debug = False self.lock = threading.Lock() - self.object_data = RequestProcessor(context=weakref.proxy(self)) - self.object_data.add_filter(self.field_filter()) + self.object_data = RequestProcessor( + self.field_filter(), context=weakref.proxy(self) + ) self.kspec = self.schema.compiled[self.table]['idx'] self.knorm = self.schema.compiled[self.table]['norm_idx'] self.spec = self.schema.compiled[self.table]['all_names'] @@ -347,15 +351,12 @@ def __init__( def new_spec(cls, spec, context=None, localhost=None): if isinstance(spec, Record): spec = spec._as_dict() - spec = cls.spec_normalize(spec) - rp = RequestProcessor(context=spec) - rp.add_filter(cls.field_filter()) - rp.update(spec) + rp = RequestProcessor(cls.field_filter(), context=spec, prime=spec) if isinstance(context, dict): rp.update(context) if 'target' not in rp and localhost is not None: rp['target'] = localhost - return rp + return cls.spec_normalize(rp, spec) @staticmethod def resolve(view, spec, fields, policy=RSLV_IGNORE): @@ -417,9 +418,11 @@ def __exit__(self, exc_type, exc_value, traceback): def __hash__(self): return id(self) + @check_auth('obj:read') def __getitem__(self, key): return dict.__getitem__(self, key) + @check_auth('obj:modify') def __setitem__(self, key, value): for nkey, nvalue in self.object_data.filter(key, value).items(): if self.get(nkey) == nvalue: @@ -429,7 +432,9 @@ def __setitem__(self, key, value): self.log.debug( f'prepare replace {nkey} = {nvalue} in {self.key}' ) - self._replace = type(self)(self.view, self.key) + self._replace = type(self)( + self.view, self.key, auth_managers=self.auth_managers + ) self.state.set('replace') else: raise ValueError( @@ -450,6 +455,7 @@ def fields(self, *argv): def key_repr(self): return repr(self.key) + @cli.change_pointer def create(self, **spec): ''' Create an RTNL object of the same type, and add it to the @@ -483,6 +489,8 @@ def create(self, **spec): spec['ndb_chain'] = self return self.view[spec] + @cli.show_result + @check_auth('obj:read') def show(self, fmt=None): ''' Return the object in a specified format. The format may be @@ -556,7 +564,8 @@ def wr_handler(wr, fname, *argv): ) ) - async def snapshot(self, ctxid=None): + @check_auth('obj:modify') + def snapshot(self, ctxid=None): ''' Create and return a snapshot of the object. The method creates corresponding SQL tables for the object itself and for detected @@ -571,10 +580,11 @@ async def snapshot(self, ctxid=None): else: key = self._replace.key snp = type(self)( - self.view, key, ctxid=ctxid, flags=ObjectFlags.SNAPSHOT + self.view, key, ctxid=ctxid, auth_managers=self.auth_managers + ) + self.ndb.task_manager.db_save_deps( + ctxid, weakref.ref(snp), self.iclass ) - snp.register() - self.ndb.schema.save_deps(ctxid, weakref.ref(snp), self.iclass) snp.changed = set(self.changed) return snp @@ -591,7 +601,7 @@ def complete_key(self, key): It is an internal method and is not supposed to be used externally. ''' - self.log.debug(f'complete key {key} from table {self.etable}') + self.log.debug('complete key %s from table %s' % (key, self.etable)) fetch = [] if isinstance(key, Record): key = key._as_dict() @@ -607,7 +617,7 @@ def complete_key(self, key): for name in self.kspec: if name not in key: - fetch.append(f'f_{name}') + fetch.append('f_%s' % name) if fetch: keys = [] @@ -617,9 +627,9 @@ def complete_key(self, key): if nla_name in self.spec: name = nla_name if value is not None and name in self.spec: - keys.append(f'f_{name} = ?') + keys.append('f_%s = %s' % (name, self.schema.plch)) values.append(value) - spec = self.ndb.schema.fetchone( + spec = self.ndb.task_manager.db_fetchone( 'SELECT %s FROM %s WHERE %s' % (' , '.join(fetch), self.etable, ' AND '.join(keys)), values, @@ -630,7 +640,7 @@ def complete_key(self, key): for name, value in zip(fetch, spec): key[name[2:]] = value - self.log.debug(f'got {key}') + self.log.debug('got %s' % key) return key def exists(self, key): @@ -639,7 +649,8 @@ def exists(self, key): ''' return self.view.exists(key) - async def rollback(self, snapshot=None): + @check_auth('obj:modify') + def rollback(self, snapshot=None): ''' Try to rollback the object state using the snapshot provided as an argument or using `self.last_save`. @@ -648,7 +659,9 @@ async def rollback(self, snapshot=None): self.log.debug( 'rollback replace: %s :: %s' % (self.key, self._replace.key) ) - new_replace = type(self)(self.view, self.key) + new_replace = type(self)( + self.view, self.key, auth_managers=self.auth_managers + ) new_replace.state.set('remove') self.state.set('replace') self.update(self._replace) @@ -656,16 +669,13 @@ async def rollback(self, snapshot=None): self.log.debug('rollback: %s' % str(self.state.events)) snapshot = snapshot or self.last_save if snapshot == -1: - self.remove() - await self.apply() - return + return self.remove().apply() else: snapshot.state.set(self.state.get()) snapshot.rollback_chain = self._apply_script_snapshots - snapshot.flags &= ~ObjectFlags.SNAPSHOT - await snapshot.apply(rollback=True) + snapshot.apply(rollback=True) for link, snp in snapshot.snapshot_deps: - await link.rollback(snapshot=snp) + link.rollback(snapshot=snp) return self def clear(self): @@ -679,7 +689,8 @@ def clean(self): and not self._apply_script ) - async def commit(self): + @check_auth('obj:modify') + def commit(self): ''' Commit the pending changes. If an exception is raised during `commit()`, automatically `rollback()` to the latest saved snapshot. @@ -688,7 +699,7 @@ async def commit(self): return self if self.chain: - await self.chain.commit() + self.chain.commit() self.log.debug('commit: %s' % str(self.state.events)) # Is it a new object? if self.state == 'invalid': @@ -696,7 +707,7 @@ async def commit(self): save = dict(self) self.last_save = -1 try: - return await self.apply(mode='commit') + return self.apply(mode='commit') except Exception as e_i: # Save the debug info e_i.trace = traceback.format_exc() @@ -716,17 +727,17 @@ async def commit(self): # variable will be saved in the traceback, so the tables will be # available to debug. If the traceback will be saved somewhere then # the tables will never be dropped by the GC, so you can do it - # manually by `ndb.schema.purge_snapshots()` -- to invalidate + # manually by `ndb.task_manager.db_purge_snapshots()` -- to invalidate # all the snapshots and to drop the associated tables. - self.last_save = await self.snapshot() + self.last_save = self.snapshot() # Apply the changes try: - await self.apply(mode='commit') + self.apply(mode='commit') except Exception as e_c: # Rollback in the case of any error try: - await self.rollback() + self.rollback() except Exception as e_r: e_c.chain = [e_r] if hasattr(e_r, 'chain'): @@ -785,10 +796,10 @@ def get_count(self): conditions = [] values = [] for name in self.kspec: - conditions.append(f'f_{name} = ?') + conditions.append('f_%s = %s' % (name, self.schema.plch)) values.append(self.get(self.iclass.nla2name(name), None)) return ( - self.ndb.schema.fetchone( + self.ndb.task_manager.db_fetchone( ''' SELECT count(*) FROM %s WHERE %s ''' @@ -797,29 +808,31 @@ def get_count(self): ) )[0] - async def hook_apply(self, method, **spec): + def hook_apply(self, method, **spec): pass - async def save_context(self): + @check_auth('obj:modify') + def save_context(self): if self.state == 'invalid': self.last_save = -1 else: - self.last_save = await self.snapshot() + self.last_save = self.snapshot() return self - async def apply(self, rollback=False, req_filter=None, mode='apply'): + @check_auth('obj:modify') + def apply(self, rollback=False, req_filter=None, mode='apply'): ''' Apply the pending changes. If an exception is raised during `apply()`, no `rollback()` is called. No automatic snapshots - are made. + are madre. In order to properly revert the changes, you have to run:: - await obj.save_context() + obj.save_context() try: - await obj.apply() + obj.apply() except Exception: - await obj.rollback() + obj.rollback() ''' # Resolve the fields @@ -837,7 +850,7 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): # Load the current state try: - self.schema.commit() + self.task_manager.db_commit() except Exception: pass self.load_sql(set_state=False) @@ -887,11 +900,11 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): for itn in range(10): try: self.log.debug('API call %s (%s)' % (method, req)) - await self.sources[self['target']].api(self.api, method, **req) + (self.sources[self['target']].api(self.api, method, **req)) first_call_success = True - await self.hook_apply(method, **req) + (self.hook_apply(method, **req)) except NetlinkError as e: - self.log.debug('error: %s' % e) + (self.log.debug('error: %s' % e)) if not first_call_success: self.log.debug('error on the first API call, escalate') raise @@ -914,13 +927,13 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): if isinstance( self.fallback_for[method][e.code], str ): - await self.sources[self['target']].api( + self.sources[self['target']].api( self.api, self.fallback_for[method][e.code], **req, ) else: - await self.fallback_for[method][e.code]( + self.fallback_for[method][e.code]( self, idx_req, req ) except NetlinkError: @@ -945,14 +958,11 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): self.log.debug('checked') break self.log.debug('check failed') - try: - await asyncio.wait_for(self.load_event.wait(), 1) - except asyncio.TimeoutError: - pass + self.load_event.wait(wtime) self.load_event.clear() else: self.log.debug('stats: %s apply %s fail' % (id(self), method)) - if not await self.use_db_resync(lambda x: x, self.check): + if not self.use_db_resync(lambda x: x, self.check): self._apply_script = [] raise Exception('could not apply the changes') @@ -960,7 +970,7 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): # if state == 'replace': self._replace.remove() - await self._replace.apply() + self._replace.apply() # if rollback: # @@ -970,7 +980,7 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): continue table = cls.table # comprare the tables - diff = self.ndb.schema.fetch( + diff = self.ndb.task_manager.db_fetch( ''' SELECT * FROM %s_%s EXCEPT @@ -998,18 +1008,18 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): obj.state.set('invalid') obj.register() try: - await obj.apply() + obj.apply() except Exception as e: self.errors.append((time.time(), obj, e)) for obj in reversed(self.rollback_chain): - await obj.rollback() + obj.rollback() else: apply_script = self._apply_script self._apply_script = [] for op, kwarg in apply_script: kwarg['self'] = self kwarg['mode'] = mode - ret = await self.use_db_resync( + ret = self.use_db_resync( lambda x: not isinstance(x, KeyError), op, tuple(), kwarg ) if not isinstance(ret, list): @@ -1021,28 +1031,34 @@ async def apply(self, rollback=False, req_filter=None, mode='apply'): self._apply_script_snapshots.append(obj) return self - async def use_db_resync(self, criteria, method, argv=None, kwarg=None): + def use_db_resync(self, criteria, method, argv=None, kwarg=None): ret = None argv = argv or [] kwarg = kwarg or {} self.log.debug(f'criteria {criteria}') self.log.debug(f'method {method}, {argv}, {kwarg}') for attempt in range(3): - ret = [] - for k in method(*argv, **kwarg): - if isinstance(k, Awaitable): - k = await k - ret.append(k) + ret = method(*argv, **kwarg) self.log.debug(f'ret {ret}') if criteria(ret): self.log.debug('criteria matched') return ret self.log.debug(f'resync the DB attempt {attempt}') - self.ndb.schema.flush(self['target']) + self.ndb.task_manager.db_flush(self['target']) self.load_event.clear() - await self.sources[self['target']].api('dump') - # await self.evq.put(cmsg_event(self['target'], self.load_event)) - await self.load_event.wait() + ( + self.ndb._event_queue.put( + self.sources[self['target']].api('dump'), + source=self['target'], + ) + ) + ( + self.ndb._event_queue.put( + (cmsg_event(self['target'], self.load_event),), + source=self['target'], + ) + ) + self.load_event.wait(self.wtime(1)) self.load_event.clear() return ret @@ -1086,6 +1102,7 @@ def load_sql(self, table=None, ctxid=None, set_state=True): ''' Load the data from the database. ''' + if not self.key: return @@ -1098,12 +1115,12 @@ def load_sql(self, table=None, ctxid=None, set_state=True): values = [] for name, value in self.key.items(): - keys.append(f'f_{name} = ?') + keys.append('f_%s = %s' % (name, self.schema.plch)) if isinstance(value, (list, tuple, dict)): value = json.dumps(value) values.append(value) - spec = self.ndb.schema.fetchone( + spec = self.ndb.task_manager.db_fetchone( 'SELECT * FROM %s WHERE %s' % (table, ' AND '.join(keys)), values ) self.log.debug('load_sql load: %s' % str(spec)) @@ -1120,21 +1137,17 @@ def load_sql(self, table=None, ctxid=None, set_state=True): self.state.set('system') return spec - async def load_rtnlmsg(self, sources, target, event): + def load_rtnlmsg(self, target, event): ''' Check if the RTNL event matches the object and load the data from the database if it does. ''' # TODO: partial match (object rename / restore) # ... - if ObjectFlags.SNAPSHOT in self.flags: - return # full match for norm, name in zip(self.knorm, self.kspec): value = self.get(norm) - if value is None: - continue if name == 'target': if value != target: return @@ -1150,102 +1163,3 @@ async def load_rtnlmsg(self, sources, target, event): else: self.load_sql() self.load_event.set() - - -class RTNL_Object(SyncBase): - - # FIXME 3.9: Union - def apply( - self, - rollback: bool = False, - req_filter: Union[None, Callable[[Req], Req]] = None, - mode: str = 'apply', - ) -> SyncBase: - self._main_async_call(self.asyncore.apply, rollback, req_filter, mode) - return self - - @property - def state(self): - return self.asyncore.state - - @property - def chain(self): - return self._get_sync_class( - self.asyncore.chain, key=self.asyncore.chain.table - ) - - @property - def table(self): - return self.asyncore.table - - @property - def etable(self): - return self.asyncore.etable - - @property - def key(self): - return self.asyncore.key - - def complete_key(self, key): - return self._main_sync_call(self.asyncore.complete_key, key) - - def exists(self, key): - return self._main_sync_call(self.asyncore.exists, key) - - def load_sql(self, table=None, ctxid=None, set_state=True): - return self._main_sync_call( - self.asyncore.load_sql, table, ctxid, set_state - ) - - def load_value(self, key, value): - return self._main_sync_call(self.asyncore.load_value, key, value) - - def snapshot(self, ctxid=None): - return self._main_async_call(self.asyncore.snapshot, ctxid) - - def create(self, **spec): - item = self._main_sync_call(self.asyncore.create, **spec) - return type(self)(self.event_loop, item) - - def commit(self) -> SyncBase: - self._main_async_call(self.asyncore.commit) - return self - - def rollback(self, snapshot=None): - self._main_async_call(self.asyncore.rollback, snapshot) - return self - - def show(self, fmt=None): - return self.asyncore.show(fmt) - - def keys(self): - return self.asyncore.keys() - - def items(self): - return self.asyncore.items() - - def set(self, *argv, **kwarg): - self._main_sync_call(self.asyncore.set, *argv, **kwarg) - return self - - def get(self, key, *argv): - return self.asyncore.get(key, *argv) - - def remove(self): - self.asyncore.remove() - return self - - def __enter__(self): - return self - - def __exit__(self, ext_type, exc_value, traceback): - self.commit() - - def __repr__(self): - return repr(self.asyncore) - - def __getitem__(self, key): - return self.asyncore[key] - - def __setitem__(self, key, value): - return self.set(key, value) diff --git a/pyroute2/ndb/objects/address.py b/pyroute2/ndb/objects/address.py index 7bcadd5eb..81a16462e 100644 --- a/pyroute2/ndb/objects/address.py +++ b/pyroute2/ndb/objects/address.py @@ -5,7 +5,7 @@ from socket import AF_INET from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True ndb = NDB() .. testcleanup:: * @@ -140,7 +140,7 @@ Access an address as a separate RTNL object: -.. code:: +.. testcode:: x3 print(ndb.addresses['192.168.122.28/24']) @@ -159,14 +159,14 @@ from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg from pyroute2.requests.address import AddressFieldFilter -from ..objects import AsyncObject +from ..objects import RTNL_Object -async def load_ifaddrmsg(schema, sources, target, event): +def load_ifaddrmsg(schema, target, event): # # bypass # - await schema.load_netlink('addresses', sources, target, event) + schema.load_netlink('addresses', target, event) # # last address removal should trigger routes flush # Bug-Url: https://github.com/svinota/pyroute2/issues/849 @@ -177,43 +177,25 @@ async def load_ifaddrmsg(schema, sources, target, event): # addresses = schema.execute( ''' - SELECT * FROM addresses WHERE - f_target = ? AND - f_index = ? AND - f_family = 2 - ''', + SELECT * FROM addresses WHERE + f_target = %s AND + f_index = %s AND + f_family = 2 + ''' + % (schema.plch, schema.plch), (target, event['index']), ).fetchmany() if not len(addresses): schema.execute( ''' - DELETE FROM routes WHERE - f_target = ? AND - f_RTA_OIF = ? OR - f_RTA_IIF = ? - ''', + DELETE FROM routes WHERE + f_target = %s AND + f_RTA_OIF = %s OR + f_RTA_IIF = %s + ''' + % (schema.plch, schema.plch, schema.plch), (target, event['index'], event['index']), ) - # Take care of multipath routes - schema.execute( - ''' - DELETE FROM nh WHERE - f_target = ? AND - f_oif = ? - ''', - (target, event['index']), - ) - - schema.execute( - ''' - DELETE FROM routes WHERE - f_target = ? AND - f_deps = 1 AND - f_route_id NOT IN - (SELECT n.f_route_id FROM nh n) - ''', - (target,), - ) ifaddr_spec = ( @@ -233,7 +215,7 @@ async def load_ifaddrmsg(schema, sources, target, event): } -class Address(AsyncObject): +class Address(RTNL_Object): table = 'addresses' msg_class = ifaddrmsg field_filter = AddressFieldFilter @@ -242,23 +224,28 @@ class Address(AsyncObject): @classmethod def _count(cls, view): if view.chain: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table} WHERE f_index = ?', + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s WHERE f_index = %s' + % (view.table, view.ndb.schema.plch), [view.chain['index']], ) else: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table}' + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s' % view.table ) @classmethod def _dump_where(cls, view): if view.chain: + plch = view.ndb.schema.plch where = ''' - WHERE - main.f_target = ? AND - main.f_index = ? - ''' + WHERE + main.f_target = %s AND + main.f_index = %s + ''' % ( + plch, + plch, + ) values = [view.chain['target'], view.chain['index']] else: where = '' @@ -281,16 +268,18 @@ def summary(cls, view): ''' yield ('target', 'tflags', 'ifname', 'address', 'prefixlen') where, values = cls._dump_where(view) - for record in view.ndb.schema.fetch(req + where, values): + for record in view.ndb.task_manager.db_fetch(req + where, values): yield record def mark_tflags(self, mark): + plch = (self.schema.plch,) * 3 self.schema.execute( ''' - UPDATE interfaces SET - f_tflags = ? - WHERE f_index = ? AND f_target = ? - ''', + UPDATE interfaces SET + f_tflags = %s + WHERE f_index = %s AND f_target = %s + ''' + % plch, (mark, self['index'], self['target']), ) @@ -308,7 +297,7 @@ def compare_record(left, right): ) @classmethod - def spec_normalize(cls, spec): + def spec_normalize(cls, processed, spec): ''' Address key normalization:: @@ -317,8 +306,8 @@ def spec_normalize(cls, spec): "prefixlen": 24} ''' if isinstance(spec, str): - return {'address': spec} - return spec + processed['address'] = spec + return processed def key_repr(self): return '%s/%s %s/%s' % ( diff --git a/pyroute2/ndb/objects/interface.py b/pyroute2/ndb/objects/interface.py index 1b5367d11..a995a19ff 100644 --- a/pyroute2/ndb/objects/interface.py +++ b/pyroute2/ndb/objects/interface.py @@ -2,10 +2,11 @@ .. testsetup:: + from pyroute2 import IPMock as IPRoute from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True .. testsetup:: preset_1 @@ -13,12 +14,12 @@ from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True ndb = NDB( sources=[ - {'target': 'localhost', 'kind': 'local'}, - {'target': 'worker1.sample.com', 'kind': 'local'}, - {'target': 'worker2.sample.com', 'kind': 'local'}, + {'target': 'localhost', 'kind': 'IPMock'}, + {'target': 'worker1.sample.com', 'kind': 'IPMock'}, + {'target': 'worker2.sample.com', 'kind': 'IPMock'}, ] ) @@ -26,7 +27,7 @@ from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True ndb = NDB() ndb.interfaces.create(ifname='eth1', kind='dummy').commit() ndb.interfaces.create(ifname='br0', kind='bridge').commit() @@ -36,7 +37,7 @@ from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True ndb = NDB() ndb.interfaces.create(ifname='br0', kind='bridge').commit() ndb.interfaces['br0'].add_port('eth0').commit() @@ -150,7 +151,7 @@ Add bridge and bond ports one can use specific API: -.. code:: +.. testcode:: preset_br0_1 with ndb.interfaces['br0'] as br0: br0.add_port('eth0') @@ -164,7 +165,7 @@ To remove a port: -.. code:: +.. testcode:: preset_br0_2 with ndb.interfaces['br0'] as br0: br0.del_port('eth0') @@ -172,7 +173,7 @@ Or by setting the master property on a port, in the same way as with `IPRoute`: -.. code:: +.. testcode:: preset_br0_1 index = ndb.interfaces['br0']['index'] @@ -187,7 +188,6 @@ import errno import json -import os import traceback from pyroute2.common import basestring @@ -197,21 +197,20 @@ from pyroute2.netlink.rtnl.p2pmsg import p2pmsg from pyroute2.requests.link import LinkFieldFilter -from ..objects import AsyncObject, RTNL_Object -from ..sync_api import Flags, SyncView +from ..auth_manager import AuthManager, check_auth +from ..objects import RTNL_Object -async def load_ifinfmsg(schema, sources, target, event): +def load_ifinfmsg(schema, target, event): # # link goes down: flush all related routes # if not event['flags'] & 1: schema.execute( - ''' - DELETE FROM routes WHERE - f_target = ? AND - f_RTA_OIF = ? OR f_RTA_IIF = ? - ''', + 'DELETE FROM routes WHERE ' + 'f_target = %s AND ' + 'f_RTA_OIF = %s OR f_RTA_IIF = %s' + % (schema.plch, schema.plch, schema.plch), (target, event['index'], event['index']), ) # @@ -233,7 +232,7 @@ async def load_ifinfmsg(schema, sources, target, event): # if event['family'] == AF_BRIDGE: # - await schema.load_netlink('af_bridge_ifs', sources, target, event) + schema.load_netlink('af_bridge_ifs', target, event) try: vlans = event.get_attr('IFLA_AF_SPEC').get_attrs( 'IFLA_BRIDGE_VLAN_INFO' @@ -246,21 +245,22 @@ async def load_ifinfmsg(schema, sources, target, event): # flush the old vlans info schema.execute( ''' - DELETE FROM af_bridge_vlans - WHERE - f_target = ? - AND f_index = ? - ''', + DELETE FROM af_bridge_vlans + WHERE + f_target = %s + AND f_index = %s + ''' + % (schema.plch, schema.plch), (target, event['index']), ) for v in vlans: v['index'] = event['index'] v['header'] = {'type': event['header']['type']} - await schema.load_netlink('af_bridge_vlans', sources, target, v) + schema.load_netlink('af_bridge_vlans', target, v) return - await schema.load_netlink('interfaces', sources, target, event) + schema.load_netlink('interfaces', target, event) # # load ifinfo, if exists # @@ -268,7 +268,7 @@ async def load_ifinfmsg(schema, sources, target, event): linkinfo = event.get_attr('IFLA_LINKINFO') if linkinfo is not None: iftype = linkinfo.get_attr('IFLA_INFO_KIND') - table = f'ifinfo_{iftype}' + table = 'ifinfo_%s' % iftype if iftype == 'gre': ifdata = linkinfo.get_attr('IFLA_INFO_DATA') local = ifdata.get_attr('IFLA_GRE_LOCAL') @@ -277,7 +277,7 @@ async def load_ifinfmsg(schema, sources, target, event): p2p['index'] = event['index'] p2p['family'] = 2 p2p['attrs'] = [('P2P_LOCAL', local), ('P2P_REMOTE', remote)] - await schema.load_netlink('p2p', sources, target, p2p) + schema.load_netlink('p2p', target, p2p) elif iftype == 'veth': link = event.get_attr('IFLA_LINK') ifname = event.get_attr('IFLA_IFNAME') @@ -286,14 +286,14 @@ async def load_ifinfmsg(schema, sources, target, event): if (not link) and ( (target,) in schema.fetch('SELECT f_target FROM SOURCES') ): - schema.log.debug(f'reload veth {event["index"]}') + schema.log.debug('reload veth %s' % event['index']) try: - update = await sources[target].api( + update = schema.sources[target].api( 'link', 'get', index=event['index'] ) update = tuple(update)[0] - return await schema.load_netlink( - 'interfaces', sources, target, update + return schema.load_netlink( + 'interfaces', target, update ) except NetlinkError as e: if e.code == errno.ENODEV: @@ -304,7 +304,7 @@ async def load_ifinfmsg(schema, sources, target, event): if ifdata is not None: ifdata['header'] = {} ifdata['index'] = event['index'] - await schema.load_netlink(table, sources, target, ifdata) + schema.load_netlink(table, target, ifdata) ip_tunnels = ('gre', 'gretap', 'ip6gre', 'ip6gretap', 'ip6tnl', 'sit', 'ipip') @@ -386,7 +386,7 @@ async def load_ifinfmsg(schema, sources, target, event): # load supported ifinfo # for name, data in supported_ifinfo.items(): - name = f'ifinfo_{name}' + name = 'ifinfo_%s' % name init['classes'].append([name, data]) schema = ( data.sql_schema() @@ -410,7 +410,7 @@ def _cmp_master(self, value): return False -class Vlan(AsyncObject): +class Vlan(RTNL_Object): table = 'af_bridge_vlans' msg_class = ifinfmsg.af_spec_bridge.vlan_info api = 'vlan_filter' @@ -418,19 +418,28 @@ class Vlan(AsyncObject): @classmethod def _count(cls, view): if view.chain: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table} WHERE f_index = ?', + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s WHERE f_index = %s' + % (view.table, view.ndb.schema.plch), [view.chain['index']], ) else: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table}' + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s' % view.table ) @classmethod def _dump_where(cls, view): if view.chain: - where = 'WHERE main.f_target = ? AND main.f_index = ?' + plch = view.ndb.schema.plch + where = ''' + WHERE + main.f_target = %s AND + main.f_index = %s + ''' % ( + plch, + plch, + ) values = [view.chain['target'], view.chain['index']] else: where = '' @@ -453,7 +462,7 @@ def summary(cls, view): ''' yield ('target', 'tflags', 'vid', 'ifname') where, values = cls._dump_where(view) - for record in view.ndb.schema.fetch(req + where, values): + for record in view.ndb.task_manager.db_fetch(req + where, values): yield record @staticmethod @@ -463,6 +472,14 @@ def compare_record(left, right): def __init__(self, *argv, **kwarg): kwarg['iclass'] = ifinfmsg.af_spec_bridge.vlan_info + if 'auth_managers' not in kwarg or kwarg['auth_managers'] is None: + kwarg['auth_managers'] = [] + log = argv[0].ndb.log.channel('vlan auth') + kwarg['auth_managers'].append( + AuthManager( + {'obj:read': True, 'obj:list': True, 'obj:modify': False}, log + ) + ) super(Vlan, self).__init__(*argv, **kwarg) def make_req(self, prime): @@ -478,7 +495,7 @@ def make_idx_req(self, prime): return self.make_req(prime) -class Interface(AsyncObject): +class Interface(RTNL_Object): table = 'interfaces' msg_class = ifinfmsg api = 'link' @@ -489,24 +506,32 @@ class Interface(AsyncObject): 'alt_ifname_list': lambda x: list(json.loads(x or '[]')) } field_filter = LinkFieldFilter - old_ifname = None @classmethod def _count(cls, view): if view.chain: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table} WHERE f_IFLA_MASTER = ?', + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s WHERE f_IFLA_MASTER = %s' + % (view.table, view.ndb.schema.plch), [view.chain['index']], ) else: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table}' + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s' % view.table ) @classmethod def _dump_where(cls, view): if view.chain: - where = 'WHERE f_target = ? AND f_IFLA_MASTER = ?' + plch = view.ndb.schema.plch + where = ''' + WHERE + f_target = %s AND + f_IFLA_MASTER = %s + ''' % ( + plch, + plch, + ) values = [view.chain['target'], view.chain['index']] else: where = 'WHERE f_index != 0' @@ -533,16 +558,18 @@ def summary(cls, view): 'kind', ) where, values = cls._dump_where(view) - for record in view.ndb.schema.fetch(req + where, values): + for record in view.ndb.task_manager.db_fetch(req + where, values): yield record def mark_tflags(self, mark): + plch = (self.schema.plch,) * 3 self.schema.execute( ''' - UPDATE interfaces SET - f_tflags = ? - WHERE f_index = ? AND f_target = ? - ''', + UPDATE interfaces SET + f_tflags = %s + WHERE f_index = %s AND f_target = %s + ''' + % plch, (mark, self['index'], self['target']), ) @@ -614,10 +641,11 @@ def compare_record(self, left, right): if isinstance(right, basestring): return right == left['ifname'] or right == left['address'] + @check_auth('obj:modify') def add_vlan(self, spec): def do_add_vlan(self, mode, spec): try: - method = getattr(self.vlans.create(spec), mode) + method = getattr(self.vlan.create(spec), mode) return [method()] except Exception as e_s: e_s.trace = traceback.format_stack() @@ -626,10 +654,11 @@ def do_add_vlan(self, mode, spec): self._apply_script.append((do_add_vlan, {'spec': spec})) return self + @check_auth('obj:modify') def ensure_vlan(self, spec): def do_ensure_vlan(self, mode, spec): try: - method = getattr(self.vlans.create(spec), mode) + method = getattr(self.vlan.create(spec), mode) return [method()] except KeyError: return [] @@ -640,10 +669,11 @@ def do_ensure_vlan(self, mode, spec): self._apply_script.append((do_ensure_vlan, {'spec': spec})) return self + @check_auth('obj:modify') def del_vlan(self, spec): def do_del_vlan(self, mode, spec): try: - method = getattr(self.vlans[spec].remove(), mode) + method = getattr(self.vlan[spec].remove(), mode) return [method()] except Exception as e_s: e_s.trace = traceback.format_stack() @@ -652,6 +682,7 @@ def do_del_vlan(self, mode, spec): self._apply_script.append((do_del_vlan, {'spec': spec})) return self + @check_auth('obj:modify') def add_neighbour(self, spec=None, **kwarg): spec = spec or kwarg @@ -666,6 +697,7 @@ def do_add_neighbour(self, mode, spec): self._apply_script.append((do_add_neighbour, {'spec': spec})) return self + @check_auth('obj:modify') def ensure_neighbour(self, spec=None, **kwarg): spec = spec or kwarg @@ -682,6 +714,7 @@ def do_ensure_neighbour(self, mode, spec): self._apply_script.append((do_ensure_neighbour, {'spec': spec})) return self + @check_auth('obj:modify') def del_neighbour(self, spec=None, **kwarg): spec = spec or dict(kwarg) @@ -690,13 +723,11 @@ def do_del_neighbour(self, mode, spec): if isinstance(spec, basestring): specs = [spec] elif callable(spec): - specs = self.neighbours.dump() + specs = self.ipaddr.dump() specs.select_records(spec) - specs.materialize() else: - specs = self.neighbours.dump() + specs = self.ipaddr.dump() specs.select_records(**spec) - specs.materialize() for sp in specs: try: method = getattr(self.neighbours.locate(sp).remove(), mode) @@ -713,6 +744,7 @@ def do_del_neighbour(self, mode, spec): self._apply_script.append((do_del_neighbour, {'spec': spec})) return self + @check_auth('obj:modify') def add_ip(self, spec=None, **kwarg): spec = spec or kwarg @@ -727,6 +759,7 @@ def do_add_ip(self, mode, spec): self._apply_script.append((do_add_ip, {'spec': spec})) return self + @check_auth('obj:modify') def ensure_ip(self, spec=None, **kwarg): spec = spec or kwarg @@ -743,6 +776,7 @@ def do_ensure_ip(self, mode, spec): self._apply_script.append((do_ensure_ip, {'spec': spec})) return self + @check_auth('obj:modify') def del_ip(self, spec=None, **kwarg): spec = spec or kwarg @@ -753,11 +787,9 @@ def do_del_ip(self, mode, spec): elif callable(spec): specs = self.ipaddr.dump() specs.select_records(spec) - specs.materialize() else: specs = self.ipaddr.dump() specs.select_records(**spec) - specs.materialize() for sp in specs: try: method = getattr(self.ipaddr.locate(sp).remove(), mode) @@ -768,22 +800,22 @@ def do_del_ip(self, mode, spec): e_s.trace = traceback.format_stack() ret.append(e_s) if not ret: - ret = [KeyError('no address records matched')] + ret = KeyError('no address records matched') return ret self._apply_script.append((do_del_ip, {'spec': spec})) return self - def add_port(self, spec=None, **kwarg): - spec = spec or kwarg - + @check_auth('obj:modify') + def add_port(self, spec): def do_add_port(self, mode, spec): try: port = self.view[spec] if port['target'] != self['target']: raise ValueError('target must be the same') port['master'] = self['index'] - return [getattr(port, mode)()] + getattr(port, mode)() + return [port] except Exception as e_s: e_s.trace = traceback.format_stack() return [e_s] @@ -791,9 +823,8 @@ def do_add_port(self, mode, spec): self._apply_script.append((do_add_port, {'spec': spec})) return self - def del_port(self, spec=None, **kwarg): - spec = spec or kwarg - + @check_auth('obj:modify') + def del_port(self, spec): def do_del_port(self, mode, spec): try: port = self.view[spec] @@ -802,7 +833,8 @@ def do_del_port(self, mode, spec): if port['target'] != self['target']: raise ValueError('target must be the same') port['master'] = 0 - return [getattr(port, mode)()] + getattr(port, mode)() + return [port] except Exception as e_s: e_s.trace = traceback.format_stack() return [e_s] @@ -810,27 +842,22 @@ def do_del_port(self, mode, spec): self._apply_script.append((do_del_port, {'spec': spec})) return self + @check_auth('obj:modify') def add_altname(self, ifname): new_list = set(self['alt_ifname_list']) new_list.add(ifname) self['alt_ifname_list'] = list(new_list) - return self + @check_auth('obj:modify') def del_altname(self, ifname): new_list = set(self['alt_ifname_list']) new_list.remove(ifname) self['alt_ifname_list'] = list(new_list) - return self + @check_auth('obj:modify') def __setitem__(self, key, value): if key == 'peer': dict.__setitem__(self, key, value) - elif key == 'ifname': - if value in self['alt_ifname_list']: - self.del_altname(value) - if key in self and self.old_ifname is None: - self.old_ifname = self[key] - super(Interface, self).__setitem__(key, value) elif key == 'target' and self.state == 'invalid': dict.__setitem__(self, key, value) elif key == 'net_ns_fd' and self.state == 'invalid': @@ -843,7 +870,7 @@ def __setitem__(self, key, value): super(Interface, self).__setitem__(key, value) @classmethod - def spec_normalize(cls, spec): + def spec_normalize(cls, processed, spec): ''' Interface key normalization:: @@ -853,15 +880,10 @@ def spec_normalize(cls, spec): ''' if isinstance(spec, basestring): - return {'ifname': spec} + processed['ifname'] = spec elif isinstance(spec, int): - return {'index': spec} - elif isinstance(spec, ifinfmsg): - ret = {'ifname': spec.get('ifname')} - if spec.get('index') != 0: - ret['index'] = spec.get('index') - return ret - return spec + processed['index'] = spec + return processed def complete_key(self, key): if isinstance(key, dict): @@ -874,7 +896,7 @@ def complete_key(self, key): ret_key['index'] = key return super(Interface, self).complete_key(ret_key) - async def is_peer(self, other): + def is_peer(self, other): '''Evaluate whether the given interface "points at" this one.''' if other['kind'] == 'vlan': return ( @@ -896,10 +918,11 @@ async def is_peer(self, other): other_link_netnsid = other.get('link_netnsid') if other_link_netnsid is not None: + self_source = self.sources[self['target']] other_source = other.sources[other['target']] - info = await other_source.api( + info = other_source.api( 'get_netnsid', - pid=os.getpid(), + pid=self_source.api('get_pid'), target_nsid=other_link_netnsid, ) return info['current_nsid'] == other_link_netnsid @@ -911,23 +934,25 @@ def set_xdp_fd(self, fd): 'link', 'set', index=self['index'], xdp_fd=fd ) - async def snapshot(self, ctxid=None): + def snapshot(self, ctxid=None): # 1. make own snapshot - snp = await super().snapshot(ctxid=ctxid) + snp = super(Interface, self).snapshot(ctxid=ctxid) # 2. collect dependencies and store in self.snapshot_deps - for spec in self.ndb.interfaces.asyncore.getmany( + for spec in self.ndb.interfaces.getmany( {'IFLA_MASTER': self['index']} ): # bridge ports - link = type(self)(self.view, spec) - snp.snapshot_deps.append((link, await link.snapshot())) - for spec in self.ndb.interfaces.asyncore.getmany( - {'IFLA_LINK': self['index']} - ): - link = type(self)(self.view, spec) + link = type(self)( + self.view, spec, auth_managers=self.auth_managers + ) + snp.snapshot_deps.append((link, link.snapshot())) + for spec in self.ndb.interfaces.getmany({'IFLA_LINK': self['index']}): + link = type(self)( + self.view, spec, auth_managers=self.auth_managers + ) # vlans & veth - if await self.is_peer(link) and not await link.is_peer(self): - snp.snapshot_deps.append((link, await link.snapshot())) + if self.is_peer(link) and not link.is_peer(self): + snp.snapshot_deps.append((link, link.snapshot())) # return the root node return snp @@ -951,79 +976,44 @@ def make_req(self, prime): req[key] = self[key] return req - async def apply_altnames( - self, alt_ifname_setup, alt_ifname_current, old_ifname=None - ): - if 'alt_ifname_list' in self.changed: - self.changed.remove('alt_ifname_list') - if alt_ifname_current is None: - # load the current state - await self.load_from_system() - self.load_sql(set_state=False) - alt_ifname_current = set(self['alt_ifname_list']) - - alt_ifname_remove = alt_ifname_current - alt_ifname_setup - alt_ifname_add = alt_ifname_setup - alt_ifname_current + @check_auth('obj:modify') + def apply_altnames(self, alt_ifname_setup): + alt_ifname_remove = set(self['alt_ifname_list']) - alt_ifname_setup + alt_ifname_add = alt_ifname_setup - set(self['alt_ifname_list']) for ifname in alt_ifname_remove: - await self.sources[self['target']].api( + self.sources[self['target']].api( 'link', 'property_del', index=self['index'], altname=ifname ) for ifname in alt_ifname_add: - await self.sources[self['target']].api( + self.sources[self['target']].api( 'link', 'property_add', index=self['index'], altname=ifname ) - # reload alt ifnames from the system to check the state - await self.load_from_system() + self.load_from_system() self.load_sql(set_state=False) - if old_ifname is not None and old_ifname in self['alt_ifname_list']: - alt_ifname_setup.add(old_ifname) if set(self['alt_ifname_list']) != alt_ifname_setup: raise Exception('could not setup alt ifnames') - async def apply(self, rollback=False, req_filter=None, mode='apply'): + @check_auth('obj:modify') + def apply(self, rollback=False, req_filter=None, mode='apply'): # translate string link references into numbers for key in ('link', 'master'): if key in self and isinstance(self[key], basestring): - self[key] = self.ndb.interfaces.asyncore[self[key]]['index'] + self[key] = self.ndb.interfaces[self[key]]['index'] setns = self.state.get() == 'setns' remove = self.state.get() == 'remove' alt_ifname_setup = set(self['alt_ifname_list']) - old_ifname = self.old_ifname if 'ifname' in self.changed else None + if 'alt_ifname_list' in self.changed: + self.changed.remove('alt_ifname_list') try: - if 'index' in self and ( - self.old_ifname or 'alt_ifname_list' in self.changed - ): - await self.apply_altnames(alt_ifname_setup, None) - if 'alt_ifname_list' in self.changed: - self.changed.remove('alt_ifname_list') - await super().apply(rollback, req_filter, mode) - if setns and self['net_ns_fd'] in self.sources: + super(Interface, self).apply(rollback, req_filter, mode) + if setns: self.load_value('target', self['net_ns_fd']) dict.__setitem__(self, 'net_ns_fd', None) - for link in await self.sources[self['target']].api( - 'link', 'get', ifname=self['ifname'] - ): - # after interface move the name is the same, - # but the index may change - # - # in this case .load_sql() will not update - # the object, and the engine will try to apply - # the interface's attributes to another interface - # with the same index as our old one - # - # so resync the index first - # - # Bug-Url: https://github.com/svinota/pyroute2/issues/1181 - # - self.load_value('index', link['index']) - break spec = self.load_sql() if spec: self.state.set('system') - if not remove and self.state != 'invalid': - await self.apply_altnames( - alt_ifname_setup, set(self['alt_ifname_list']), old_ifname - ) + if not remove: + self.apply_altnames(alt_ifname_setup) except NetlinkError as e: if ( @@ -1045,8 +1035,8 @@ def req_filter(req): ] ) - await self.apply(rollback, req_filter, mode) - await self.apply(rollback, None, mode) + self.apply(rollback, req_filter, mode) + self.apply(rollback, None, mode) elif ( e.code == 95 @@ -1066,24 +1056,22 @@ def req_filter(req): ] ) - await self.apply(rollback, req_filter, mode) + self.apply(rollback, req_filter, mode) else: raise - finally: - self.old_ifname = None if ('net_ns_fd' in self.get('peer', {})) and ( self['peer']['net_ns_fd'] in self.view.ndb.sources ): # wait for the peer in net_ns_fd, only if the netns # is connected to the NDB instance - await self.view.wait( + self.view.wait( target=self['peer']['net_ns_fd'], ifname=self['peer']['ifname'], timeout=5, ) return self - async def hook_apply(self, method, **spec): + def hook_apply(self, method, **spec): if method == 'set': if self['kind'] == 'bridge': keys = filter(lambda x: x.startswith('br_'), self.changed) @@ -1095,119 +1083,52 @@ async def hook_apply(self, method, **spec): } for key in keys: req[key] = self[key] - await self.sources[self['target']].api( - self.api, method, **req - ) + self.sources[self['target']].api(self.api, method, **req) # FIXME: make a reasonable shortcut for this - await self.load_from_system() + self.load_from_system() elif self['kind'] in ip_tunnels and self['state'] == 'down': # force reading attributes for tunnels in the down state - await self.load_from_system() + self.load_from_system() elif method == 'add': if self['kind'] == 'tun': self.load_sql() - await self.load_event.wait() + self.load_event.wait(0.1) if 'index' not in self: raise NetlinkError(errno.EAGAIN) - update = await self.sources[self['target']].api( + update = self.sources[self['target']].api( self.api, 'get', index=self['index'] ) self.ndb._event_queue.put(update) - async def load_from_system(self): - self.load_event.clear() - await self.sources[self['target']].api( - self.api, 'get', index=self['index'] + def load_from_system(self): + ( + self.ndb._event_queue.put( + self.sources[self['target']].api( + self.api, 'get', index=self['index'] + ) + ) ) - await self.load_event.wait() def load_sql(self, *argv, **kwarg): spec = super(Interface, self).load_sql(*argv, **kwarg) if spec: - tname = f'ifinfo_{self["kind"]}' + tname = 'ifinfo_%s' % self['kind'] if tname in self.schema.compiled: names = self.schema.compiled[tname]['norm_names'] - spec = self.ndb.schema.fetchone( - f'SELECT * from {tname} WHERE f_index = ?', + spec = self.ndb.task_manager.db_fetchone( + 'SELECT * from %s WHERE f_index = %s' + % (tname, self.schema.plch), (self['index'],), ) if spec: self.update(dict(zip(names, spec))) return spec - async def load_rtnlmsg(self, *argv, **kwarg): - await super().load_rtnlmsg(*argv, **kwarg) + def load_rtnlmsg(self, *argv, **kwarg): + super(Interface, self).load_rtnlmsg(*argv, **kwarg) def key_repr(self): return '%s/%s' % ( self.get('target', ''), self.get('ifname', self.get('index', '')), ) - - -class SyncInterface(RTNL_Object): - - def __init__(self, event_loop, obj, class_map=None, flags=Flags.RO): - super().__init__(event_loop, obj, class_map, flags) - self.ipaddr = SyncView( - event_loop, obj.ipaddr, self.class_map, self.flags - ) - self.neighbours = SyncView( - event_loop, obj.neighbours, self.class_map, self.flags - ) - self.ports = SyncView( - event_loop, obj.ports, self.class_map, self.flags - ) - self.routes = SyncView( - event_loop, obj.routes, self.class_map, self.flags - ) - self.vlans = SyncView( - event_loop, obj.vlans, self.class_map, self.flags - ) - - @property - def state(self): - return self.asyncore.state - - def add_ip(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.add_ip, spec, **kwarg) - return self - - def del_ip(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.del_ip, spec, **kwarg) - return self - - def ensure_ip(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.ensure_ip, spec, **kwarg) - return self - - def add_neighbour(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.add_neighbour, spec, **kwarg) - return self - - def del_neighbour(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.del_neighbour, spec, **kwarg) - return self - - def ensure_neighbour(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.ensure_neighbour, spec, **kwarg) - return self - - def add_port(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.add_port, spec, **kwarg) - return self - - def del_port(self, spec=None, **kwarg): - self._main_sync_call(self.asyncore.del_port, spec, **kwarg) - return self - - def add_altname(self, ifname): - self._main_sync_call(self.asyncore.add_altname, ifname) - return self - - def del_altname(self, ifname): - self._main_sync_call(self.asyncore.del_altname, ifname) - return self - - def load_from_system(self): - self._main_async_call(self.asyncore.load_from_system) diff --git a/pyroute2/ndb/objects/neighbour.py b/pyroute2/ndb/objects/neighbour.py index aaa0af923..f8f55390b 100644 --- a/pyroute2/ndb/objects/neighbour.py +++ b/pyroute2/ndb/objects/neighbour.py @@ -6,10 +6,10 @@ from pyroute2.requests.neighbour import NeighbourFieldFilter from ..events import RescheduleException -from ..objects import AsyncObject +from ..objects import RTNL_Object -async def load_ndmsg(schema, sources, target, event): +def load_ndmsg(schema, target, event): # # ignore events with ifindex == 0 # @@ -26,14 +26,12 @@ async def load_ndmsg(schema, sources, target, event): # bypass for now # try: - await schema.load_netlink( - 'af_bridge_fdb', sources, target, event, propagate=True - ) + schema.load_netlink('af_bridge_fdb', target, event, propagate=True) except Exception: raise RescheduleException() else: - await schema.load_netlink('neighbours', sources, target, event) + schema.load_netlink('neighbours', target, event) ndmsg_schema = ( @@ -50,7 +48,7 @@ async def load_ndmsg(schema, sources, target, event): brmsg_schema = ( ndmsg.sql_schema() - .unique_index('ifindex', 'flags', 'NDA_DST', 'NDA_LLADDR', 'NDA_VLAN') + .unique_index('ifindex', 'NDA_LLADDR', 'NDA_DST', 'NDA_VLAN') .constraint('NDA_LLADDR', "NOT NULL DEFAULT ''") .constraint('NDA_DST', "NOT NULL DEFAULT ''") .constraint('NDA_VLAN', "NOT NULL DEFAULT 0") @@ -68,13 +66,17 @@ async def load_ndmsg(schema, sources, target, event): } -async def fallback_add(self, idx_req, req): - async for msg in await self.sources[self['target']].api(self.api, 'dump'): - await self.sources[self['target']].evq.put(msg) +def fallback_add(self, idx_req, req): + ( + self.ndb._event_queue.put( + self.sources[self['target']].api(self.api, 'dump'), + source=self['target'], + ) + ) self.load_sql() -class Neighbour(AsyncObject): +class Neighbour(RTNL_Object): table = 'neighbours' msg_class = ndmsg field_filter = NeighbourFieldFilter @@ -83,23 +85,28 @@ class Neighbour(AsyncObject): @classmethod def _count(cls, view): if view.chain: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table} WHERE f_ifindex = ?', + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s WHERE f_ifindex = %s' + % (view.table, view.ndb.schema.plch), [view.chain['index']], ) else: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table}' + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s' % view.table ) @classmethod def _dump_where(cls, view): if view.chain: + plch = view.ndb.schema.plch where = ''' - WHERE - main.f_target = ? AND - main.f_ifindex = ? - ''' + WHERE + main.f_target = %s AND + main.f_ifindex = %s + ''' % ( + plch, + plch, + ) values = [view.chain['target'], view.chain['index']] else: where = '' @@ -122,7 +129,7 @@ def summary(cls, view): ''' yield ('target', 'tflags', 'ifname', 'lladdr', 'dst') where, values = cls._dump_where(view) - for record in view.ndb.schema.fetch(req + where, values): + for record in view.ndb.task_manager.db_fetch(req + where, values): yield record def __init__(self, *argv, **kwarg): diff --git a/pyroute2/ndb/objects/netns.py b/pyroute2/ndb/objects/netns.py index 977fe6d8a..8675fd453 100644 --- a/pyroute2/ndb/objects/netns.py +++ b/pyroute2/ndb/objects/netns.py @@ -5,10 +5,10 @@ from pyroute2.netlink.rtnl.nsinfmsg import nsinfmsg from pyroute2.requests.netns import NetNSFieldFilter -from ..objects import AsyncObject +from ..objects import RTNL_Object -async def load_nsinfmsg(schema, sources, target, event): +def load_nsinfmsg(schema, target, event): # # check if there is corresponding source # @@ -20,7 +20,7 @@ async def load_nsinfmsg(schema, sources, target, event): warnings.warn( 'automatic netns sourcing is being refactored', DeprecationWarning ) - await schema.load_netlink('netns', sources, target, event) + schema.load_netlink('netns', target, event) schema = nsinfmsg.sql_schema().unique_index('NSINFO_PATH') @@ -32,7 +32,7 @@ async def load_nsinfmsg(schema, sources, target, event): } -class NetNS(AsyncObject): +class NetNS(RTNL_Object): table = 'netns' msg_class = nsinfmsg table_alias = 'n' @@ -45,17 +45,17 @@ def __init__(self, *argv, **kwarg): super(NetNS, self).__init__(*argv, **kwarg) @classmethod - def spec_normalize(cls, spec): + def spec_normalize(cls, processed, spec): if isinstance(spec, basestring): - spec = {'path': spec} - path = netns._get_netnspath(spec['path']) + processed['path'] = spec + path = netns._get_netnspath(processed['path']) # on Python3 _get_netnspath() returns bytes, not str, so # we have to decode it here in order to avoid issues with # cache keys and DB inserts if hasattr(path, 'decode'): path = path.decode('utf-8') - spec['path'] = path - return spec + processed['path'] = path + return processed def __setitem__(self, key, value): if self.state == 'system': diff --git a/pyroute2/ndb/objects/probe.py b/pyroute2/ndb/objects/probe.py deleted file mode 100644 index 811c49970..000000000 --- a/pyroute2/ndb/objects/probe.py +++ /dev/null @@ -1,82 +0,0 @@ -''' -Run a network probe -=================== - -A successful network probe neither creates real network objects -like interfaces or addresses, nor database records. The only -important thing it does -- it raises no exception. - -On the contrary, an unsuccessful network probe raises a -`NetlinkError` exception, cancelling the whole transaction. - -A network probe is always run from the corresponding netlink -target: a local system, a remote system, a network namespace, -a container. - -An example scenario: - - * target alpha, set up eth0 10.0.0.2/24 - * target beta, set up eth0 10.0.0.4/24 - * ping 10.0.0.4 (beta) from target alpha - * ping 10.0.0.2 (alpha) from target beta - -The code below sets up the addresses and checks ICMP responses. If -any step fails, the whole transaction will be rolled back automatically:: - - with NDB(log='debug') as ndb: - ndb.sources.add(kind='remote', hostname='alpha', username='root') - ndb.sources.add(kind='remote', hostname='beta', username='root') - - with ndb.begin() as trx: - trx.push( - (ndb - .interfaces[{'target': 'alpha', 'ifname': 'eth0'}] - .set(state='up') - .add_ip(address='10.0.0.2', prefixlen=24) - ), - (ndb - .interfaces[{'target': 'beta', 'ifname': 'eth0'}] - .set(state='up') - .add_ip(address='10.0.0.4', prefixlen=24) - ), - (ndb - .probes - .create(target='alpha', kind='ping', dst='10.0.0.4') - ), - (ndb - .probes - .create(target='beta', kind='ping', dst='10.0.0.2') - ), - ) -''' - -from pyroute2.netlink.rtnl.probe_msg import probe_msg - -from ..objects import AsyncObject - - -async def load_probe_msg(schema, sources, target, event): - pass - - -schema = probe_msg.sql_schema().unique_index() -init = { - 'specs': [['probes', schema]], - 'classes': [['probes', probe_msg]], - 'event_map': {probe_msg: [load_probe_msg]}, -} - - -class Probe(AsyncObject): - - table = 'probes' - msg_class = probe_msg - api = 'probe' - - def __init__(self, *argv, **kwarg): - kwarg['iclass'] = probe_msg - self.event_map = {probe_msg: 'load_probe_msg'} - super().__init__(*argv, **kwarg) - - def check(self): - return True diff --git a/pyroute2/ndb/objects/route.py b/pyroute2/ndb/objects/route.py index 5bfc20df9..a2ae3e3a5 100644 --- a/pyroute2/ndb/objects/route.py +++ b/pyroute2/ndb/objects/route.py @@ -4,14 +4,14 @@ from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True ndb = NDB() .. testsetup:: tables from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True ndb = NDB() ndb.routes.create( dst='1.1.1.1/32', gateway='127.0.0.10', oif=1, table=101 @@ -27,7 +27,7 @@ from pyroute2 import NDB from pyroute2 import config - config.mock_netlink = True + config.mock_iproute = True ndb = NDB() ndb.routes.create( dst='10.0.0.0/24', gateway='127.0.0.10' @@ -121,9 +121,9 @@ from pyroute2.requests.common import MPLSTarget from pyroute2.requests.route import RouteFieldFilter -from ..objects import AsyncObject, RTNL_Object +from ..auth_manager import check_auth +from ..objects import RTNL_Object from ..report import Record -from ..sync_api import Flags, SyncBase _dump_rt = ['main.f_%s' % x[0] for x in rtmsg.sql_schema()][:-2] _dump_nh = ['nh.f_%s' % x[0] for x in nh.sql_schema()][:-2] @@ -134,10 +134,10 @@ def get_route_id(schema, target, event): - keys = ['f_target = ?'] + keys = ['f_target = %s' % schema.plch] values = [target] for key in schema.indices['routes']: - keys.append(f'f_{key} = ?') + keys.append('f_%s = %s' % (key, schema.plch)) values.append(event.get(key) or event.get_attr(key)) # spec = 'WHERE %s' % ' AND '.join(keys) @@ -153,7 +153,7 @@ def get_route_id(schema, target, event): return str(uuid.uuid4()) -async def load_rtmsg(schema, sources, target, event): +def load_rtmsg(schema, target, event): route_id = None post = [] @@ -209,12 +209,7 @@ async def load_rtmsg(schema, sources, target, event): mp[idx]['nh_id'] = idx # add NH number post.append( partial( - schema.load_netlink, - 'nh', - sources, - target, - mp[idx], - 'routes', + schema.load_netlink, 'nh', target, mp[idx], 'routes' ) ) event['deps'] |= F_RTA_MULTIPATH @@ -230,12 +225,7 @@ async def load_rtmsg(schema, sources, target, event): encap['route_id'] = route_id post.append( partial( - schema.load_netlink, - 'enc_mpls', - sources, - target, - encap, - 'routes', + schema.load_netlink, 'enc_mpls', target, encap, 'routes' ) ) event['deps'] |= F_RTA_ENCAP @@ -252,22 +242,17 @@ async def load_rtmsg(schema, sources, target, event): metrics['route_id'] = route_id post.append( partial( - schema.load_netlink, - 'metrics', - sources, - target, - metrics, - 'routes', + schema.load_netlink, 'metrics', target, metrics, 'routes' ) ) event['deps'] |= F_RTA_METRICS # if route_id is not None: event['route_id'] = route_id - await schema.load_netlink('routes', sources, target, event) + schema.load_netlink('routes', target, event) # for procedure in post: - await procedure() + procedure() def rtmsg_gc_mark(schema, target, event, gc_mark=None): @@ -280,14 +265,17 @@ def rtmsg_gc_mark(schema, target, event, gc_mark=None): # select all routes for that OIF where f_gc_mark is not null # key_fields = ','.join(['f_%s' % x for x in schema.indices['routes']]) - key_query = ' AND '.join([f'f_{x} = ?' for x in schema.indices['routes']]) + key_query = ' AND '.join( + ['f_%s = %s' % (x, schema.plch) for x in schema.indices['routes']] + ) routes = schema.execute( - f''' - SELECT {key_fields},f_RTA_GATEWAY FROM routes WHERE - f_target = ? AND f_RTA_OIF = ? AND - f_RTA_GATEWAY IS NOT NULL {gc_clause} AND - f_family = 2 - ''', + ''' + SELECT %s,f_RTA_GATEWAY FROM routes WHERE + f_target = %s AND f_RTA_OIF = %s AND + f_RTA_GATEWAY IS NOT NULL %s AND + f_family = 2 + ''' + % (key_fields, schema.plch, schema.plch, gc_clause), (target, event.get_attr('RTA_OIF')), ).fetchmany() # @@ -309,10 +297,9 @@ def rtmsg_gc_mark(schema, target, event, gc_mark=None): if gwnet == net: ( schema.execute( - f''' - UPDATE routes SET f_gc_mark = ? - WHERE f_target = ? AND {key_query} - ''', + 'UPDATE routes SET f_gc_mark = %s ' + 'WHERE f_target = %s AND %s' + % (schema.plch, schema.plch, key_query), (gc_mark, target) + route[:-1], ) ) @@ -416,7 +403,7 @@ def __repr__(self): return repr(dict(self)) -class Route(AsyncObject): +class Route(RTNL_Object): table = 'routes' msg_class = rtmsg hidden_fields = ['route_id'] @@ -428,23 +415,28 @@ class Route(AsyncObject): @classmethod def _count(cls, view): if view.chain: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table} WHERE f_RTA_OIF = ?', + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s WHERE f_RTA_OIF = %s' + % (view.table, view.ndb.schema.plch), [view.chain['index']], ) else: - return view.ndb.schema.fetchone( - f'SELECT count(*) FROM {view.table}' + return view.ndb.task_manager.db_fetchone( + 'SELECT count(*) FROM %s' % view.table ) @classmethod def _dump_where(cls, view): if view.chain: + plch = view.ndb.schema.plch where = ''' WHERE - main.f_target = ? AND - main.f_RTA_OIF = ? - ''' + main.f_target = %s AND + main.f_RTA_OIF = %s + ''' % ( + plch, + plch, + ) values = [view.chain['target'], view.chain['index']] else: where = '' @@ -493,7 +485,7 @@ def summary(cls, view): 'gateway', ) where, values = cls._dump_where(view) - for record in view.ndb.schema.fetch(req + where, values): + for record in view.ndb.task_manager.db_fetch(req + where, values): yield record @classmethod @@ -514,16 +506,20 @@ def dump(cls, view): + ['metrics', 'encap'] ) yield header + plch = view.ndb.schema.plch where, values = cls._dump_where(view) - for record in view.ndb.schema.fetch(req + where, values): + for record in view.ndb.task_manager.db_fetch(req + where, values): route_id = record[-1] record = list(record[:-1]) if route_id is not None: # # fetch metrics metrics = tuple( - view.ndb.schema.fetch( - 'SELECT * FROM metrics WHERE f_route_id = ?', + view.ndb.task_manager.db_fetch( + ''' + SELECT * FROM metrics WHERE f_route_id = %s + ''' + % (plch,), (route_id,), ) ) @@ -543,8 +539,11 @@ def dump(cls, view): # # fetch encap enc_mpls = tuple( - view.ndb.schema.fetch( - 'SELECT * FROM enc_mpls WHERE f_route_id = ?', + view.ndb.task_manager.db_fetch( + ''' + SELECT * FROM enc_mpls WHERE f_route_id = %s + ''' + % (plch,), (route_id,), ) ) @@ -557,10 +556,10 @@ def dump(cls, view): yield record @classmethod - def spec_normalize(cls, spec): + def spec_normalize(cls, processed, spec): if isinstance(spec, basestring): - return {'dst': spec} - return spec + processed['dst'] = spec + return processed @classmethod def compare_record(self, left, right): @@ -586,14 +585,16 @@ def _cmp_encap(self, right): } def mark_tflags(self, mark): + plch = (self.schema.plch,) * 4 self.schema.execute( ''' - UPDATE interfaces SET - f_tflags = ? - WHERE - (f_index = ? OR f_index = ?) - AND f_target = ? - ''', + UPDATE interfaces SET + f_tflags = %s + WHERE + (f_index = %s OR f_index = %s) + AND f_target = %s + ''' + % plch, (mark, self['iif'], self['oif'], self['target']), ) @@ -680,6 +681,7 @@ def make_req(self, prime): req['gateway'] = self['gateway'] return req + @check_auth('obj:modify') def __setitem__(self, key, value): if key == 'route_id': raise ValueError('route_id is read only') @@ -689,7 +691,9 @@ def __setitem__(self, key, value): mp = dict(mp) if self.state == 'invalid': mp['create'] = True - obj = NextHop(self, self.view, mp) + obj = NextHop( + self, self.view, mp, auth_managers=self.auth_managers + ) obj.state.set(self.state.get()) self['multipath'].append(obj) if key in self.changed: @@ -698,7 +702,9 @@ def __setitem__(self, key, value): value = dict(value) if not isinstance(self['metrics'], Metrics): value['create'] = True - obj = Metrics(self, self.view, value) + obj = Metrics( + self, self.view, value, auth_managers=self.auth_managers + ) obj.state.set(self.state.get()) super(Route, self).__setitem__('metrics', obj) if key in self.changed: @@ -721,6 +727,7 @@ def __setitem__(self, key, value): else: super(Route, self).__setitem__(key, value) + @check_auth('obj:modify') def apply(self, rollback=False, req_filter=None, mode='apply'): if ( (self.get('table') == 255) @@ -751,8 +758,9 @@ def load_sql(self, *argv, **kwarg): if self['deps'] & F_RTA_ENCAP: for _ in range(5): enc = tuple( - self.schema.fetch( - 'SELECT * FROM enc_mpls WHERE f_route_id = ?', + self.task_manager.db_fetch( + 'SELECT * FROM enc_mpls WHERE f_route_id = %s' + % (self.schema.plch,), (self['route_id'],), ) ) @@ -772,14 +780,18 @@ def load_sql(self, *argv, **kwarg): if self['deps'] & F_RTA_METRICS: for _ in range(5): metrics = tuple( - self.schema.fetch( - 'SELECT * FROM metrics WHERE f_route_id = ?', + self.task_manager.db_fetch( + 'SELECT * FROM metrics WHERE f_route_id = %s' + % (self.schema.plch,), (self['route_id'],), ) ) if metrics: self['metrics'] = Metrics( - self, self.view, {'route_id': self['route_id']} + self, + self.view, + {'route_id': self['route_id']}, + auth_managers=self.auth_managers, ) break time.sleep(0.1) @@ -792,13 +804,9 @@ def load_sql(self, *argv, **kwarg): # # FIXME: use self['deps'] if 'nh_id' not in self and self.get('route_id') is not None: - nhs = iter( - tuple( - self.schema.fetch( - 'SELECT * FROM nh WHERE f_route_id = ?', - (self['route_id'],), - ) - ) + nhs = self.task_manager.db_fetch( + 'SELECT * FROM nh WHERE f_route_id = %s' % (self.schema.plch,), + (self['route_id'],), ) flush = False @@ -824,23 +832,51 @@ def load_sql(self, *argv, **kwarg): for nexthop in nhs: key = {'route_id': self['route_id'], 'nh_id': nexthop[-1]} - (self['multipath'].append(NextHop(self, self.view, key))) + ( + self['multipath'].append( + NextHop( + self, + self.view, + key, + auth_managers=self.auth_managers, + ) + ) + ) + + +class RouteSub: + def apply(self, rollback=False, req_filter=None, mode='apply'): + return self.route.apply(rollback, req_filter, mode) + def commit(self): + return self.route.commit() + + def set(self, key, value): + self[key] = value + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.commit() -class NextHop(AsyncObject): + +class NextHop(RouteSub, RTNL_Object): msg_class = nh table = 'nh' hidden_fields = ('route_id', 'target') def mark_tflags(self, mark): + plch = (self.schema.plch,) * 4 self.schema.execute( ''' - UPDATE interfaces SET - f_tflags = ? - WHERE - (f_index = ? OR f_index = ?) - AND f_target = ? - ''', + UPDATE interfaces SET + f_tflags = %s + WHERE + (f_index = %s OR f_index = %s) + AND f_target = %s + ''' + % plch, (mark, self.route['iif'], self.route['oif'], self.route['target']), ) @@ -851,7 +887,7 @@ def __init__(self, route, *argv, **kwarg): super(NextHop, self).__init__(*argv, **kwarg) -class MetricsStub(dict): +class MetricsStub(RouteSub, dict): def __init__(self, route): self.route = route @@ -864,20 +900,22 @@ def __getitem__(self, key): raise KeyError('metrics not initialized for this route') -class Metrics(AsyncObject): +class Metrics(RouteSub, RTNL_Object): msg_class = rtmsg.metrics table = 'metrics' hidden_fields = ('route_id', 'target') def mark_tflags(self, mark): + plch = (self.schema.plch,) * 4 self.schema.execute( ''' - UPDATE interfaces SET - f_tflags = ? - WHERE - (f_index = ? OR f_index = ?) - AND f_target = ? - ''', + UPDATE interfaces SET + f_tflags = %s + WHERE + (f_index = %s OR f_index = %s) + AND f_target = %s + ''' + % plch, (mark, self.route['iif'], self.route['oif'], self.route['target']), ) @@ -886,61 +924,3 @@ def __init__(self, route, *argv, **kwarg): kwarg['iclass'] = rtmsg.metrics kwarg['check'] = False super(Metrics, self).__init__(*argv, **kwarg) - - -class RouteSub(SyncBase): - def apply(self, rollback=False, req_filter=None, mode='apply'): - return self.route.apply(rollback, req_filter, mode) - - def commit(self): - return self.route.commit() - - def set(self, key, value): - self[key] = value - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.commit() - - def __getitem__(self, key): - return self._main_sync_call(self.asyncore.__getitem__, key) - - def __setitem__(self, key, value): - return self._main_sync_call(self.asyncore.__setitem__, key, value) - - -class SyncRoute(RTNL_Object): - - def __init__(self, event_loop, asyncore, class_map=None, flags=Flags.RO): - super().__init__(event_loop, asyncore, class_map, flags) - - @property - def metrics(self): - metrics = self._main_sync_call(self.asyncore.__getitem__, 'metrics') - ret = RouteSub(self.event_loop, metrics, self.class_map, self.flags) - ret.route = self - return ret - - @property - def multipath(self): - mp = self._main_sync_call(self.asyncore.__getitem__, 'multipath') - ret = [] - for nexthop in mp: - rs = RouteSub(self.event_loop, nexthop, self.class_map, self.flags) - rs.route = self - ret.append(rs) - return ret - - def __getitem__(self, key): - if key == 'metrics': - return self.metrics - if key == 'multipath': - return self.multipath - return super().__getitem__(key) - - def get(self, key, *argv): - if key in ('metrics', 'multipath'): - return self[key] - return super().get(key, *argv) diff --git a/pyroute2/ndb/objects/rule.py b/pyroute2/ndb/objects/rule.py index c29388078..84caf4f98 100644 --- a/pyroute2/ndb/objects/rule.py +++ b/pyroute2/ndb/objects/rule.py @@ -3,7 +3,7 @@ from pyroute2.common import get_address_family from pyroute2.netlink.rtnl.fibmsg import fibmsg -from ..objects import AsyncObject +from ..objects import RTNL_Object schema = fibmsg.sql_schema().unique_index( 'family', @@ -40,7 +40,7 @@ } -class Rule(AsyncObject): +class Rule(RTNL_Object): table = 'rules' msg_class = fibmsg api = 'rule' @@ -56,7 +56,7 @@ def summary(cls, view): rules ''' yield ('target', 'tflags', 'family', 'priority', 'action', 'table') - for record in view.ndb.schema.fetch(req): + for record in view.ndb.task_manager.db_fetch(req): yield record def __init__(self, *argv, **kwarg): diff --git a/pyroute2/ndb/query.py b/pyroute2/ndb/query.py new file mode 100644 index 000000000..9f92ae2fe --- /dev/null +++ b/pyroute2/ndb/query.py @@ -0,0 +1,170 @@ +from .report import RecordSet + + +class Query(object): + def __init__(self, schema, fmt='raw'): + self._schema = schema + self._fmt = fmt + + def _formatter(self, cursor, fmt=None, header=None, transform=None): + fmt = fmt or self._fmt + + if fmt == 'csv': + if header: + yield ','.join(header) + for record in cursor: + if transform: + record = transform(record) + if isinstance(record, (list, tuple)): + yield ','.join([str(x) for x in record]) + else: + yield record + elif fmt == 'raw': + if header: + yield header + for record in cursor: + if transform: + record = transform(record) + yield record + else: + raise TypeError('format not supported') + + def nodes(self, fmt=None): + ''' + List all the nodes within the cluster. + ''' + header = ('nodename',) + return RecordSet( + self._formatter( + self._schema.fetch( + ''' + SELECT DISTINCT f_target + FROM interfaces + ''' + ), + fmt, + header, + ) + ) + + def p2p_edges(self, fmt=None): + ''' + List point to point edges within the cluster, like + GRE or PPP interfaces. + ''' + header = ('left_node', 'right_node') + return RecordSet( + self._formatter( + self._schema.fetch( + ''' + SELECT DISTINCT + l.f_target, r.f_target + FROM p2p AS l + INNER JOIN p2p AS r + ON + l.f_p2p_local = r.f_p2p_remote + AND l.f_target != r.f_target + ''' + ), + fmt, + header, + ) + ) + + def l2_edges(self, fmt=None): + ''' + List l2 links within the cluster, reconstructed + from the ARP caches on the nodes. Works as follows: + + 1. for every node take the ARP cache + 2. for every record in the cache reconstruct two triplets: + + * the interface index -> the local interface name + * the neighbour lladdr -> the remote node and interface name + + Issues: does not filter out fake lladdr, so CARP interfaces + produce fake l2 edges within the cluster. + ''' + header = ( + 'left_node', + 'left_ifname', + 'left_lladdr', + 'right_node', + 'right_ifname', + 'right_lladdr', + ) + return RecordSet( + self._formatter( + self._schema.fetch( + ''' + SELECT DISTINCT + j.f_target, j.f_IFLA_IFNAME, j.f_IFLA_ADDRESS, + d.f_target, d.f_IFLA_IFNAME, j.f_NDA_LLADDR + FROM + (SELECT + n.f_target, i.f_IFLA_IFNAME, + i.f_IFLA_ADDRESS, n.f_NDA_LLADDR + FROM + neighbours AS n + INNER JOIN + interfaces AS i + ON + n.f_target = i.f_target + AND i.f_IFLA_ADDRESS != '00:00:00:00:00:00' + AND n.f_ifindex = i.f_index) AS j + INNER JOIN + interfaces AS d + ON + j.f_NDA_LLADDR = d.f_IFLA_ADDRESS + AND j.f_target != d.f_target + ''' + ), + fmt, + header, + ) + ) + + def l3_edges(self, fmt=None): + ''' + List l3 edges. For every address on every node look + if it is used as a gateway on remote nodes. Such cases + are reported as l3 edges. + + Issues: does not report routes (edges) via point to point + connections like GRE where local addresses are used as + gateways. To be fixed. + ''' + header = ( + 'source_node', + 'gateway_node', + 'gateway_address', + 'dst', + 'dst_len', + ) + return RecordSet( + self._formatter( + self._schema.fetch( + ''' + SELECT DISTINCT + r.f_target, a.f_target, a.f_IFA_ADDRESS, + r.f_RTA_DST, r.f_dst_len + FROM + addresses AS a + INNER JOIN + routes AS r + ON + r.f_target != a.f_target + AND r.f_RTA_GATEWAY = a.f_IFA_ADDRESS + AND r.f_RTA_GATEWAY NOT IN + (SELECT + f_IFA_ADDRESS + FROM + addresses + WHERE + f_target = r.f_target) + ''' + ), + fmt, + header, + ) + ) diff --git a/pyroute2/ndb/report.py b/pyroute2/ndb/report.py index 8445f76f3..4e9d70792 100644 --- a/pyroute2/ndb/report.py +++ b/pyroute2/ndb/report.py @@ -3,13 +3,10 @@ .. testsetup:: - from pyroute2 import config from pyroute2 import NDB + ndb = NDB(sources=[{'target': 'localhost', 'kind': 'IPMock'}]) - config.mock_netlink = True - ndb = NDB() - -.. testcleanup:: +.. testcleanup:: * for key, value in tuple(globals().items()): if key.startswith('ndb') and hasattr(value, 'close'): @@ -36,7 +33,10 @@ ''' import json -import sys +import warnings +from itertools import chain + +from pyroute2 import cli MAX_REPORT_LINES = 10000 @@ -178,18 +178,11 @@ def __eq__(self, right): class BaseRecordSet(object): - def __init__(self, source, ellipsis='(...)'): - self.source = source - self.generator = source + def __init__(self, generator, ellipsis='(...)'): + self.generator = generator self.ellipsis = ellipsis - self.materialized = None - self.filters = [] - if hasattr(sys, 'ps1'): - self.materialize() def __iter__(self): - if self.materialized is not None: - self.generator = iter(self.materialized) return self def __next__(self): @@ -218,8 +211,19 @@ def __repr__(self): ret.pop() return ''.join(ret) - def materialize(self): - self.materialized = tuple(self) + +class RecordSetConfig(dict): + def __init__(self, prime): + if isinstance(prime, dict): + for key, value in prime.items(): + self[key] = value + else: + raise ValueError('only dict allowed') + + def __setitem__(self, key, value): + if isinstance(value, str): + value = json.loads(value) + return super().__setitem__(key, value) class RecordSet(BaseRecordSet): @@ -232,6 +236,11 @@ class RecordSet(BaseRecordSet): to make chains of filters. ''' + def __init__(self, generator, config=None, ellipsis=True): + super().__init__(generator, ellipsis) + self.filters = [] + self.config = RecordSetConfig(config) if config is not None else {} + def __next__(self): while True: record = next(self.generator) @@ -242,6 +251,7 @@ def __next__(self): else: return record + @cli.show_result def select_fields(self, *fields): ''' Select only chosen fields for every record: @@ -260,8 +270,10 @@ def select_fields(self, *fields): 2,'eth0' ''' self.filters.append(lambda x: x._select_fields(*fields)) - return self + if self.config.get('recordset_pipe'): + return RecordSet(self, config=self.config) + @cli.show_result def select_records(self, f=None, **spec): ''' Select records based on a function f() or a spec match. A spec @@ -280,8 +292,10 @@ def select_records(self, f=None, **spec): 'localhost',0,'eth0','192.168.122.28',24 ''' self.filters.append(lambda x: x if x._match(f, **spec) else None) - return self + if self.config.get('recordset_pipe'): + return RecordSet(self, config=self.config) + @cli.show_result def transform_fields(self, **kwarg): ''' Transform fields with a function. Function must accept @@ -304,8 +318,83 @@ def transform_fields(self, **kwarg): 'eth0','192.168.122.28/24' ''' self.filters.append(lambda x: x._transform_fields(**kwarg)) - return self - + if self.config.get('recordset_pipe'): + return RecordSet(self, config=self.config) + + @cli.show_result + def transform(self, **kwarg): + warnings.warn(deprecation_notice, DeprecationWarning) + + def g(): + for record in self.generator: + if isinstance(record, Record): + values = [] + names = record._names + for name, value in zip(names, record._values): + if name in kwarg: + value = kwarg[name](value) + values.append(value) + record = Record(names, values, record._ref_class) + yield record + + return RecordSet(g()) + + @cli.show_result + def filter(self, f=None, **kwarg): + warnings.warn(deprecation_notice, DeprecationWarning) + + def g(): + for record in self.generator: + m = True + for key in kwarg: + if kwarg[key] != getattr(record, key): + m = False + if m: + if f is None: + yield record + elif f(record): + yield record + + return RecordSet(g()) + + @cli.show_result + def select(self, *argv): + warnings.warn(deprecation_notice, DeprecationWarning) + return self.fields(*argv) + + @cli.show_result + def fields(self, *fields): + warnings.warn(deprecation_notice, DeprecationWarning) + + def g(): + for record in self.generator: + yield record._select_fields(*fields) + + return RecordSet(g()) + + @cli.show_result + def join(self, right, condition=lambda r1, r2: True, prefix=''): + warnings.warn(deprecation_notice, DeprecationWarning) + # fetch all the records from the right + # ACHTUNG it may consume a lot of memory + right = tuple(right) + + def g(): + for r1 in self.generator: + for r2 in right: + if condition(r1, r2): + n = tuple( + chain( + r1._names, + ['%s%s' % (prefix, x) for x in r2._names], + ) + ) + v = tuple(chain(r1._values, r2._values)) + yield Record(n, v, r1._ref_class) + + return RecordSet(g()) + + @cli.show_result def format(self, kind): ''' Return an iterator over text lines in the chosen format. diff --git a/pyroute2/ndb/schema.py b/pyroute2/ndb/schema.py index 8c9914420..97bf83ec6 100644 --- a/pyroute2/ndb/schema.py +++ b/pyroute2/ndb/schema.py @@ -4,12 +4,35 @@ NDB stores all the records in an SQL database. By default it uses the SQLite3 module, which is a part of the Python stdlib, so no -extra packages are required. +extra packages are required:: + + # SQLite3 -- simple in-memory DB + ndb = NDB() + + # SQLite3 -- same as above with explicit arguments + ndb = NDB(db_provider='sqlite3', db_spec=':memory:') + + # SQLite3 -- file DB + ndb = NDB(db_provider='sqlite3', db_spec='test.db') + +It is also possible to use a PostgreSQL database via psycopg2 +module:: + + # PostgreSQL -- local DB + ndb = NDB(db_provider='psycopg2', + db_spec={'dbname': 'test'}) + + # PostgreSQL -- remote DB + ndb = NDB(db_provider='psycopg2', + db_spec={'dbname': 'test', + 'host': 'db1.example.com'}) Database backup --------------- -Built-in database backup is implemented now only for SQLite3 backend:: +Built-in database backup is implemented now only for SQLite3 backend. +For the PostgresSQL backend you have to use external utilities like +`pg_dump`:: # create an NDB instance ndb = NDB() # the defaults: db_provider='sqlite3', db_spec=':memory:' @@ -94,6 +117,7 @@ on startup. ''' +import enum import json import random import sqlite3 @@ -105,19 +129,31 @@ from pyroute2 import config from pyroute2.common import basestring, uuid32 -from pyroute2.netlink import NLM_F_REPLACE # -from .objects import address, interface, neighbour, netns, probe, route, rule +from .objects import address, interface, neighbour, netns, route, rule + +try: + import psycopg2 +except ImportError: + psycopg2 = None # # the order is important # -plugins = [interface, address, neighbour, route, netns, rule, probe] +plugins = [interface, address, neighbour, route, netns, rule] MAX_ATTEMPTS = 5 +class DBProvider(enum.Enum): + sqlite3 = 'sqlite3' + psycopg2 = 'psycopg2' + + def __eq__(self, r): + return str(self) == r + + def publish(f): if isinstance(f, str): @@ -136,9 +172,14 @@ def __init__(self, schema, table): self.schema = schema self.table = table + @publish('get') def __getitem__(self, key): for (record,) in self.schema.fetch( - f'SELECT f_value FROM {self.table} WHERE f_key = ?', (key,) + f''' + SELECT f_value FROM {self.table} + WHERE f_key = {self.schema.plch} + ''', + (key,), ): return json.loads(record) raise KeyError(f'key {key} not found') @@ -147,13 +188,21 @@ def __getitem__(self, key): def __setitem__(self, key, value): del self[key] self.schema.execute( - f'INSERT INTO {self.table} VALUES (?, ?)', (key, json.dumps(value)) + f''' + INSERT INTO {self.table} + VALUES ({self.schema.plch}, {self.schema.plch}) + ''', + (key, json.dumps(value)), ) @publish('del') def __delitem__(self, key): self.schema.execute( - f'DELETE FROM {self.table} WHERE f_key = ?', (key,) + f''' + DELETE FROM {self.table} + WHERE f_key = {self.schema.plch} + ''', + (key,), ) @publish @@ -190,11 +239,13 @@ class DBSchema: indices = {} foreign_keys = {} - def __init__(self, config, event_map, log_channel): + def __init__(self, config, sources, event_map, log_channel): global plugins + self.sources = sources self.config = DBDict(self, 'config') self.stats = {} self.connection = None + self.cursor = None self.log = log_channel self.snapshots = {} self.key_defaults = {} @@ -204,7 +255,7 @@ def __init__(self, config, event_map, log_channel): # means also that these variables can not be changed # in runtime self.rtnl_log = config['rtnl_debug'] - self.provider = None + self.provider = config['provider'] # for plugin in plugins: # @@ -240,8 +291,16 @@ def __init__(self, config, event_map, log_channel): def initdb(self, config): if self.connection is not None: self.close() - self.connection = sqlite3.connect(config['spec']) - self.connection.execute('PRAGMA foreign_keys = ON') + if config['provider'] == DBProvider.sqlite3: + self.connection = sqlite3.connect(config['spec']) + self.plch = '?' + self.connection.execute('PRAGMA foreign_keys = ON') + elif config['provider'] == DBProvider.psycopg2: + self.connection = psycopg2.connect(**config['spec']) + self.plch = '%s' + else: + raise TypeError('DB provider not supported') + self.cursor = self.connection.cursor() # # compile request lines # @@ -305,11 +364,11 @@ def merge_spec(self, table1, table2, table, schema_idx): all_names = spec1['all_names'] + spec2['all_names'][2:-1] norm_names = spec1['norm_names'] + spec2['norm_names'][2:-1] idx = ('target', 'tflags') + schema_idx - f_names = [f'f_{x}' for x in all_names] - f_set = [f'f_{x} = ?' for x in all_names] - f_idx = [f'f_{x}' for x in idx] - f_idx_match = [f'{table}.{x} = ?' for x in f_idx] - plchs = ['?'] * len(f_names) + f_names = ['f_%s' % x for x in all_names] + f_set = ['f_%s = %s' % (x, self.plch) for x in all_names] + f_idx = ['f_%s' % x for x in idx] + f_idx_match = ['%s.%s = %s' % (table2, x, self.plch) for x in f_idx] + plchs = [self.plch] * len(f_names) return { 'names': names, 'all_names': all_names, @@ -354,17 +413,22 @@ def compile_spec(self, table, schema_names, schema_idx): # and we can not use them; neither can we change the # C structure # - f_names = [f'f_{x}' for x in all_names] + f_names = ['f_%s' % x for x in all_names] # # set the fields # # e.g.: f_flags = ?, f_IFLA_IFNAME = ? # - f_set = [f'f_{x} = ?' for x in all_names] + # there are different placeholders: + # ? -- SQLite3 + # %s -- PostgreSQL + # so use self.plch here + # + f_set = ['f_%s = %s' % (x, self.plch) for x in all_names] # # the set of the placeholders to use in the INSERT statements # - plchs = ['?'] * len(f_names) + plchs = [self.plch] * len(f_names) # # the index schema; use target and tflags in every index # @@ -372,7 +436,7 @@ def compile_spec(self, table, schema_names, schema_idx): # # the same, escaped: f_target, f_tflags etc. # - f_idx = [f'f_{x}' for x in idx] + f_idx = ['f_%s' % x for x in idx] # # normalized idx names # @@ -384,7 +448,7 @@ def compile_spec(self, table, schema_names, schema_idx): # # the same issue with the placeholders # - f_idx_match = [f'{table}.{x} = ?' for x in f_idx] + f_idx_match = ['%s.%s = %s' % (table, x, self.plch) for x in f_idx] return { 'names': names, @@ -409,23 +473,26 @@ def add_nl_source(self, target, kind, spec): self.execute( ''' DELETE FROM sources_options - WHERE f_target = ? - ''', + WHERE f_target = %s + ''' + % self.plch, (target,), ) self.execute( ''' DELETE FROM sources - WHERE f_target = ? - ''', + WHERE f_target = %s + ''' + % self.plch, (target,), ) # add self.execute( ''' INSERT INTO sources (f_target, f_kind) - VALUES (?, ?) - ''', + VALUES (%s, %s) + ''' + % (self.plch, self.plch), (target, kind), ) for key, value in spec.items(): @@ -434,23 +501,23 @@ def add_nl_source(self, target, kind, spec): ''' INSERT INTO sources_options (f_target, f_name, f_type, f_value) - VALUES (?, ?, ?, ?) - ''', + VALUES (%s, %s, %s, %s) + ''' + % (self.plch, self.plch, self.plch, self.plch), (target, key, vtype, value), ) def execute(self, *argv, **kwarg): - cursor = self.connection.cursor() try: # # FIXME: add logging # for _ in range(MAX_ATTEMPTS): try: - cursor.execute(*argv, **kwarg) + self.cursor.execute(*argv, **kwarg) break except (sqlite3.InterfaceError, sqlite3.OperationalError) as e: - self.log.debug(f'{e}') + self.log.debug('%s' % e) # # Retry on: # -- InterfaceError: Error binding parameter ... @@ -458,32 +525,39 @@ def execute(self, *argv, **kwarg): # pass else: - raise Exception('DB execute error: {argv} {kwarg}') + raise Exception('DB execute error: %s %s' % (argv, kwarg)) except Exception: raise finally: self.connection.commit() # no performance optimisation yet - return cursor + return self.cursor + @publish def fetchone(self, *argv, **kwarg): for row in self.fetch(*argv, **kwarg): return row return None + @publish def fetch(self, *argv, **kwarg): - cursor = self.execute(*argv, **kwarg) + self.execute(*argv, **kwarg) while True: - row_set = cursor.fetchmany() + row_set = self.cursor.fetchmany() if not row_set: return for row in row_set: yield row + @publish def backup(self, spec): - backup_connection = sqlite3.connect(spec) - self.connection.backup(backup_connection) - backup_connection.close() + if sys.version_info >= (3, 7) and self.provider == DBProvider.sqlite3: + backup_connection = sqlite3.connect(spec) + self.connection.backup(backup_connection) + backup_connection.close() + else: + raise NotImplementedError() + @publish def export(self, f='stdout'): close = False if f in ('stdout', 'stderr'): @@ -493,13 +567,13 @@ def export(self, f='stdout'): close = True try: for table in self.spec.keys(): - f.write(f'\ntable {table}\n') - for record in self.execute(f'SELECT * FROM {table}'): + f.write('\ntable %s\n' % table) + for record in self.execute('SELECT * FROM %s' % table): f.write(' '.join([str(x) for x in record])) f.write('\n') if self.rtnl_log: - f.write(f'\ntable {table}_log\n') - for record in self.execute(f'SELECT * FROM {table}_log'): + f.write('\ntable %s_log\n' % table) + for record in self.execute('SELECT * FROM %s_log' % table): f.write(' '.join([str(x) for x in record])) f.write('\n') finally: @@ -604,14 +678,24 @@ def create_table(self, table): def mark(self, target, mark): for table in self.spec: self.execute( - 'UPDATE ? SET f_tflags = ? WHERE f_target = ?', - (table, mark, target), + ''' + UPDATE %s SET f_tflags = %s + WHERE f_target = %s + ''' + % (table, self.plch, self.plch), + (mark, target), ) @publish def flush(self, target): for table in self.spec: - self.execute(f'DELETE FROM {table} WHERE f_target = ?', (target,)) + self.execute( + ''' + DELETE FROM %s WHERE f_target = %s + ''' + % (table, self.plch), + (target,), + ) @publish def save_deps(self, ctxid, weak_ref, iclass): @@ -622,7 +706,7 @@ def save_deps(self, ctxid, weak_ref, iclass): conditions = [] values = [] for key in idx: - conditions.append(f'f_{key} = ?') + conditions.append('f_%s = %s' % (key, self.plch)) if key in obj_k: values.append(obj_k[key]) else: @@ -653,22 +737,26 @@ def save_deps(self, ctxid, weak_ref, iclass): # create the snapshot table # self.execute( - f''' - CREATE TABLE IF NOT EXISTS {table}_{ctxid} - AS SELECT * FROM {table} - WHERE f_tflags IS NULL ''' + CREATE TABLE IF NOT EXISTS %s_%s + AS SELECT * FROM %s + WHERE + f_tflags IS NULL + ''' + % (table, ctxid, table) ) # # copy the data -- is it possible to do it in one step? # self.execute( - f''' - INSERT INTO {table}_{ctxid} - SELECT * FROM {table} - WHERE f_tflags = ? - ''', - (uuid,), + ''' + INSERT INTO %s_%s + SELECT * FROM %s + WHERE + f_tflags = %s + ''' + % (table, ctxid, table, self.plch), + [uuid], ) # # unmark all the data @@ -676,15 +764,24 @@ def save_deps(self, ctxid, weak_ref, iclass): obj.mark_tflags(tflags) for table in self.spec: - self.execute(f'UPDATE {table}_{ctxid} SET f_tflags = ?', (tflags,)) - self.snapshots[f'{table}_{ctxid}'] = weak_ref + self.execute( + ''' + UPDATE %s_%s SET f_tflags = %s + ''' + % (table, ctxid, self.plch), + [tflags], + ) + self.snapshots['%s_%s' % (table, ctxid)] = weak_ref @publish def purge_snapshots(self): for table in tuple(self.snapshots): for _ in range(MAX_ATTEMPTS): try: - self.execute(f'DROP TABLE {table}') + if self.provider == DBProvider.sqlite3: + self.execute('DROP TABLE %s' % table) + elif self.provider == DBProvider.psycopg2: + self.execute('DROP TABLE %s CASCADE' % table) self.connection.commit() del self.snapshots[table] break @@ -713,7 +810,7 @@ def get(self, table, spec): key = cls.name2nla(key) if key not in cspec['all_names']: raise KeyError('field name not found') - conditions.append(f'f_{key} = ?') + conditions.append('f_%s = %s' % (key, self.plch)) values.append(value) req = 'SELECT * FROM %s WHERE %s' % (table, ' AND '.join(conditions)) for record in self.fetch(req, values): @@ -727,7 +824,7 @@ def log_netlink(self, table, target, event, ctable=None): fields = ','.join( ['f_tstamp', 'f_target', 'f_event'] + ['f_%s' % x for x in fkeys] ) - pch = ','.join(['?'] * (len(fkeys) + 3)) + pch = ','.join([self.plch] * (len(fkeys) + 3)) values = [ int(time.time() * 1000), target, @@ -745,9 +842,7 @@ def log_netlink(self, table, target, event, ctable=None): values, ) - async def load_netlink( - self, table, sources, target, event, ctable=None, propagate=False - ): + def load_netlink(self, table, target, event, ctable=None, propagate=False): # if self.rtnl_log: self.log_netlink(table, target, event, ctable) @@ -775,7 +870,7 @@ async def load_netlink( # clean marked routes self.execute( - 'DELETE FROM routes WHERE ' '(f_gc_mark + 5) < ?', + 'DELETE FROM routes WHERE ' '(f_gc_mark + 5) < %s' % self.plch, (int(time.time()),), ) # @@ -785,10 +880,10 @@ async def load_netlink( # # Delete an object # - conditions = ['f_target = ?'] + conditions = ['f_target = %s' % self.plch] values = [target] for key in self.indices[table]: - conditions.append(f'f_{key} = ?') + conditions.append('f_%s = %s' % (key, self.plch)) value = event.get(key) or event.get_attr(key) if value is None: value = self.key_defaults[table][key] @@ -811,34 +906,6 @@ async def load_netlink( compiled = self.compiled[table] # a map of sub-NLAs nodes = {} - # replace - r_conditions = [] - r_values = [] - - # Check route replace - if ( - event['header'].get('flags', 0) == NLM_F_REPLACE - and event['event'] == 'RTM_NEWROUTE' - ): - # Replace existing route - r_conditions = [table + '.f_target = ?'] - r_values = [target] - for key in self.indices[table]: - if key not in [ - 'RTA_DST', - 'dst_len', - 'table', - 'RTA_PRIORITY', - ]: - continue - - r_conditions.append(table + f'.f_{key} = ?') - value = event.get(key) or event.get_attr(key) - if value is None: - value = self.key_defaults[table][key] - if isinstance(value, (dict, list, tuple, set)): - value = json.dumps(value) - r_values.append(value) # fetch values (exc. the first two columns) for fname, ftype in self.spec[table].items(): @@ -875,41 +942,62 @@ async def load_netlink( values.append(value) try: - w_fidx = compiled['fidx'] - w_ivalues = ivalues - if r_conditions: - w_fidx = ' AND '.join(r_conditions) - w_ivalues = r_values - - # - # SQLite3 >= 3.24 actually has UPSERT, but ... - # - # We can not use here INSERT OR REPLACE as well, since - # it drops (almost always) records with foreign key - # dependencies. Maybe a bug in SQLite3, who knows. - # - count = ( - self.execute( - f'SELECT count(*) FROM {table} WHERE {w_fidx}', - w_ivalues, - ).fetchone() - )[0] - if count == 0: - self.execute( - f''' - INSERT INTO {table} ({compiled["fnames"]}) - VALUES ({compiled["plchs"]}) - ''', - values, + if self.provider == DBProvider.psycopg2: + # + # run UPSERT -- the DB provider must support it + # + ( + self.execute( + 'INSERT INTO %s (%s) VALUES (%s) ' + 'ON CONFLICT (%s) ' + 'DO UPDATE SET %s WHERE %s' + % ( + table, + compiled['fnames'], + compiled['plchs'], + compiled['knames'], + compiled['fset'], + compiled['fidx'], + ), + (values + values + ivalues), + ) ) + # + elif self.provider == DBProvider.sqlite3: + # + # SQLite3 >= 3.24 actually has UPSERT, but ... + # + # We can not use here INSERT OR REPLACE as well, since + # it drops (almost always) records with foreign key + # dependencies. Maybe a bug in SQLite3, who knows. + # + count = ( + self.execute( + ''' + SELECT count(*) FROM %s WHERE %s + ''' + % (table, compiled['fidx']), + ivalues, + ).fetchone() + )[0] + if count == 0: + self.execute( + ''' + INSERT INTO %s (%s) VALUES (%s) + ''' + % (table, compiled['fnames'], compiled['plchs']), + values, + ) + else: + self.execute( + ''' + UPDATE %s SET %s WHERE %s + ''' + % (table, compiled['fset'], compiled['fidx']), + (values + ivalues), + ) else: - self.execute( - f''' - UPDATE {table} SET {compiled["fset"]} - WHERE {w_fidx} - ''', - (values + w_ivalues), - ) + raise NotImplementedError() # except Exception as e: # diff --git a/pyroute2/ndb/source.py b/pyroute2/ndb/source.py index 251f0c451..f6143db4b 100644 --- a/pyroute2/ndb/source.py +++ b/pyroute2/ndb/source.py @@ -57,6 +57,27 @@ 'netns': 'test01'} # See also: :ref:`netns` + +Remote systems +-------------- + +It is possible also to connect to remote systems using SSH. In order to +use this kind of sources it is required to install the +`mitogen `_ module. The `remote` kind +of sources uses the `RemoteIPRoute` class. The short form:: + + ndb.sources.add(hostname='worker1.example.com') + + +In some more extended form:: + + ndb.sources.add(**{'target': 'worker1.example.com', + 'kind': 'remote', + 'hostname': 'worker1.example.com', + 'username': 'jenkins', + 'check_host_keys': False}) + +See also: :ref:`remote` ''' import errno @@ -70,18 +91,19 @@ import uuid from pyroute2.common import basestring -from pyroute2.iproute import AsyncIPRoute +from pyroute2.iproute import IPRoute from pyroute2.netlink.exceptions import NetlinkError +from pyroute2.netlink.nlsocket import NetlinkSocketBase from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg +from pyroute2.remote import RemoteIPRoute from .events import ShutdownException, State -from .messages import cmsg_event -from .objects import RTNL_Object +from .messages import cmsg_event, cmsg_failed, cmsg_sstart if sys.platform.startswith('linux'): from pyroute2 import netns - from pyroute2.iproute.linux import NetNS from pyroute2.netns.manager import NetNSManager + from pyroute2.nslink.nslink import NetNS else: NetNS = None NetNSManager = None @@ -134,28 +156,31 @@ class Source(dict): summary_header = None view = None table = 'sources' - vmap = {'local': AsyncIPRoute, 'netns': NetNS, 'nsmanager': NetNSManager} + vmap = { + 'local': IPRoute, + 'netns': NetNS, + 'remote': RemoteIPRoute, + 'nsmanager': NetNSManager, + } def __init__(self, ndb, **spec): self.th = None - self.task = None self.nl = None self.ndb = ndb - self.evq = self.ndb.task_manager.event_queue + self.evq = self.ndb._event_queue # the target id -- just in case self.target = spec['target'] self.kind = spec.pop('kind', 'local') self.max_errors = spec.pop('max_errors', SOURCE_MAX_ERRORS) self.event = spec.pop('event') # RTNL API + self.nl_prime = self.get_prime(self.kind) self.nl_kwarg = spec - self.nl_kwarg['nlm_echo'] = True # if self.ndb.messenger is not None: self.ndb.messenger.targets.add(self.target) # self.errors_counter = 0 - self.exception = None self.shutdown = threading.Event() self.started = threading.Event() self.lock = threading.RLock() @@ -164,12 +189,14 @@ def __init__(self, ndb, **spec): self.log = ndb.log.channel('sources.%s' % self.target) self.state = State(log=self.log, wait_list=['running']) self.state.set('init') - self.ndb.schema.add_nl_source(self.target, self.kind, spec) + self.ndb.task_manager.db_add_nl_source(self.target, self.kind, spec) self.load_sql() @classmethod def _count(cls, view): - return view.ndb.schema.fetchone(f'SELECT count(*) FROM {view.table}') + return view.ndb.task_manager.db_fetchone( + "SELECT count(*) FROM %s" % view.table + ) @property def must_restart(self): @@ -190,10 +217,14 @@ def bind_arguments(self): ) ) - async def set_ready(self): + def set_ready(self): try: if self.event is not None: - await self.evq.put(cmsg_event(self.target, self.event)) + self.evq.put( + (cmsg_event(self.target, self.event),), source=self.target + ) + else: + self.evq.put((cmsg_sstart(self.target),), source=self.target) except ShutdownException: self.state.set('stop') return False @@ -217,7 +248,12 @@ def defaults(cls, spec): return ret def __repr__(self): - return '[%s] <%s>' % (self.state.get(), self.nl_kwarg) + if isinstance(self.nl_prime, NetlinkSocketBase): + name = self.nl_prime.__class__.__name__ + elif isinstance(self.nl_prime, type): + name = self.nl_prime.__name__ + + return '[%s] <%s %s>' % (self.state.get(), name, self.nl_kwarg) @classmethod def nla2name(cls, name): @@ -248,16 +284,12 @@ def get_prime(self, name): importlib.import_module('pyroute2'), self.kind ) - async def api(self, name, *argv, **kwarg): + def api(self, name, *argv, **kwarg): for _ in range(100): # FIXME make a constant with self.lock: try: self.log.debug(f'source api run {name} {argv} {kwarg}') - result = await getattr(self.nl, name)(*argv, **kwarg) - if isinstance(result, list): - for msg in result: - await self.evq.put(msg) - return result + return getattr(self.nl, name)(*argv, **kwarg) except ( NetlinkError, AttributeError, @@ -275,13 +307,12 @@ async def api(self, name, *argv, **kwarg): time.sleep(1) raise RuntimeError('api call failed') - async def fake_zero_if(self): + def fake_zero_if(self): url = 'https://github.com/svinota/pyroute2/issues/737' zero_if = ifinfmsg() zero_if['index'] = 0 zero_if['state'] = 'up' zero_if['flags'] = 1 - zero_if['family'] = 0 zero_if['header']['flags'] = 2 zero_if['header']['type'] = 16 zero_if['header']['target'] = self.target @@ -291,48 +322,140 @@ async def fake_zero_if(self): ('IFLA_ADDRESS', '00:00:00:00:00:00'), ] zero_if.encode() - await self.evq.put(zero_if) + self.evq.put([zero_if], source=self.target) - async def receiver(self): + def receiver(self): # # The source thread routine -- get events from the # channel and forward them into the common event queue # # The routine exists on an event with error code == 104 # - if self.nl is not None: - try: - self.nl.close(code=0) - except Exception as e: - self.log.warning('source restart: %s' % e) + while self.state.get() != 'stop': + if self.shutdown.is_set(): + break + + with self.lock: + if self.nl is not None: + try: + self.nl.close(code=0) + except Exception as e: + self.log.warning('source restart: %s' % e) + try: + self.state.set('connecting') + if isinstance(self.nl_prime, type): + spec = {} + spec.update(self.nl_kwarg) + if self.kind in ('nsmanager',): + spec['libc'] = self.ndb.libc + self.nl = self.nl_prime(**spec) + else: + raise TypeError('source channel not supported') + self.state.set('loading') + # + self.nl.bind(**self.bind_arguments) + # + # Initial load -- enqueue the data + # + try: + self.ndb.task_manager.db_flush(self.target) + if self.kind in ('local', 'netns', 'remote'): + self.fake_zero_if() + self.evq.put(self.nl.dump(), source=self.target) + finally: + pass + self.errors_counter = 0 + except Exception as e: + self.errors_counter += 1 + self.started.set() + self.state.set(f'failed, counter {self.errors_counter}') + self.log.error(f'source error: {type(e)} {e}') + try: + self.evq.put( + (cmsg_failed(self.target),), source=self.target + ) + except ShutdownException: + self.state.set('stop') + break + if self.must_restart: + self.log.debug('sleeping before restart') + self.state.set('restart') + self.shutdown.wait(SOURCE_FAIL_PAUSE) + if self.shutdown.is_set(): + self.log.debug('source shutdown') + self.state.set('stop') + break + else: + return self.set_ready() + continue + + with self.lock: + if self.state.get() == 'loading': + if not self.set_ready(): + break + self.started.set() + self.shutdown.clear() + self.state.set('running') + + while self.state.get() not in ('stop', 'restart'): + try: + msg = tuple(self.nl.get()) + except Exception as e: + self.errors_counter += 1 + self.log.error('source error: %s %s' % (type(e), e)) + msg = None + if self.must_restart: + self.state.set('restart') + else: + self.state.set('stop') + break + + code = 0 + if msg and msg[0]['header']['error']: + code = msg[0]['header']['error'].code + + if msg is None or code == errno.ECONNRESET: + self.state.set('stop') + break + + try: + self.evq.put(msg, source=self.target) + except ShutdownException: + self.state.set('stop') + break + + # thus we make sure that all the events from + # this source are consumed by the main loop + # in __dbm__() routine try: - self.state.set('connecting') - spec = {} - spec.update(self.nl_kwarg) - self.nl = AsyncIPRoute(**spec) - self.state.set('loading') - # - await self.nl.setup_endpoint() - await self.nl.bind(**self.bind_arguments) - self.log.debug(f'source fd {self.nl.fileno()}') - # - # Initial load -- enqueue the data - # - self.ndb.schema.flush(self.target) - if self.kind in ('local', 'netns', 'remote'): - await self.fake_zero_if() - async for x in await self.nl.dump(): - await self.evq.put(x) - self.state.set('running') - finally: - await self.set_ready() - - while self.state.get() not in ('stop', 'restart'): - async for msg in self.nl.get(): - error = msg['header']['error'] - if error: - raise error - await self.evq.put(msg) + self.sync() + self.log.debug('flush DB for the target') + self.ndb.task_manager.db_flush(self.target) + except ShutdownException: + self.log.debug('shutdown handled by the main thread') + pass + self.state.set('stopped') + + def sync(self): + self.log.debug('sync') + sync = threading.Event() + self.evq.put((cmsg_event(self.target, sync),), source=self.target) + sync.wait() + + def start(self): + # + # Start source thread + with self.lock: + self.log.debug('starting the source') + if (self.th is not None) and self.th.is_alive(): + raise RuntimeError('source is running') + + self.th = threading.Thread( + target=self.receiver, + name='NDB event source: %s' % (self.target), + ) + self.th.start() + return self def close(self, code=errno.ECONNRESET, sync=True): with self.shutdown_lock: @@ -376,42 +499,22 @@ def __exit__(self, exc_type, exc_value, traceback): def load_sql(self): # - spec = self.ndb.schema.fetchone( - 'SELECT * FROM sources WHERE f_target = ?', (self.target,) + spec = self.ndb.task_manager.db_fetchone( + ''' + SELECT * FROM sources + WHERE f_target = %s + ''' + % self.ndb.schema.plch, + (self.target,), ) self['target'], self['kind'] = spec - for spec in self.ndb.schema.fetch( - 'SELECT * FROM sources_options WHERE f_target = ?', (self.target,) + for spec in self.ndb.task_manager.db_fetch( + ''' + SELECT * FROM sources_options + WHERE f_target = %s + ''' + % self.ndb.schema.plch, + (self.target,), ): f_target, f_name, f_type, f_value = spec self[f_name] = int(f_value) if f_type == 'int' else f_value - - -class SyncSource(RTNL_Object): - - @property - def nl(self): - return self.asyncore.nl - - def api(self, name, *argv, **kwarg): - return self._main_sync_call(self.asyncore.api, name, *argv, **kwarg) - - def set(self, key, value): - if key == 'state': - self.asyncore.ndb.task_manager.task_map[ - self.asyncore.task_id - ].state.set(value) - return - raise RuntimeError('unknown property') - - def restart(self, reason='no reason'): - self.asyncore.event.clear() - self.close(next_state='running') - self._main_async_call(self.asyncore.event.wait) - - def close(self, code=errno.ECONNRESET, sync=None, next_state='stopped'): - self.set('state', next_state) - self._main_sync_call( - self.asyncore.ndb.schema.flush, self.asyncore.target - ) - self._main_sync_call(self.asyncore.nl.close, code) diff --git a/pyroute2/ndb/sync_api.py b/pyroute2/ndb/sync_api.py deleted file mode 100644 index 1b846901a..000000000 --- a/pyroute2/ndb/sync_api.py +++ /dev/null @@ -1,206 +0,0 @@ -import asyncio -import errno -from enum import IntFlag -from queue import Queue -from typing import ClassVar - -from .report import RecordSet - - -class Flags(IntFlag): - UNSPEC = 0 - RO = 1 - - -class Permit: - def __init__(self, *names: str): - self.names = names - - def __iter__(self): - return iter(self.names) - - -class SyncBase: - - permit_calls: ClassVar[Permit] = Permit('test') - - def __init__(self, event_loop, obj, class_map=None, flags=Flags.UNSPEC): - self.event_loop = event_loop - self.asyncore = obj - self.flags = flags - self.class_map = {} if class_map is None else class_map - - def check_permissions(self, name): - if name in self.permit_calls: - return - if self.flags & Flags.RO: - raise PermissionError('Access denied') - - def _get_sync_class(self, item, key=None): - if key is None: - key = self.asyncore.table - return self.class_map.get(key, self.class_map.get('default'))( - self.event_loop, item, self.class_map, self.flags - ) - - async def _tm_sync_generator(self, queue, func, *argv, **kwarg): - for record in func(*argv, **kwarg): - queue.put(record) - queue.put(None) - - def _main_sync_generator(self, func, *argv, **kwarg): - self.check_permissions(func.__name__) - queue = Queue() - task = asyncio.run_coroutine_threadsafe( - self._tm_sync_generator(queue, func, *argv, **kwarg), - self.event_loop, - ) - while True: - record = queue.get() - if record is None: - return - yield record - ret = task.result() - if isinstance(ret, Exception): - raise ret - - async def _tm_sync_call(self, func, *argv, **kwarg): - return func(*argv, **kwarg) - - def _main_sync_call(self, func, *argv, **kwarg): - self.check_permissions(func.__name__) - task = asyncio.run_coroutine_threadsafe( - self._tm_sync_call(func, *argv, **kwarg), self.event_loop - ) - ret = task.result() - if isinstance(ret, Exception): - raise ret - return ret - - def _main_async_call(self, func, *argv, **kwarg): - self.check_permissions(func.__name__) - task = asyncio.run_coroutine_threadsafe( - func(*argv, **kwarg), self.event_loop - ) - ret = task.result() - if isinstance(ret, Exception): - raise ret - return ret - - def __eq__(self, other): - return self.asyncore == getattr(other, 'asyncore', None) - - -class SyncDB(SyncBase): - - def export(self, f='stdout'): - return self._main_sync_call(self.asyncore.schema.export) - - def backup(self, spec): - return self._main_sync_call(self.asyncore.schema.backup, spec) - - def fetchone(self, query): - return self._main_sync_call(self.asyncore.schema.fetchone, query) - - -class SyncView(SyncBase): - - permit_calls = Permit('summary', 'dump', '__getitem__') - - def __getitem__(self, key, table=None): - item = self._main_sync_call(self.asyncore.__getitem__, key, table) - return self._get_sync_class(item) - - def __contains__(self, key): - return key in self.keys() - - def __iter__(self): - return self.keys() - - def __len__(self): - return self._main_sync_call(self.asyncore.__len__) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - pass - - @property - def cache(self): - return self.asyncore.cache - - def get(self, spec=None, table=None, **kwarg): - item = self._main_sync_call(self.asyncore.get, spec, table, **kwarg) - if item is not None: - return self._get_sync_class(item) - - def getmany(self, spec, table=None): - for item in tuple( - self._main_sync_generator(self.asyncore.getmany, spec, table) - ): - yield item - - def getone(self, spec, table=None): - return self._main_sync_call(self.asyncore.getone, spec, table) - - def items(self): - for key in self.keys(): - yield (key, self[key]) - - def keys(self): - for record in self.dump(): - yield record - - def count(self): - return self._main_sync_call(self.asyncore.count) - - def create(self, *argspec, **kwarg): - item = self._main_sync_call(self.asyncore.create, *argspec, **kwarg) - return self._get_sync_class(item) - - def ensure(self, *argspec, **kwarg): - item = self._main_sync_call(self.asyncore.ensure, *argspec, **kwarg) - return self._get_sync_class(item) - - def add(self, *argspec, **kwarg): - item = self._main_sync_call(self.asyncore.add, *argspec, **kwarg) - return self._get_sync_class(item) - - def wait(self, **spec): - item = self._main_async_call(self.asyncore.wait, **spec) - return self._get_sync_class(item) - - def exists(self, key, table=None): - return self._main_sync_call(self.asyncore.exists, key, table) - - def locate(self, spec=None, table=None, **kwarg): - item = self._main_sync_call(self.asyncore.locate, spec, table, **kwarg) - return self._get_sync_class(item) - - def summary(self): - return RecordSet(self._main_sync_generator(self.asyncore.summary)) - - def dump(self): - return RecordSet(self._main_sync_generator(self.asyncore.dump)) - - -class SyncSources(SyncView): - - permit_calls = Permit('add', 'remove', 'keys', '__getitem__') - - def __getitem__(self, key): - item = self._main_sync_call(self.asyncore.__getitem__, key) - return self._get_sync_class(item) - - def add(self, **spec): - item = self._main_async_call(self.asyncore.add, **spec) - return self._get_sync_class(item) - - def remove(self, target, code=errno.ECONNRESET, sync=True): - item = self._main_sync_call(self.asyncore.remove, target, code, sync) - return self._get_sync_class(item) - - def keys(self): - for record in self.asyncore.keys(): - yield record diff --git a/pyroute2/ndb/task_manager.py b/pyroute2/ndb/task_manager.py index 7a5602f85..7e05d65fe 100644 --- a/pyroute2/ndb/task_manager.py +++ b/pyroute2/ndb/task_manager.py @@ -1,8 +1,10 @@ -import asyncio +import inspect import logging +import queue import threading import time import traceback +from functools import partial from pyroute2 import config @@ -12,13 +14,19 @@ InvalidateHandlerException, RescheduleException, ShutdownException, - State, ) -from .messages import cmsg_event, cmsg_failed +from .messages import cmsg, cmsg_event, cmsg_failed, cmsg_sstart log = logging.getLogger(__name__) +def Events(*argv): + for sequence in argv: + if sequence is not None: + for item in sequence: + yield item + + class NDBConfig(dict): def __init__(self, task_manager): self.task_manager = task_manager @@ -42,54 +50,14 @@ def values(self): return self.task_manager.config_values() -class TaskAdapter: - def __init__(self): - self.event = asyncio.Event() - self.state = State() - - -class Task: - def __init__(self, coro, target_state, obj): - self.exception = None - self.coro = coro - self.target_state = target_state - self.obj = obj if obj is not None else TaskAdapter() - self.restart() - - def restart(self): - self.task = asyncio.create_task(self.coro()) - self.obj.task_id = id(self.task) - - def commit(self): - self.exception = self.task.exception() - - @property - def event(self): - return self.obj.event - - @property - def state(self): - return self.obj.state - - @property - def task_id(self): - return id(self.task) - - class TaskManager: def __init__(self, ndb): self.ndb = ndb self.log = ndb.log self.event_map = {} - self.task_map = {} - self.event_queue = ( - asyncio.Queue() - ) # LoggingQueue(log=self.ndb.log.channel('queue')) - self.stop_event = asyncio.Event() - self.reload_event = asyncio.Event() + self.event_queue = ndb._event_queue self.thread = None self.ctime = self.gctime = time.time() - self.ready = asyncio.Event() def register_handler(self, event, handler): if event not in self.event_map: @@ -99,129 +67,211 @@ def register_handler(self, event, handler): def unregister_handler(self, event, handler): self.event_map[event].remove(handler) - async def handler_default(self, sources, target, event): + def default_handler(self, target, event): if isinstance(getattr(event, 'payload', None), Exception): raise event.payload log.debug('unsupported event ignored: %s' % type(event)) - async def handler_event(self, sources, target, event): - event.payload.set() - - async def handler_failed(self, sources, target, event): - self.ndb.schema.mark(target, 1) - - def main(self): - asyncio.run(self.run()) - - def create_task(self, coro, state='running', obj=None): - task = Task(coro, state, obj) - self.task_map[task.task_id] = task - self.reload_event.set() - return task - - def restart_task(self, task): - task.restart() - self.task_map[task.task_id] = task - self.reload_event.set() - return task - - async def stop(self): - await self.stop_event.wait() - - async def reload(self): - await self.reload_event.wait() - - async def task_watch(self): - while True: - tasks = list([x.task for x in self.task_map.values()]) - done, pending = await asyncio.wait( - tasks, return_when=asyncio.FIRST_COMPLETED - ) - self.log.debug(f'task done {done}') - if self.stop_event.is_set(): - return - for t in done: - task = self.task_map.pop(id(t)) - task.commit() - if task.target_state == 'running': - self.restart_task(task) - self.reload_event.clear() - - async def receiver(self): - while True: - event = await self.event_queue.get() - reschedule = [] - handlers = self.event_map.get( - event.__class__, [self.handler_default] - ) - - for handler in tuple(handlers): - try: - target = event['header']['target'] - # self.log.debug(f'await {handler} for {event}') - await handler(self.ndb.sources.asyncore, target, event) - except RescheduleException: - if 'rcounter' not in event['header']: - event['header']['rcounter'] = 0 - if event['header']['rcounter'] < 3: - event['header']['rcounter'] += 1 - self.log.debug('reschedule %s' % (event,)) - reschedule.append(event) + def check_sources_started(self, _locals, target, event): + _locals['countdown'] -= 1 + if _locals['countdown'] == 0: + self.ndb._dbm_ready.set() + + def wrap_method(self, method): + # + # this wrapper will be published in the DBM thread + # + def _do_local_generator(target, request): + try: + for item in method(*request.argv, **request.kwarg): + request.response.put(item) + request.response.put(StopIteration()) + except Exception as e: + request.response.put(e) + + def _do_local_single(target, request): + try: + (request.response.put(method(*request.argv, **request.kwarg))) + except Exception as e: + (request.response.put(e)) + + # + # this class will be used to map the requests + # + class cmsg_req(cmsg): + def __init__(self, response, *argv, **kwarg): + self['header'] = {'target': None} + self.response = response + self.argv = argv + self.kwarg = kwarg + + # + # this method will proxy the original one + # + def _do_dispatch_generator(self, *argv, **kwarg): + if self.thread == id(threading.current_thread()): + # same thread, run method locally + for item in method(*argv, **kwarg): + yield item + else: + # another thread, run via message bus + response = queue.Queue() + request = cmsg_req(response, *argv, **kwarg) + self.event_queue.put((request,)) + while True: + item = response.get() + if isinstance(item, StopIteration): + return + elif isinstance(item, Exception): + raise item else: - self.log.error('drop %s' % (event,)) - except InvalidateHandlerException: - try: - handlers.remove(handler) - except Exception: - self.log.error( - 'could not invalidate ' - 'event handler:\n%s' % traceback.format_exc() - ) - except ShutdownException: - return - except DBMExitException: - return - except Exception: - self.log.error( - 'could not load event:\n%s\n%s' - % (event, traceback.format_exc()) - ) - if time.time() - self.gctime > config.gc_timeout: - self.gctime = time.time() - - async def run(self): + yield item + + def _do_dispatch_single(self, *argv, **kwarg): + if self.thread == id(threading.current_thread()): + # same thread, run method locally + return method(*argv, **kwarg) + else: + # another thread, run via message bus + response = queue.Queue(maxsize=1) + request = cmsg_req(response, *argv, **kwarg) + self.event_queue.put((request,)) + ret = response.get() + if isinstance(ret, Exception): + raise ret + else: + return ret + + # + # return the method spec to be announced + # + handler = _do_local_single + proxy = _do_dispatch_single + if inspect.isgeneratorfunction(method): + handler = _do_local_generator + proxy = _do_dispatch_generator + return (cmsg_req, handler, proxy) + + def register_api(self, api_obj, prefix=''): + for name in dir(api_obj): + method = getattr(api_obj, name, None) + if hasattr(method, 'publish'): + if isinstance(method.publish, str): + name = method.publish + name = f'{prefix}{name}' + event, handler, proxy = self.wrap_method(method) + setattr(self, name, partial(proxy, self)) + self.event_map[event] = [handler] + + def run(self): + _locals = {'countdown': len(self.ndb._nl)} self.thread = id(threading.current_thread()) # init the events map event_map = { - cmsg_event: [self.handler_event], - cmsg_failed: [self.handler_failed], + cmsg_event: [lambda t, x: x.payload.set()], + cmsg_failed: [lambda t, x: (self.ndb.schema.mark(t, 1))], + cmsg_sstart: [partial(self.check_sources_started, _locals)], } self.event_map = event_map try: self.ndb.schema = schema.DBSchema( - self.ndb.config, self.event_map, self.log.channel('schema') + self.ndb.config, + self.ndb.sources, + self.event_map, + self.log.channel('schema'), ) - self.ndb.config = NDBConfig(self) + self.register_api(self.ndb.schema, 'db_') + self.register_api(self.ndb.schema.config, 'config_') + self.ndb.bonfig = NDBConfig(self) except Exception as e: self.ndb._dbm_error = e - self.ready.set() + self.ndb._dbm_ready.set() return + for spec in self.ndb._nl: + spec['event'] = None + self.ndb.sources.add(**spec) + for event, handlers in self.ndb.schema.event_map.items(): for handler in handlers: self.register_handler(event, handler) - # create an event loop - self.event_loop = asyncio.get_event_loop() - self.ndb.event_loop = self.event_loop - self.create_task(self.receiver) - self.create_task(self.stop) - self.create_task(self.reload) - self.ndb._dbm_ready.set() - await self.task_watch() + stop = False + source = None + reschedule = [] + while not stop: + source, events = self.event_queue.get() + events = Events(events, reschedule) + reschedule = [] + try: + for event in events: + handlers = event_map.get( + event.__class__, [self.default_handler] + ) + + for handler in tuple(handlers): + try: + target = event['header']['target'] + handler(target, event) + except RescheduleException: + if 'rcounter' not in event['header']: + event['header']['rcounter'] = 0 + if event['header']['rcounter'] < 3: + event['header']['rcounter'] += 1 + self.log.debug('reschedule %s' % (event,)) + reschedule.append(event) + else: + self.log.error('drop %s' % (event,)) + except InvalidateHandlerException: + try: + handlers.remove(handler) + except Exception: + self.log.error( + 'could not invalidate ' + 'event handler:\n%s' + % traceback.format_exc() + ) + except ShutdownException: + stop = True + break + except DBMExitException: + return + except Exception: + self.log.error( + 'could not load event:\n%s\n%s' + % (event, traceback.format_exc()) + ) + if time.time() - self.gctime > config.gc_timeout: + self.gctime = time.time() + except Exception as e: + self.log.error(f'exception <{e}> in source {source}') + # restart the target + try: + self.log.debug(f'requesting source {source} restart') + self.ndb.sources[source].state.set('restart') + except KeyError: + self.log.debug(f'key error for {source}') + pass + + # release all the sources + for target in tuple(self.ndb.sources.cache): + source = self.ndb.sources.remove(target, sync=False) + if source is not None and source.th is not None: + self.log.debug(f'closing source {source}') + source.close() + if self.ndb.schema.config['db_cleanup']: + self.log.debug('flush DB for the target %s' % target) + self.ndb.schema.flush(target) + else: + self.log.debug('leave DB for debug') + + # close the database + self.ndb.schema.commit() self.ndb.schema.close() - self.ndb._dbm_shutdown.set() - self.ready.set() + + # close the logging + for handler in self.log.logger.handlers: + handler.close() diff --git a/pyroute2/ndb/transport.py b/pyroute2/ndb/transport.py new file mode 100644 index 000000000..c3c4f7f19 --- /dev/null +++ b/pyroute2/ndb/transport.py @@ -0,0 +1,210 @@ +import pickle +import select +import socket +import struct +import time +import uuid + + +class IdCache(dict): + def invalidate(self): + current_time = time.time() + collect_time = current_time - 60 + for mid, meta in tuple(self.items()): + if meta < collect_time: + self.pop(mid) + + def __setitem__(self, key, value): + if len(self) > 100: + self.invalidate() + dict.__setitem__(self, key, value) + + +class Peer(object): + def __init__(self, remote_id, local_id, address, port, cache): + self.address = address + self.port = port + self.socket = None + self.remote_id = remote_id + self.local_id = local_id + self.cache = cache + self.version = 0 + self.last_exception_time = 0 + + @property + def connected(self): + return self.socket is not None + + def __repr__(self): + if self.connected: + connected = 'not connected' + else: + connected = 'connected' + return '[%s-%s] %s:%s [%s]' % ( + self.local_id, + self.remote_id, + self.address, + self.port, + connected, + ) + + def hello(self): + while True: + message_id = str(uuid.uuid4().hex) + if message_id not in self.cache: + self.cache[message_id] = time.time() + break + data = pickle.dumps( + {'type': 'system', 'id': message_id, 'data': 'HELLO'} + ) + self.send(data) + + def send(self, data): + length = len(data) + data = struct.pack('III', length, self.version, self.local_id) + data + if self.socket is None: + if time.time() - self.last_exception_time < 5: + return + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + self.socket.connect((self.address, self.port)) + self.hello() + except Exception: + self.last_exception_time = time.time() + self.socket = None + return + try: + self.socket.send(data) + except Exception: + try: + self.socket.close() + except Exception: + pass + self.socket = None + + def close(self): + self.socket.close() + + +class Transport(object): + def __init__(self, address, port): + self.peers = [] + self.address = address + self.port = port + self.version = 0 + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1048576) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.socket.bind((self.address, self.port)) + self.socket.listen(16) + self.stream_endpoints = [] + + def add_peer(self, peer): + self.peers.append(peer) + + def send(self, data, exclude=None): + exclude = exclude or [] + ret = [] + for peer in self.peers: + if peer.remote_id not in exclude: + ret.append(peer.send(data)) + return ret + + def get(self): + while True: + fds = [self.socket] + self.stream_endpoints + [rlist, wlist, xlist] = select.select(fds, [], fds) + for fd in xlist: + if fd in self.stream_endpoints: + ( + self.stream_endpoints.pop( + self.stream_endpoints.index(fd) + ) + ) + for fd in rlist: + if fd == self.socket: + new_fd, raddr = self.socket.accept() + self.stream_endpoints.append(new_fd) + else: + data = fd.recv(8) + if len(data) == 0: + ( + self.stream_endpoints.pop( + self.stream_endpoints.index(fd) + ) + ) + continue + length, version, remote_id = struct.unpack('III', data) + if version != self.version: + continue + data = b'' + while len(data) < length: + data += fd.recv(length - len(data)) + return data, remote_id + + def close(self): + self.socket.close() + + +class Messenger(object): + def __init__(self, local_id, transport=None): + self.local_id = local_id + self.transport = transport or Transport('127.0.0.1', 5680) + self.targets = set() + self.id_cache = IdCache() + + def __iter__(self): + return self + + def __next__(self): + while True: + msg = self.handle() + if msg is not None: + return msg + + def handle(self): + data, remote_id = self.transport.get() + message = pickle.loads(data) + + if message['id'] in self.id_cache: + # discard message + return None + + if message['type'] == 'system': + # forward system messages + self.transport.send(data, exclude=[remote_id]) + return message + + self.id_cache[message['id']] = time.time() + + if ( + message['type'] == 'transport' + and message['target'] in self.targets + ): + # ignore DB updates with the same target + message = None + elif ( + message['type'] == 'api' and message['target'] not in self.targets + ): + # ignore API messages with other targets + message = None + + self.transport.send(data, exclude=[remote_id]) + return message + + def emit(self, message): + while True: + message_id = '%s-%s' % ( + message.get('target', '-'), + uuid.uuid4().hex, + ) + if message_id not in self.id_cache: + self.id_cache[message_id] = time.time() + break + + message['id'] = message_id + return self.transport.send(pickle.dumps(message)) + + def add_peer(self, remote_id, address, port): + peer = Peer(remote_id, self.local_id, address, port, self.id_cache) + self.transport.add_peer(peer) diff --git a/pyroute2/ndb/view.py b/pyroute2/ndb/view.py index 83aa3b197..09da32ffb 100644 --- a/pyroute2/ndb/view.py +++ b/pyroute2/ndb/view.py @@ -46,26 +46,26 @@ === ''' -import asyncio import errno import gc import json +import queue import threading import time from collections import OrderedDict from functools import partial -from pyroute2 import config +from pyroute2 import cli, config from pyroute2.common import basestring ## # NDB stuff +from .auth_manager import check_auth from .objects import RSLV_DELETE from .objects.address import Address from .objects.interface import Interface, Vlan from .objects.neighbour import FDBRecord, Neighbour from .objects.netns import NetNS -from .objects.probe import Probe from .objects.route import Route from .objects.rule import Rule from .report import Record, RecordSet @@ -99,13 +99,18 @@ class View(dict): # ifobj1 != ifobj2 ''' - def __init__(self, ndb, table, chain=None): + def __init__(self, ndb, table, chain=None, auth_managers=None): self.ndb = ndb self.log = ndb.log.channel('view.%s' % table) self.table = table self.event = table # FIXME self.chain = chain self.cache = {} + if auth_managers is None: + auth_managers = [] + if chain: + auth_managers += chain.auth_managers + self.auth_managers = auth_managers self.constraints = {} self.classes = OrderedDict() self.classes['interfaces'] = Interface @@ -115,7 +120,6 @@ def __init__(self, ndb, table, chain=None): self.classes['routes'] = Route self.classes['rules'] = Rule self.classes['netns'] = NetNS - self.classes['probes'] = Probe self.classes['af_bridge_vlans'] = Vlan def __enter__(self): @@ -139,12 +143,14 @@ def context(self): return {} def getmany(self, spec, table=None): - return self.ndb.schema.get(table or self.table, spec) + return self.ndb.task_manager.db_get(table or self.table, spec) def getone(self, spec, table=None): for obj in self.getmany(spec, table): return obj + @cli.change_pointer + @check_auth('obj:read') def get(self, spec=None, table=None, **kwarg): spec = spec or kwarg try: @@ -159,8 +165,16 @@ def template(self, key, table=None): context = {} iclass = self.classes[table or self.table] spec = iclass.new_spec(key, context, self.default_target) - return iclass(self, spec, load=False, master=self.chain) + return iclass( + self, + spec, + load=False, + master=self.chain, + auth_managers=self.auth_managers, + ) + @cli.change_pointer + @check_auth('obj:modify') def create(self, *argspec, **kwspec): iclass = self.classes[self.table] if self.chain: @@ -175,6 +189,8 @@ def create(self, *argspec, **kwspec): spec['create'] = True return self[spec] + @cli.change_pointer + @check_auth('obj:modify') def ensure(self, *argspec, **kwspec): try: obj = self.locate(**kwspec) @@ -184,6 +200,8 @@ def ensure(self, *argspec, **kwspec): obj[key] = value return obj + @cli.change_pointer + @check_auth('obj:modify') def add(self, *argspec, **kwspec): self.log.warning( '''\n @@ -198,16 +216,17 @@ def add(self, *argspec, **kwspec): ) return self.create(*argspec, **kwspec) - async def wait(self, **spec): + @check_auth('obj:read') + def wait(self, **spec): ret = None timeout = spec.pop('timeout', -1) action = spec.pop('action', 'add') ctime = time.time() # install a limited events queue -- for a possible immediate reaction - evq = asyncio.Queue(maxsize=100) + evq = queue.Queue(maxsize=100) - async def handler(evq, target, event): + def handler(evq, target, event): # ignore the "queue full" exception # # if we miss some events here, nothing bad happens: we just @@ -217,7 +236,7 @@ async def handler(evq, target, event): # the most important here is not to allocate too much memory try: evq.put_nowait((target, event)) - except asyncio.queues.QueueFull: + except queue.Full: pass with TmpHandler(self.ndb, self.event, partial(handler, evq)): @@ -228,13 +247,14 @@ async def handler(evq, target, event): ): return ret try: - target, msg = await asyncio.wait_for(evq.get(), 1) - except asyncio.TimeoutError: + target, msg = evq.get(timeout=1) + except queue.Empty: pass if timeout > -1: if ctime + timeout < time.time(): raise TimeoutError() + @check_auth('obj:read') def locate(self, spec=None, table=None, **kwarg): ''' This method works like `__getitem__()`, but the important @@ -272,6 +292,7 @@ def locate(self, spec=None, table=None, **kwarg): raise KeyError('got an empty key') return self[request] + @check_auth('obj:read') def __getitem__(self, key, table=None): ret = self.template(key, table) @@ -346,6 +367,7 @@ def exists(self, key, table=None): table = table or self.table schema = self.ndb.schema + task_manager = self.ndb.task_manager names = schema.compiled[self.table]['all_names'] self.log.debug('check if the key %s exists in table %s' % (key, table)) @@ -356,11 +378,11 @@ def exists(self, key, table=None): if nla_name in names: name = nla_name if value is not None and name in names: - keys.append(f'f_{name} = ?') + keys.append('f_%s = %s' % (name, schema.plch)) if isinstance(value, (dict, list, tuple, set)): value = json.dumps(value) values.append(value) - spec = schema.fetchone( + spec = task_manager.db_fetchone( 'SELECT * FROM %s WHERE %s' % (self.table, ' AND '.join(keys)), values, ) @@ -383,18 +405,22 @@ def __iter__(self): def __contains__(self, key): return key in self.dump() + @check_auth('obj:list') def keys(self): for record in self.dump(): yield record + @check_auth('obj:list') def values(self): for key in self.keys(): yield self[key] + @check_auth('obj:list') def items(self): for key in self.keys(): yield (key, self[key]) + @cli.show_result def count(self): return self.classes[self.table]._count(self)[0] @@ -411,13 +437,31 @@ def _native(self, dump): for record in dump: yield Record(fnames, record, self.classes[self.table]) + @cli.show_result + @check_auth('obj:list') def dump(self): iclass = self.classes[self.table] - return RecordSet(self._native(iclass.dump(self))) + return RecordSet( + self._native(iclass.dump(self)), + config={ + 'recordset_pipe': self.ndb.config.get( + 'recordset_pipe', 'false' + ) + }, + ) + @cli.show_result + @check_auth('obj:list') def summary(self): iclass = self.classes[self.table] - return RecordSet(self._native(iclass.summary(self))) + return RecordSet( + self._native(iclass.summary(self)), + config={ + 'recordset_pipe': self.ndb.config.get( + 'recordset_pipe', 'false' + ) + }, + ) def __repr__(self): if self.chain and 'ifname' in self.chain: @@ -439,35 +483,34 @@ def __repr__(self): class SourcesView(View): - def __init__(self, ndb, table='sources'): - super(SourcesView, self).__init__(ndb, table) - self.classes[table] = Source + def __init__(self, ndb, auth_managers=None): + super(SourcesView, self).__init__(ndb, 'sources') + self.classes['sources'] = Source self.cache = {} self.proxy = {} self.lock = threading.Lock() + if auth_managers is None: + auth_managers = [] + self.auth_managers = auth_managers def async_add(self, **spec): spec = dict(Source.defaults(spec)) self.cache[spec['target']] = Source(self.ndb, **spec).start() return self.cache[spec['target']] - async def add(self, **spec): + def add(self, **spec): spec = dict(Source.defaults(spec)) target = spec['target'] if target in self: raise KeyError(f'source {target} exists') - if spec.get('event') is None: + if 'event' not in spec: sync = True - spec['event'] = asyncio.Event() + spec['event'] = threading.Event() else: sync = False - source = Source(self.ndb, **spec) - self.cache[spec['target']] = source - task = self.ndb.task_manager.create_task(source.receiver, obj=source) + self.cache[spec['target']] = Source(self.ndb, **spec).start() if sync: - await task.event.wait() - if task.exception: - raise task.exception + self.cache[spec['target']].event.wait() return self.cache[spec['target']] def remove(self, target, code=errno.ECONNRESET, sync=True): @@ -476,12 +519,10 @@ def remove(self, target, code=errno.ECONNRESET, sync=True): with self.lock: if target in self.cache: source = self.cache[target] - task = self.ndb.task_manager.task_map[source.task_id] - task.target_state = 'stopped' - self.ndb.schema.flush(target) - source.nl.close(code) + source.close(code=code, sync=sync) return self.cache.pop(target) + @check_auth('obj:list') def keys(self): for key in self.cache: yield key diff --git a/pyroute2/netlink/__init__.py b/pyroute2/netlink/__init__.py index 2266445d1..4ea5bd659 100644 --- a/pyroute2/netlink/__init__.py +++ b/pyroute2/netlink/__init__.py @@ -473,7 +473,6 @@ class my_msg(nlmsg): import io import logging -import socket import struct import sys import threading @@ -503,7 +502,8 @@ class NotInitialized(Exception): ## # That's a hack for the code linter, which works under # Python3, see unicode reference in the code below -unicode = str +if sys.version[0] == '3': + unicode = str NLMSG_MIN_TYPE = 0x10 @@ -827,7 +827,6 @@ class nlmsg_base(dict): fields = () header = () - defaults = {} pack = None # pack pragma cell_header = None align = 4 @@ -842,8 +841,6 @@ class nlmsg_base(dict): prefix = None own_parent = False header_type = None - header_fmt = None - decode_as = None # caches __compiled_nla = False __compiled_ft = False @@ -876,6 +873,8 @@ def __init__( ): global cache_jit dict.__init__(self) + for i in self.fields: + self[i[0]] = 0 # FIXME: only for number values self._buf = None self.data = data or bytearray() self.offset = offset @@ -895,7 +894,7 @@ def __init__( self.value = NotInitialized # work only on non-empty mappings if self.nla_map and not self.__class__.__compiled_nla: - self.compile_nla_table() + self.compile_nla() if self.header: self['header'] = {} @@ -984,11 +983,11 @@ def __ops(self, rvalue, op0, op1): lvalue = self.getvalue() res = self.__class__() for key, _ in res.fields: - res.pop(key, None) + del res[key] if 'header' in res: - res.pop('header', None) + del res['header'] if 'value' in res: - res.pop('value', None) + del res['value'] for key in lvalue: if key not in ('header', 'attrs', '__align'): if op0 == '__sub__': @@ -1003,10 +1002,12 @@ def __ops(self, rvalue, op0, op1): res['attrs'] = [] for attr in lvalue['attrs']: if isinstance(attr[1], nlmsg_base): + print("recursion") diff = getattr(attr[1], op0)(rvalue.get_attr(attr[0])) if diff is not None: res['attrs'].append([attr[0], diff]) else: + print("fail", type(attr[1])) if op0 == '__sub__': # operator -, complement if rvalue.get_attr(attr[0]) != attr[1]: @@ -1019,6 +1020,7 @@ def __ops(self, rvalue, op0, op1): del res['attrs'] if not res: return None + print(res) return res def __bool__(self): @@ -1231,9 +1233,8 @@ def encode(self): ) offset = self.offset for name, fmt in self.header: - default = self.defaults.get('header', {}).get(name, 0) struct.pack_into( - fmt, self.data, offset, self['header'].get(name, default) + fmt, self.data, offset, self['header'].get(name, 0) ) offset += struct.calcsize(fmt) @@ -1348,10 +1349,6 @@ def __getitem__(self, key): return self.chain[key] if key == 'value' and key not in self: return NotInitialized - if key not in self: - fields = dict(self.fields) - if key in fields: - return 0 return dict.__getitem__(self, key) def __delitem__(self, key): @@ -1390,30 +1387,6 @@ def load(self, dump): self.setvalue(dump) return self - def dump_attrs(self, attrs): - ret = [] - for i in attrs: - if isinstance(i, nlmsg_base): - ret.append(i.dump()) - elif isinstance(i, (set, list, tuple, nla_slot)): - if isinstance(i[1], nlmsg_base): - # catch nlmsg - ret.append([i[0], i[1].dump()]) - elif isinstance(i[1], (set, list, tuple)): - # catch iterables - ret.append([i[0], self.dump_attrs(i[1])]) - elif isinstance(i[1], bytes): - ret.append([i[0], hexdump(i[1])]) - else: - ret.append([i[0], i[1]]) - elif isinstance(i, dict): - if 'attrs' in i: - i['attrs'] = self.dump_attrs(i['attrs']) - ret.append(i) - else: - ret.append(i) - return ret - def dump(self): ''' Dump packet as a dict @@ -1425,11 +1398,16 @@ def dump(self): if k == 'header': ret['header'] = dict(a['header']) elif k == 'attrs': - ret['attrs'] = self.dump_attrs(v) + ret['attrs'] = attrs = [] + for i in a['attrs']: + if isinstance(i[1], nlmsg_base): + attrs.append([i[0], i[1].dump()]) + elif isinstance(i[1], set): + attrs.append([i[0], tuple(i[1])]) + else: + attrs.append([i[0], i[1]]) else: ret[k] = v - elif isinstance(a, nlmsg_base): - ret = a.dump() else: ret = a return ret @@ -1456,7 +1434,7 @@ def getvalue(self): return self - def compile_nla_table(self): + def compile_nla(self): # Bug-Url: https://github.com/svinota/pyroute2/issues/980 # Bug-Url: https://github.com/svinota/pyroute2/pull/981 if isinstance(self.nla_map, NlaMapAdapter): @@ -1637,35 +1615,27 @@ def decode_nlas(self, offset): # NLMSG fields codecs, mixin classes # class nlmsg_decoder_generic(object): - - @staticmethod - def decode_field(fmt, data, offset): - global cache_fmt - ## - # ~~ size = struct.calcsize(efmt) - # - # The use of the cache gives here a tiny performance - # improvement, but it is an improvement anyways - # - size = ( - cache_fmt.get(fmt, None) - or cache_fmt.__setitem__(fmt, struct.calcsize(fmt)) - or cache_fmt[fmt] - ) - ## - value = struct.unpack_from(fmt, data, offset) - offset += size - if len(value) == 1: - return value[0], offset - else: - return value, offset - def ft_decode(self, offset): + global cache_fmt for name, fmt in self.fields: - if isinstance(fmt, str): - self[name], offset = self.decode_field(fmt, self.data, offset) - elif isinstance(fmt, type): - self[name], offset = fmt.decode_from(self.data, offset) + ## + # ~~ size = struct.calcsize(efmt) + # + # The use of the cache gives here a tiny performance + # improvement, but it is an improvement anyways + # + size = ( + cache_fmt.get(fmt, None) + or cache_fmt.__setitem__(fmt, struct.calcsize(fmt)) + or cache_fmt[fmt] + ) + ## + value = struct.unpack_from(fmt, self.data, offset) + offset += size + if len(value) == 1: + self[name] = value[0] + else: + self[name] = value # read NLA chain if self.nla_map: offset = (offset + 4 - 1) & ~(4 - 1) @@ -1718,60 +1688,48 @@ def ft_decode(self, offset): class nlmsg_encoder_generic(object): + def ft_encode(self, offset): + for name, fmt in self.fields: + value = self[name] - @staticmethod - def encode_field(fmt, data, offset, value, zstring=0): - if fmt == 's': - length = len(value or '') + zstring - efmt = '%is' % (length) - else: - length = struct.calcsize(fmt) - efmt = fmt - - data.extend([0] * length) + if fmt == 's': + length = len(value or '') + self.zstring + efmt = '%is' % (length) + else: + length = struct.calcsize(fmt) + efmt = fmt - # in python3 we should force it - if isinstance(value, str): - value = bytes(value, 'utf-8') - elif isinstance(value, float): - value = int(value) + self.data.extend([0] * length) - try: - if fmt[-1] == 'x': - struct.pack_into(efmt, data, offset) - elif type(value) in (list, tuple, set): - struct.pack_into(efmt, data, offset, *value) - elif len(fmt) > 1 and fmt[-1] == 'B' and value == 0: - struct.pack_into(fmt[:-1] + 'x', data, offset) - else: - struct.pack_into(efmt, data, offset, value) - except struct.error: - log.error(''.join(traceback.format_stack())) - log.error(traceback.format_exc()) - log.error("error pack: %s %s %s" % (efmt, value, type(value))) - raise + # in python3 we should force it + if sys.version[0] == '3': + if isinstance(value, str): + value = bytes(value, 'utf-8') + elif isinstance(value, float): + value = int(value) + elif sys.version[0] == '2': + if isinstance(value, unicode): + value = value.encode('utf-8') - return offset + length + try: + if fmt[-1] == 'x': + struct.pack_into(efmt, self.data, offset) + elif type(value) in (list, tuple, set): + struct.pack_into(efmt, self.data, offset, *value) + else: + struct.pack_into(efmt, self.data, offset, value) + except struct.error: + log.error(''.join(traceback.format_stack())) + log.error(traceback.format_exc()) + log.error("error pack: %s %s %s" % (efmt, value, type(value))) + raise - def ft_encode(self, offset): - if hasattr(self, 'zstring'): - zs = self.zstring - else: - zs = 0 - for name, fmt in self.fields: - default = self.defaults.get(name, 0) - value = self[name] if self.get(name) is not None else default + offset += length - if isinstance(fmt, str): - offset = self.encode_field(fmt, self.data, offset, value, zs) - elif isinstance(fmt, type): - offset = fmt.encode_into(self.data, offset, value) + diff = ((offset + 4 - 1) & ~(4 - 1)) - offset + offset += diff + self.data.extend([0] * diff) - diff = 0 - if self.align > 0: - diff = ((offset + self.align - 1) & ~(self.align - 1)) - offset - offset += diff - self.data.extend([0] * diff) return offset, diff @@ -2093,37 +2051,15 @@ class target(nla_base_string): __slots__ = () sql_type = 'TEXT' family = None - family_attr = None own_parent = True - def __init__(self, *argv, **kwarg): - init = kwarg.get('init', None) - if init is not None: - key, value = init.split(',') - if key == 'family' and value.startswith('AF_'): - self.family = getattr(socket, value) - elif key == 'nla': - self.family_attr = value - super().__init__(*argv, **kwarg) - def get_family(self): if self.family is not None: return self.family pointer = self - if self.family_attr is not None: - nla = self.family_attr - else: - nla = 'family' while pointer.parent is not None: pointer = pointer.parent - family = pointer.get(nla) - if family is not None: - return family - return AF_UNSPEC - - @staticmethod - def get_addrlen(family): - return {AF_INET: 4, AF_INET6: 16, AF_MPLS: 4}.get(family, 4) + return pointer.get('family', AF_UNSPEC) def encode(self): family = self.get_family() @@ -2160,17 +2096,14 @@ def encode(self): def decode(self): nla_base_string.decode(self) family = self.get_family() - data = self['value'] if family in (AF_INET, AF_INET6): - if family == AF_INET: - data = data[:4] - elif family == AF_INET6: - data = data[:16] - self.value = inet_ntop(family, data) + self.value = inet_ntop(family, self['value']) elif family == AF_MPLS: self.value = [] - for i in range(len(data) // 4): - label = struct.unpack('>I', data[i * 4 : i * 4 + 4])[0] + for i in range(len(self['value']) // 4): + label = struct.unpack( + '>I', self['value'][i * 4 : i * 4 + 4] + )[0] record = { 'label': (label & 0xFFFFF000) >> 12, 'tc': (label & 0x00000E00) >> 9, @@ -2335,10 +2268,11 @@ def encode(self): def decode(self): nla_base_string.decode(self) self.value = self['value'] - try: - self.value = self.value.decode('utf-8') - except UnicodeDecodeError: - pass # Failed to decode, keep undecoded value + if sys.version_info[0] >= 3: + try: + self.value = self.value.decode('utf-8') + except UnicodeDecodeError: + pass # Failed to decode, keep undecoded value class asciiz(string): ''' @@ -2429,8 +2363,6 @@ class nlmsgerr(nlmsg): fields = (('error', 'i'),) - prefix = 'NLMSGERR_ATTR_' - nla_map = ( ('NLMSGERR_ATTR_UNUSED', 'none'), ('NLMSGERR_ATTR_MSG', 'asciiz'), diff --git a/pyroute2/netlink/connector/cn_proc.py b/pyroute2/netlink/connector/cn_proc.py index 1849079b7..c4128566a 100644 --- a/pyroute2/netlink/connector/cn_proc.py +++ b/pyroute2/netlink/connector/cn_proc.py @@ -59,7 +59,7 @@ class proc_event_uid(proc_event_base): ('process_pid', 'I'), ('process_tgid', 'I'), ('ruid', 'I'), - ('euid', 'I'), + ('rgid', 'I'), ) @@ -67,7 +67,7 @@ class proc_event_gid(proc_event_base): fields = proc_event_base.fields + ( ('process_pid', 'I'), ('process_tgid', 'I'), - ('rgid', 'I'), + ('euid', 'I'), ('egid', 'I'), ) @@ -141,17 +141,14 @@ class ProcEventMarshal(Marshal): PROC_EVENT_EXIT: proc_event_exit, } - def is_enough(self, msg): - return False - class ProcEventSocket(ConnectorSocket): def __init__(self, fileno=None): super().__init__(fileno=fileno) - self.set_marshal(ProcEventMarshal()) + self.marshal = ProcEventMarshal() - def bind(self, **kwarg): - return super().bind(groups=CN_IDX_PROC, **kwarg) + def bind(self): + return super().bind(groups=CN_IDX_PROC) def control(self, listen): msg = proc_event_control() diff --git a/pyroute2/netlink/core.py b/pyroute2/netlink/core.py deleted file mode 100644 index fb76c4910..000000000 --- a/pyroute2/netlink/core.py +++ /dev/null @@ -1,843 +0,0 @@ -import asyncio -import collections -import ctypes -import errno -import logging -import os -import socket -import struct -import threading -import time -import warnings -from contextlib import contextmanager -from dataclasses import asdict -from typing import Optional, Union -from urllib import parse - -from pyroute2 import config, netns -from pyroute2.common import AddrPool -from pyroute2.netlink import NLM_F_MULTI -from pyroute2.netlink.coredata import CoreConfig, CoreSocketSpec -from pyroute2.statsd import StatsDClientSocket - -MAGIC_CLOSE = 0x42 -log = logging.getLogger(__name__) -Stats = collections.namedtuple('Stats', ('qsize', 'delta', 'delay')) - - -class Average: - def __init__(self) -> None: - self.total: int = 0 - self.count: int = 0 - - def add(self, value: int) -> None: - self.total += value - self.count += 1 - - def getvalue(self) -> int: - return self.total // self.count if self.count > 0 else 0 - - -class Telemetry: - sock: Union[config.LocalMock, StatsDClientSocket] - - def __init__( - self, - address: Optional[tuple[str, int]] = None, - use_socket: Optional[socket.socket] = None, - flags: int = os.O_CREAT, - libc: Optional[ctypes.CDLL] = None, - ): - address = address or config.telemetry - self.timings: dict[str, Average] = {} - if address is None: - self.sock = config.LocalMock() - return - save: bool = config.mock_netns - config.mock_netns = False - self.sock = StatsDClientSocket(address, use_socket, flags, libc) - config.mock_netns = save - - @contextmanager - def update(self, name: str): - start = time.time_ns() - try: - yield self - finally: - stop = time.time_ns() - if name not in self.timings: - self.timings[name] = Average() - self.timings[name].add(stop - start) - self.sock.put(name, 1, 'c') - self.sock.put(name, self.timings[name].getvalue(), 'g') - self.sock.commit() - - def incr(self, name: str) -> None: - self.sock.incr(name) - - def close(self): - self.sock.close() - - -class NoClose(socket.socket): - def __init__(self, sock: socket.socket): - self.magic = 0 - fd = sock.fileno() - if fd is not None: - fd = os.dup(fd) - sock.close() - super().__init__(fileno=fd) - - def close(self): - if self.magic == MAGIC_CLOSE: - super().close() - - -class CoreMessageQueue: - - def __init__(self, event_loop): - # python versions < 3.11 require the event loop - # to be running on Queue().__init__(), while - # python 3.12+ bind the queue on get()/put() - # - # so we must require the event loop on __init__() - # just to provide the compatibility. - # - self.event_loop = event_loop - self.queues = {0: asyncio.Queue()} - self.root = self.queues[0] - - async def join(self): - return await self.root.join() - - async def reschedule(self, target): - while not self.root.empty(): - target.append(await self.root.get()) - self.root.task_done() - for key in tuple(self.queues.keys()): - del self.queues[key] - - async def get(self, tag): - ret = await self.queues[tag].get() - self.queues[tag].task_done() - return ret - - async def put(self, tag, message): - if tag not in self.queues: - tag = 0 - return await self.queues[tag].put(message) - - def ensure_tag(self, tag): - if tag not in self.queues: - self.queues[tag] = asyncio.Queue() - - def free_tag(self, tag): - del self.queues[tag] - - def put_nowait(self, tag, message): - if tag not in self.queues: - tag = 0 - return self.queues[tag].put_nowait(message) - - -class CoreProtocol(asyncio.Protocol): - def __init__(self, on_con_lost, enqueue, error_event, status): - self.transport = None - self.enqueue = enqueue - self.on_con_lost = on_con_lost - self.error_event = error_event - self.status = status - - def connection_made(self, transport): - self.transport = transport - - def enqueue_error(self, code: Optional[int]) -> None: - if code is None: - code = errno.ECOMM - try: - self.enqueue(struct.pack('IHHQIQQ', 28, 2, 0, 0, code, 0, 0), None) - except AttributeError: - # while closing, self.msg_queue can be removed already - pass - - def connection_lost(self, exc: Optional[Exception]) -> None: - self.on_con_lost.set_result(True) - self.enqueue_error(errno.ECONNRESET) - - def error_received(self, exc: OSError) -> None: - self.status['error'] = exc - self.error_event.set() - self.enqueue_error(exc.errno) - - -class CoreStreamProtocol(CoreProtocol): - - def data_received(self, data): - log.debug('SOCK_STREAM enqueue %s bytes' % len(data)) - self.enqueue(data, None) - - -class CoreDatagramProtocol(CoreProtocol): - - def datagram_received(self, data, addr): - log.debug('SOCK_DGRAM enqueue %s bytes' % len(data)) - self.enqueue(data, addr) - - -class RequestWrapper: - def __init__(self, event_loop, func): - self.event_loop = event_loop - request = self.event_loop.run_until_complete(func) - self.response = request.__aiter__() - - def __iter__(self): - return self - - def __next__(self): - try: - return self.event_loop.run_until_complete( - self.response.__anext__() - ) - except StopAsyncIteration: - raise StopIteration - - -class AsyncCoreSocket: - '''Pyroute2 core socket class. - - This class implements the core socket concept for all the pyroute2 - communications, both Netlink and internal RPC. - - The asynchronous version is the most basic. All the sync classes - are built on top of it. - ''' - - compiled = None - __spec = None - __marshal = None - - def __init__( - self, - target='localhost', - rcvsize=16384, - use_socket=None, - netns=None, - flags=os.O_CREAT, - libc=None, - groups=0, - use_event_loop=None, - telemetry=None, - ): - # 8<----------------------------------------- - self.spec = CoreSocketSpec( - CoreConfig( - target=target, - rcvsize=rcvsize, - netns=netns, - flags=flags, - groups=groups, - use_libc=libc is not None, - use_socket=use_socket is not None, - use_event_loop=use_event_loop is not None, - telemetry=telemetry, - ) - ) - self.status = self.spec.status - self.status['eids'] = set() - self.local = threading.local() - self.lock = threading.Lock() - self.libc = libc - self.use_socket = use_socket - self.use_event_loop = use_event_loop - self.request_proxy = None - self._tid = threading.get_ident() - self._error_event = threading.Event() - url = parse.urlparse(self.spec['target']) - self.scheme = url.scheme if url.scheme else url.path - self.callbacks = [] # [(predicate, callback, args), ...] - self.addr_pool = AddrPool(minaddr=0x000000FF, maxaddr=0x0000FFFF) - self.marshal = None - self.buffer = [] - self.telemetry = Telemetry(telemetry) - self.msg_reschedule = [] - self.__open_sockets = set() - self.__open_resources = set() - self.__msg_queues = set() - - def _check_tid( - self, - tag: str = '', - level: int = logging.ERROR, - message: Optional[str] = None, - ): - if self._tid != threading.get_ident(): - if message is None: - message = ( - f'calling #{tag} from another thread may require' - f' additional resource cleanup' - ) - if level == logging.ERROR: - raise RuntimeError(message) - elif level == logging.WARN: - warnings.warn(message) - - def _register_loop_ref(self): - self.__open_resources.add((self.event_loop, self.msg_queue)) - - def get_loop(self): - return self.event_loop - - @property - def spec(self): - return self.__spec - - @spec.setter - def spec(self, value): - if self.__spec is None: - self.__spec = value - - @property - def marshal(self): - return self.__marshal - - @marshal.setter - def marshal(self, value): - if self.__marshal is None: - self.__marshal = value - else: - log.debug( - 'preventing override of marshal %s with %s', - self.__marshal, - value, - ) - - def set_marshal(self, value): - self.__marshal = value - - # 8<-------------------------------------------------------------- - # Thread local section - @property - def msg_queue(self): - return self.local.msg_queue - - @property - def connection_lost(self): - if self._error_event.is_set(): - raise self.status['error'] - if not hasattr(self.local, 'connection_lost'): - self.local.connection_lost = self.event_loop.create_future() - return self.local.connection_lost - - @property - def socket(self): - '''socket infrastructure reconciler.''' - if self._error_event.is_set(): - raise self.status['error'] - if self.use_socket is not None and not hasattr(self.local, 'socket'): - if not hasattr(self.use_socket, '_lock'): - self._check_tid(tag='use_socket', level=logging.ERROR) - self.local.prime = None - self.local.socket = self.use_socket - if not hasattr(self.local, 'prime'): - self.local.prime = self.setup_socket() - if not hasattr(self.local, 'msg_queue'): - self.local.msg_queue = CoreMessageQueue(event_loop=self.event_loop) - self.__msg_queues.add(self.local.msg_queue) - if not hasattr(self.local, 'socket'): - self.local.socket = NoClose(self.local.prime) - self.__open_sockets.add(self.local.socket) - return self.local.socket - - @property - def event_loop(self): - if self._error_event.is_set(): - raise self.status['error'] - if not hasattr(self.local, 'event_loop'): - if self.status['use_event_loop']: - self._check_tid( - level=logging.ERROR, - message='Predefined event loop can not be used ' - 'in another thread', - ) - self.local.event_loop = self.setup_event_loop() - if self.local.event_loop.is_closed(): - raise OSError(errno.EBADF, 'Bad file descriptor') - return self.local.event_loop - - @property - def transport(self): - return self.local.transport - - @property - def protocol(self): - if not hasattr(self.local, 'protocol'): - self.local.protocol = None - return self.local.protocol - - # - # 8<-------------------------------------------------------------- - # - def setup_event_loop(self): - try: - event_loop = asyncio.get_running_loop() - self.status['event_loop'] = 'auto' - except RuntimeError: - event_loop = asyncio.new_event_loop() - self.status['event_loop'] = 'new' - self.status['eids'].add(id(event_loop)) - return event_loop - - def setup_socket(self): - if self.use_socket is not None: - return self.use_socket - sock = netns.create_socket( - self.spec['netns'], - socket.AF_INET, - socket.SOCK_STREAM, - flags=self.spec['flags'], - libc=self.libc, - ) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return sock - - async def setup_endpoint(self): - # Setup asyncio - if getattr(self.local, 'transport', None) is not None: - return - self.local.transport, self.local.protocol = ( - await self.event_loop.connect_accepted_socket( - lambda: CoreStreamProtocol( - self.connection_lost, - self.enqueue, - self._error_event, - self.status, - ), - sock=self.socket, - ) - ) - - # 8<-------------------------------------------------------------- - - def __getattr__(self, attr): - if attr in ( - 'getsockname', - 'getsockopt', - 'makefile', - 'setsockopt', - 'setblocking', - 'settimeout', - 'gettimeout', - 'shutdown', - 'recvfrom', - 'recvfrom_into', - 'fileno', - 'sendto', - 'connect', - 'listen', - ): - return getattr(self.socket, attr) - elif attr in ('_sendto', '_recv', '_recv_into'): - return getattr(self.socket, attr.lstrip("_")) - raise AttributeError(attr) - - async def bind(self, addr): - '''Bind the socket to the address.''' - await self.setup_endpoint() - return self.socket.bind(addr) - - def close(self, code=errno.ECONNRESET): - '''Terminate the object.''' - - def send_terminator(msg_queue): - msg_queue.put_nowait(0, b'') - - with self.lock: - for event_loop, msg_queue in self.__open_resources: - if not event_loop.is_closed(): - event_loop.call_soon_threadsafe(send_terminator, msg_queue) - if self.telemetry is not None: - self.telemetry.close() - if hasattr(self.local, 'transport'): - self.local.transport.abort() - self.local.transport.close() - del self.local.transport - del self.local.protocol - if self.status['event_loop'] == 'new': - if ( - hasattr(self.local, 'event_loop') - and not self.local.event_loop.is_closed() - ): - self.event_loop.run_until_complete( - self.event_loop.shutdown_asyncgens() - ) - self.event_loop.stop() - self.event_loop.close() - del self.local.event_loop - for sock in tuple(self.__open_sockets): - sock.magic = MAGIC_CLOSE - sock.close() - - def clone(self): - '''Return a copy of itself with a new underlying socket. - - This method can not work if `use_socket` or `event_loop` - was used in the object constructor.''' - if self.status['use_socket'] or self.status['event_loop'] != 'none': - raise RuntimeError('can not clone socket') - new_spec = {} - for key, value in asdict(self.spec.config).items(): - if key in self.__init__.__code__.co_varnames: - new_spec[key] = value - # post fix - new_spec['use_socket'] = self.use_socket - new_spec['use_event_loop'] = self.use_event_loop - new_spec['libc'] = self.libc - return type(self)(**new_spec) - - def setsockopt(self, level, optname, value): - return self.socket.setsockopt(level, optname, value) - - def getsockopt(self, level, optname): - return self.socket.getsockopt(level, optname) - - def recv(self, buffersize, flags=0): - '''Get one buffer from the socket.''' - return self.socket.recv(buffersize, flags) - - def send(self, data, flags=0): - '''Send one buffer via the socket.''' - return self.socket.send(data, flags) - - def accept(self): - if self.use_socket is not None: - return (self, None) - (connection, address) = self.socket.accept() - new_socket = self.clone() - new_socket.socket = connection - return (new_socket, address) - - def connect(self, address): - self.socket.connect(address) - - def enqueue(self, data, addr): - return self.msg_queue.put_nowait(0, data) - - async def get( - self, msg_seq=0, terminate=None, callback=None, noraise=False - ): - '''Get a conversation answer from the socket.''' - await self.setup_endpoint() - log.debug( - "get: %s / %s / %s / %s", msg_seq, terminate, callback, noraise - ) - if msg_seq == -1: - msg_seq = 0 - enough = False - started = False - error = None - while not enough: - log.debug('await data on %s', self.msg_queue) - data = await self.msg_queue.get(msg_seq) - messages = tuple(self.marshal.parse(data, msg_seq, callback)) - if len(messages) == 0: - break - for msg in messages: - log.debug("message %s", msg) - if msg.get('header', {}).get('error') is not None: - error = msg['header']['error'] - enough = True - break - if self.marshal.is_enough(msg): - enough = True - break - msg['header']['target'] = self.status['target'] - msg['header']['stats'] = Stats(0, 0, 0) - started = True - log.debug("yield %s", msg['header']) - yield msg - - if started and ( - (msg_seq == 0) - or (not msg['header'].get('flags', 0) & NLM_F_MULTI) - or (callable(terminate) and terminate(msg)) - ): - enough = True - if not noraise and error: - raise error - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - self.close() - - def register_callback(self, callback, predicate=lambda x: True, args=None): - ''' - Register a callback to run on a message arrival. - - Callback is the function that will be called with the - message as the first argument. Predicate is the optional - callable object, that returns True or False. Upon True, - the callback will be called. Upon False it will not. - Args is a list or tuple of arguments. - - Simplest example, assume ipr is the IPRoute() instance:: - - # create a simplest callback that will print messages - def cb(msg): - print(msg) - - # register callback for any message: - ipr.register_callback(cb) - - More complex example, with filtering:: - - # Set object's attribute after the message key - def cb(msg, obj): - obj.some_attr = msg["some key"] - - # Register the callback only for the loopback device, index 1: - ipr.register_callback(cb, - lambda x: x.get('index', None) == 1, - (self, )) - - Please note: you do **not** need to register the default 0 queue - to invoke callbacks on broadcast messages. Callbacks are - iterated **before** messages get enqueued. - ''' - if args is None: - args = [] - self.callbacks.append((predicate, callback, args)) - - def unregister_callback(self, callback): - ''' - Remove the first reference to the function from the callback - register - ''' - cb = tuple(self.callbacks) - for cr in cb: - if cr[1] == callback: - self.callbacks.pop(cb.index(cr)) - return - - def register_policy(self, policy, msg_class=None): - ''' - Register netlink encoding/decoding policy. Can - be specified in two ways: - `nlsocket.register_policy(MSG_ID, msg_class)` - to register one particular rule, or - `nlsocket.register_policy({MSG_ID1: msg_class})` - to register several rules at once. - E.g.:: - - policy = {RTM_NEWLINK: ifinfmsg, - RTM_DELLINK: ifinfmsg, - RTM_NEWADDR: ifaddrmsg, - RTM_DELADDR: ifaddrmsg} - nlsocket.register_policy(policy) - - One can call `register_policy()` as many times, - as one want to -- it will just extend the current - policy scheme, not replace it. - ''' - if isinstance(policy, int) and msg_class is not None: - policy = {policy: msg_class} - - if not isinstance(policy, dict): - raise TypeError('wrong policy type') - for key in policy: - self.marshal.msg_map[key] = policy[key] - - return self.marshal.msg_map - - def unregister_policy(self, policy): - ''' - Unregister policy. Policy can be: - - - int -- then it will just remove one policy - - list or tuple of ints -- remove all given - - dict -- remove policies by keys from dict - - In the last case the routine will ignore dict values, - it is implemented so just to make it compatible with - `get_policy_map()` return value. - ''' - if isinstance(policy, int): - policy = [policy] - elif isinstance(policy, dict): - policy = list(policy) - - if not isinstance(policy, (tuple, list, set)): - raise TypeError('wrong policy type') - - for key in policy: - del self.marshal.msg_map[key] - - return self.marshal.msg_map - - def get_policy_map(self, policy=None): - ''' - Return policy for a given message type or for all - message types. Policy parameter can be either int, - or a list of ints. Always return dictionary. - ''' - if policy is None: - return self.marshal.msg_map - - if isinstance(policy, int): - policy = [policy] - - if not isinstance(policy, (list, tuple, set)): - raise TypeError('wrong policy type') - - ret = {} - for key in policy: - ret[key] = self.marshal.msg_map[key] - - return ret - - -class SyncAPI: - ''' - Synchronous API wrapper around asynchronous classes - ''' - - @property - def marshal(self): - return self.asyncore.marshal - - @marshal.setter - def marshal(self, value): - self.asyncore.marshal = value - - def set_marshal(self, value): - return self.asyncore.set_marshal(value) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def __getattr__(self, key): - if key in ( - 'epid', - 'pid', - 'spec', - 'status', - 'send', - 'recv', - 'lock', - 'sendto', - 'setsockopt', - 'getsockopt', - 'register_policy', - 'unregister_policy', - ): - return getattr(self.asyncore, key) - raise AttributeError(key) - - @property - def fileno(self): - return self.asyncore.socket.fileno - - def _setup_transport(self): - if self.asyncore.status['closed']: - raise OSError(errno.EBADF, 'Bad file descriptor') - if hasattr(self.asyncore.local, 'event_loop'): - return - self.asyncore.event_loop.run_until_complete( - self.asyncore.setup_endpoint() - ) - - def _cleanup_transport(self) -> None: - if not self.asyncore.status['closed'] and hasattr( - self.asyncore.local, 'keep_event_loop' - ): - return - if hasattr(self.asyncore, 'transport'): - self.asyncore.transport.abort() - self.asyncore.transport.close() - self.asyncore.event_loop.run_until_complete( - self.asyncore.event_loop.shutdown_asyncgens() - ) - self.asyncore.event_loop.stop() - self.asyncore.event_loop.close() - if hasattr(self.asyncore.local, 'transport'): - del self.asyncore.local.transport - if hasattr(self.asyncore.local, 'connection_lost'): - del self.asyncore.local.connection_lost - if hasattr(self.asyncore.local, 'event_loop'): - del self.asyncore.local.event_loop - - def _generate_with_cleanup(self, func, *argv, **kwarg): - if hasattr(func, '__name__'): - telemetry_tag = func.__name__ - elif hasattr(func, '__func__') and hasattr(func.__func__, '__name__'): - telemetry_tag = func.__func__.__name__ - else: - telemetry_tag = '' - try: - self._setup_transport() - with self.asyncore.telemetry.update(telemetry_tag): - for item in RequestWrapper( - event_loop=self.asyncore.event_loop, - func=func(*argv, **kwarg), - ): - yield item - finally: - self._cleanup_transport() - - def _run_sync_cleanup(self, func, *argv, **kwarg): - return tuple(self._generate_with_cleanup(func, *argv, **kwarg)) - - def _run_with_cleanup(self, func, *argv, **kwarg): - if hasattr(func, '__name__'): - telemetry_tag = func.__name__ - elif hasattr(func, '__func__') and hasattr(func.__func__, '__name__'): - telemetry_tag = func.__func__.__name__ - else: - telemetry_tag = '' - try: - self._setup_transport() - with self.asyncore.telemetry.update(telemetry_tag): - return self.asyncore.event_loop.run_until_complete( - func(*argv, **kwarg) - ) - finally: - self._cleanup_transport() - - def mock_data(self, data): - if getattr(self.asyncore.local, 'msg_queue', None) is None: - self.asyncore.local.msg_queue = CoreMessageQueue( - event_loop=self.event_loop - ) - self.asyncore.msg_queue.put_nowait(0, data) - - def close(self, code=errno.ECONNRESET): - '''Correctly close the socket and free all the resources.''' - self.asyncore.status['closed'] = True - self.asyncore.close(code) - - -class CoreSocket(SyncAPI): - def __init__( - self, - target='localhost', - rcvsize=16384, - use_socket=None, - netns=None, - flags=os.O_CREAT, - libc=None, - groups=0, - ): - self.asyncore = AsyncCoreSocket( - target, rcvsize, use_socket, netns, flags, libc, groups - ) - self.asyncore.status['event_loop'] = 'new' - self.asyncore.local.keep_event_loop = True - self.asyncore.event_loop.run_until_complete( - self.asyncore.setup_endpoint() - ) - if self.asyncore.socket.fileno() == -1: - raise OSError(9, 'Bad file descriptor') diff --git a/pyroute2/netlink/coredata.py b/pyroute2/netlink/coredata.py deleted file mode 100644 index 71c9f9b2b..000000000 --- a/pyroute2/netlink/coredata.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -from dataclasses import asdict, dataclass -from typing import Optional, Type, Union - -from pyroute2 import config -from pyroute2.requests.main import RequestFilter, RequestProcessor - - -@dataclass -class CoreConfig: - target: str = 'localhost' - flags: int = os.O_CREAT - groups: int = 0 - rcvsize: int = 16384 - netns: Optional[str] = None - tag_field: str = '' - address: Optional[tuple[str, int]] = None - telemetry: Optional[tuple[str, int]] = None - use_socket: bool = False - use_event_loop: bool = False - use_libc: bool = False - - -class CoreSocketSpec: - defaults: dict[str, Union[bool, int, str, None, tuple[str, ...]]] = { - 'closed': False, - 'compiled': None, - 'uname': config.uname, - } - status_filters: list[Type[RequestFilter]] = [] - - def __init__(self, config: CoreConfig): - self.config = config - self.status = RequestProcessor() - for flt in self.status_filters: - self.status.add_filter(flt()) - self.status.update(self.defaults) - self.status.update(asdict(self.config)) - - def __setitem__(self, key, value): - setattr(self.config, key, value) - self.status.update(asdict(self.config)) - - def __getitem__(self, key): - return getattr(self.config, key) - - def serializable(self): - return not any( - ( - self.config.use_socket, - self.config.use_event_loop, - self.config.use_libc, - ) - ) diff --git a/pyroute2/netlink/devlink/__init__.py b/pyroute2/netlink/devlink/__init__.py index e1cfdb15c..dc0a18864 100644 --- a/pyroute2/netlink/devlink/__init__.py +++ b/pyroute2/netlink/devlink/__init__.py @@ -5,10 +5,7 @@ from pyroute2.common import map_namespace from pyroute2.netlink import genlmsg, nla -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) +from pyroute2.netlink.generic import GenericNetlinkSocket from pyroute2.netlink.nlsocket import Marshal # devlink commands @@ -495,15 +492,12 @@ def fix_message(self, msg): pass -class AsyncDevlinkSocket(AsyncGenericNetlinkSocket): - marshal_class = MarshalDevlink - - async def bind(self, groups=0, **kwarg): - await super().bind('devlink', devlinkcmd, groups, None, **kwarg) - - class DevlinkSocket(GenericNetlinkSocket): - async_class = AsyncDevlinkSocket + def __init__(self, *args, **kwargs): + GenericNetlinkSocket.__init__(self, *args, **kwargs) + self.marshal = MarshalDevlink() def bind(self, groups=0, **kwarg): - return self._run_with_cleanup(self.asyncore.bind, groups, **kwarg) + GenericNetlinkSocket.bind( + self, 'devlink', devlinkcmd, groups, None, **kwarg + ) diff --git a/pyroute2/netlink/diag/__init__.py b/pyroute2/netlink/diag/__init__.py index b05622dc3..223adfbaf 100644 --- a/pyroute2/netlink/diag/__init__.py +++ b/pyroute2/netlink/diag/__init__.py @@ -328,7 +328,7 @@ class DiagSocket(NetlinkSocket): def __init__(self, fileno=None): super(DiagSocket, self).__init__(NETLINK_SOCK_DIAG) - self.set_marshal(MarshalDiag()) + self.marshal = MarshalDiag() def get_sock_stats( self, diff --git a/pyroute2/netlink/event/__init__.py b/pyroute2/netlink/event/__init__.py index 9022116b7..98b9d8dd1 100644 --- a/pyroute2/netlink/event/__init__.py +++ b/pyroute2/netlink/event/__init__.py @@ -1,43 +1,27 @@ -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) +from pyroute2.config import kernel +from pyroute2.netlink.generic import GenericNetlinkSocket -class AsyncEventSocket(AsyncGenericNetlinkSocket): +class EventSocket(GenericNetlinkSocket): + marshal_class = None genl_family = None - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - await super().setup_endpoint() - await self.bind() + def __init__(self, *args, **kwargs): + GenericNetlinkSocket.__init__(self, *args, **kwargs) + self.marshal = self.marshal_class() + if kernel[0] <= 2: + self.bind(groups=0xFFFFFF) + else: + self.bind() for group in self.mcast_groups: self.add_membership(group) - async def bind(self, groups=0, **kwarg): - await super().bind( + def bind(self, groups=0, **kwarg): + GenericNetlinkSocket.bind( + self, self.genl_family, self.marshal_class.msg_map[0], groups, None, - **kwarg, - ) - - -class EventSocket(GenericNetlinkSocket): - async_class = AsyncEventSocket - marshal_class = None - genl_family = None - - def __init__(self, *args, **kwarg): - if self.marshal_class is not None: - self.async_class.marshal_class = self.marshal_class - if self.genl_family is not None: - self.async_class.genl_family = self.genl_family - super().__init__(*args, **kwarg) - - def bind(self, groups=0, **kwarg): - return self._run_with_cleanup( - self.asyncore.bind, groups=groups, **kwarg + **kwarg ) diff --git a/pyroute2/netlink/event/acpi_event.py b/pyroute2/netlink/event/acpi_event.py index 6425f5853..93b6ca39b 100644 --- a/pyroute2/netlink/event/acpi_event.py +++ b/pyroute2/netlink/event/acpi_event.py @@ -7,7 +7,7 @@ import pyroute2 pyroute2.AcpiEventSocket = acpi_event.AcpiEventMock -.. code:: +.. testcode:: from pprint import pprint from pyroute2 import AcpiEventSocket @@ -27,7 +27,7 @@ from pyroute2.common import load_dump from pyroute2.netlink import genlmsg, nla -from pyroute2.netlink.event import AsyncEventSocket, EventSocket +from pyroute2.netlink.event import EventSocket from pyroute2.netlink.nlsocket import Marshal ACPI_GENL_CMD_UNSPEC = 0 @@ -60,15 +60,11 @@ class MarshalAcpiEvent(Marshal): msg_map = {ACPI_GENL_CMD_UNSPEC: acpimsg, ACPI_GENL_CMD_EVENT: acpimsg} -class AsyncAcpiEventSocket(AsyncEventSocket): +class AcpiEventSocket(EventSocket): marshal_class = MarshalAcpiEvent genl_family = 'acpi_event' -class AcpiEventSocket(EventSocket): - async_class = AsyncAcpiEventSocket - - class AcpiEventMock(AcpiEventSocket): input_from_buffer_queue = True sample_data = ''' @@ -83,5 +79,5 @@ def bind(self, groups=0, **kwarg): self.marshal.msg_map[27] = acpimsg def get(self): - self.mock_data(load_dump(self.sample_data)) + self.buffer_queue.put(load_dump(self.sample_data)) return super().get() diff --git a/pyroute2/netlink/event/dquot.py b/pyroute2/netlink/event/dquot.py index 6037ee0cc..9b243e021 100644 --- a/pyroute2/netlink/event/dquot.py +++ b/pyroute2/netlink/event/dquot.py @@ -7,7 +7,7 @@ import pyroute2 pyroute2.DQuotSocket = dquot.DQuotMock -.. code:: +.. testcode:: from pyroute2 import DQuotSocket @@ -26,7 +26,7 @@ from pyroute2.common import load_dump from pyroute2.netlink import genlmsg -from pyroute2.netlink.event import AsyncEventSocket, EventSocket +from pyroute2.netlink.event import EventSocket from pyroute2.netlink.nlsocket import Marshal QUOTA_NL_C_UNSPEC = 0 @@ -51,15 +51,11 @@ class MarshalDQuot(Marshal): msg_map = {QUOTA_NL_C_UNSPEC: dquotmsg, QUOTA_NL_C_WARNING: dquotmsg} -class AsyncDQuotSocket(AsyncEventSocket): +class DQuotSocket(EventSocket): marshal_class = MarshalDQuot genl_family = 'VFS_DQUOT' -class DQuotSocket(EventSocket): - async_class = AsyncDQuotSocket - - class DQuotMock(DQuotSocket): input_from_buffer_queue = True sample_data = ''' diff --git a/pyroute2/netlink/event/thermal.py b/pyroute2/netlink/event/thermal.py index ae7c348ee..4d2007cd4 100644 --- a/pyroute2/netlink/event/thermal.py +++ b/pyroute2/netlink/event/thermal.py @@ -4,7 +4,7 @@ from enum import Enum from pyroute2.netlink import genlmsg -from pyroute2.netlink.event import AsyncEventSocket, EventSocket +from pyroute2.netlink.event import EventSocket from pyroute2.netlink.nlsocket import Marshal @@ -69,10 +69,6 @@ class MarshalThermalEvent(Marshal): msg_map = {x.value: thermal_msg for x in ThermalGenlEvent} -class AsyncThermalEventSocket(AsyncEventSocket): +class ThermalEventSocket(EventSocket): marshal_class = MarshalThermalEvent genl_family = 'thermal' - - -class ThermalEventSocket(EventSocket): - async_class = AsyncThermalEventSocket diff --git a/pyroute2/netlink/generic/__init__.py b/pyroute2/netlink/generic/__init__.py index 36e6ca197..c3f68e78f 100644 --- a/pyroute2/netlink/generic/__init__.py +++ b/pyroute2/netlink/generic/__init__.py @@ -20,10 +20,10 @@ SOL_NETLINK, ctrlmsg, ) -from pyroute2.netlink.nlsocket import AsyncNetlinkSocket, NetlinkSocket +from pyroute2.netlink.nlsocket import NetlinkSocket -class AsyncGenericNetlinkSocket(AsyncNetlinkSocket): +class GenericNetlinkSocket(NetlinkSocket): ''' Low-level socket interface. Provides all the usual socket does, can be used in poll/select, @@ -45,24 +45,16 @@ def prid(self): else: return self._prid - async def _do_dump(self, msg, msg_flags=NLM_F_REQUEST | NLM_F_DUMP): - return await self.nlm_request( - msg, msg_type=self.prid, msg_flags=msg_flags - ) - - async def _do_request(self, msg, msg_flags=NLM_F_REQUEST | NLM_F_ACK): - return [x async for x in await self._do_dump(msg, msg_flags)] - - async def bind(self, proto, msg_class, groups=0, pid=None, **kwarg): + def bind(self, proto, msg_class, groups=0, pid=None, **kwarg): ''' Bind the socket and performs generic netlink proto lookup. The `proto` parameter is a string, like "TASKSTATS", `msg_class` is a class to parse messages with. ''' - await super().bind(groups, pid, **kwarg) + NetlinkSocket.bind(self, groups, pid, **kwarg) self.marshal.msg_map[GENL_ID_CTRL] = ctrlmsg - msg = await self.discovery(proto) + msg = self.discovery(proto) self._prid = msg.get_attr('CTRL_ATTR_FAMILY_ID') self.mcast_groups = dict( [ @@ -85,7 +77,7 @@ def drop_membership(self, group): SOL_NETLINK, NETLINK_DROP_MEMBERSHIP, self.mcast_groups[group] ) - async def discovery(self, proto): + def discovery(self, proto): ''' Resolve generic netlink protocol -- takes a string as the only parameter, return protocol description @@ -99,7 +91,7 @@ async def discovery(self, proto): msg['header']['pid'] = self.pid msg.encode() self.sendto(msg.data, (0, 0)) - (msg,) = [x async for x in self.get()] + msg = self.get()[0] err = msg['header'].get('error', None) if err is not None: if hasattr(err, 'code') and err.code == errno.ENOENT: @@ -112,7 +104,7 @@ async def discovery(self, proto): raise err return msg - async def policy(self, proto): + def policy(self, proto): ''' Extract policy information for a generic netlink protocol -- takes a string as the only parameter, return protocol policy @@ -121,44 +113,14 @@ async def policy(self, proto): msg = ctrlmsg() msg['cmd'] = CTRL_CMD_GETPOLICY msg['attrs'].append(['CTRL_ATTR_FAMILY_NAME', proto]) - return tuple( - [ - x - async for x in await self.nlm_request( - msg, - msg_type=GENL_ID_CTRL, - msg_flags=NLM_F_REQUEST | NLM_F_DUMP | NLM_F_ACK, - ) - ] - ) - - -class GenericNetlinkSocket(NetlinkSocket): - - async_class = AsyncGenericNetlinkSocket - class_gen_sync = False - - @property - def prid(self): - return self.asyncore.prid - - @property - def mcast_groups(self): - return self.asyncore.mcast_groups - - def bind(self, proto, msg_class, groups=0, pid=None, **kwarg): - return self._run_with_cleanup( - self.asyncore.bind, proto, msg_class, groups, pid, **kwarg + return self.nlm_request( + msg, + msg_type=GENL_ID_CTRL, + msg_flags=NLM_F_REQUEST | NLM_F_DUMP | NLM_F_ACK, ) - def add_membership(self, group): - return self.asyncore.add_membership(group) - - def drop_membership(self, group): - return self.asyncore.drop_membership(group) + def get(self, *argv, **kwarg): + return tuple(super().get(*argv, **kwarg)) - def discovery(self, proto): - return self._run_with_cleanup(self.asyncore.discovery, proto) - - def policy(self, proto): - return self._run_with_cleanup(self.asyncore.policy, proto) + def nlm_request(self, *argv, **kwarg): + return tuple(super().nlm_request(*argv, **kwarg)) diff --git a/pyroute2/netlink/generic/ethtool.py b/pyroute2/netlink/generic/ethtool.py index 1ed636388..4d6435a4e 100644 --- a/pyroute2/netlink/generic/ethtool.py +++ b/pyroute2/netlink/generic/ethtool.py @@ -6,10 +6,7 @@ nla, ) from pyroute2.netlink.exceptions import NetlinkError -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) +from pyroute2.netlink.generic import GenericNetlinkSocket ETHTOOL_GENL_NAME = "ethtool" ETHTOOL_GENL_VERSION = 1 @@ -188,15 +185,13 @@ class ethtool_rings_msg(genlmsg): ethtoolheader = ethtoolheader -class AsyncNlEthtool(AsyncGenericNetlinkSocket): - async def _do_request(self, msg, msg_flags=NLM_F_REQUEST): - return await self.nlm_request( - msg, msg_type=self.prid, msg_flags=msg_flags - ) +class NlEthtool(GenericNetlinkSocket): + def _do_request(self, msg, msg_flags=NLM_F_REQUEST): + return self.nlm_request(msg, msg_type=self.prid, msg_flags=msg_flags) - async def is_nlethtool_in_kernel(self): + def is_nlethtool_in_kernel(self): try: - await self.bind(ETHTOOL_GENL_NAME, ethtool_linkinfo_msg) + self.bind(ETHTOOL_GENL_NAME, ethtool_linkinfo_msg) except NetlinkError: return False return True @@ -209,7 +204,7 @@ def _get_dev_header(self, ifname=None, ifindex=None): else: raise ValueError("Need ifname or ifindex") - async def get_linkinfo(self, ifname=None, ifindex=None): + def get_linkinfo(self, ifname=None, ifindex=None): msg = ethtool_linkinfo_msg() msg["cmd"] = ETHTOOL_MSG_LINKINFO_GET msg['version'] = ETHTOOL_GENL_VERSION @@ -220,10 +215,10 @@ async def get_linkinfo(self, ifname=None, ifindex=None): ) ) - await self.bind(ETHTOOL_GENL_NAME, ethtool_linkinfo_msg) - return await self._do_request(msg) + self.bind(ETHTOOL_GENL_NAME, ethtool_linkinfo_msg) + return self._do_request(msg) - async def get_linkmode(self, ifname=None, ifindex=None): + def get_linkmode(self, ifname=None, ifindex=None): msg = ethtool_linkmode_msg() msg["cmd"] = ETHTOOL_MSG_LINKMODES_GET msg['version'] = ETHTOOL_GENL_VERSION @@ -234,10 +229,10 @@ async def get_linkmode(self, ifname=None, ifindex=None): ) ) - await self.bind(ETHTOOL_GENL_NAME, ethtool_linkmode_msg) - return await self._do_request(msg) + self.bind(ETHTOOL_GENL_NAME, ethtool_linkmode_msg) + return self._do_request(msg) - async def get_stringset(self, ifname=None, ifindex=None): + def get_stringset(self, ifname=None, ifindex=None): msg = ethtool_strset_msg() msg["cmd"] = ETHTOOL_MSG_STRSET_GET msg['version'] = ETHTOOL_GENL_VERSION @@ -245,10 +240,10 @@ async def get_stringset(self, ifname=None, ifindex=None): ('ETHTOOL_A_STRSET_HEADER', self._get_dev_header(ifname, ifindex)) ) - await self.bind(ETHTOOL_GENL_NAME, ethtool_strset_msg) - return await self._do_request(msg) + self.bind(ETHTOOL_GENL_NAME, ethtool_strset_msg) + return self._do_request(msg) - async def get_linkstate(self, ifname=None, ifindex=None): + def get_linkstate(self, ifname=None, ifindex=None): msg = ethtool_linkstate_msg() msg["cmd"] = ETHTOOL_MSG_LINKSTATE_GET msg['version'] = ETHTOOL_GENL_VERSION @@ -259,10 +254,10 @@ async def get_linkstate(self, ifname=None, ifindex=None): ) ) - await self.bind(ETHTOOL_GENL_NAME, ethtool_linkstate_msg) - return await self._do_request(msg) + self.bind(ETHTOOL_GENL_NAME, ethtool_linkstate_msg) + return self._do_request(msg) - async def get_wol(self, ifname=None, ifindex=None): + def get_wol(self, ifname=None, ifindex=None): msg = ethtool_wol_msg() msg["cmd"] = ETHTOOL_MSG_WOL_GET msg['version'] = ETHTOOL_GENL_VERSION @@ -270,10 +265,10 @@ async def get_wol(self, ifname=None, ifindex=None): ('ETHTOOL_A_WOL_HEADER', self._get_dev_header(ifname, ifindex)) ) - await self.bind(ETHTOOL_GENL_NAME, ethtool_wol_msg) - return await self._do_request(msg) + self.bind(ETHTOOL_GENL_NAME, ethtool_wol_msg) + return self._do_request(msg) - async def get_rings(self, ifname=None, ifindex=None): + def get_rings(self, ifname=None, ifindex=None): msg = ethtool_rings_msg() msg["cmd"] = ETHTOOL_MSG_RINGS_GET msg["version"] = ETHTOOL_GENL_VERSION @@ -281,56 +276,15 @@ async def get_rings(self, ifname=None, ifindex=None): ('ETHTOOL_A_RINGS_HEADER', self._get_dev_header(ifname, ifindex)) ) - await self.bind(ETHTOOL_GENL_NAME, ethtool_rings_msg) - return await self._do_request(msg) + self.bind(ETHTOOL_GENL_NAME, ethtool_rings_msg) + return self._do_request(msg) - async def set_rings(self, rings, ifname=None, ifindex=None): + def set_rings(self, rings, ifname=None, ifindex=None): rings["cmd"] = ETHTOOL_MSG_RINGS_SET rings["version"] = ETHTOOL_GENL_VERSION rings["attrs"].append( ('ETHTOOL_A_RINGS_HEADER', self._get_dev_header(ifname, ifindex)) ) - await self.bind(ETHTOOL_GENL_NAME, ethtool_rings_msg) - return await self._do_request( - rings, msg_flags=NLM_F_REQUEST | NLM_F_ACK - ) - - -class NlEthtool(GenericNetlinkSocket): - - async_class = AsyncNlEthtool - - def is_nlethtool_in_kernel(self): - return self._run_with_cleanup(self.asyncore.is_nlethtool_in_kernel) - - def get_linkinfo(self, ifname=None, ifindex=None): - return self._run_sync_cleanup( - self.asyncore.get_linkinfo, ifname, ifindex - ) - - def get_linkmode(self, ifname=None, ifindex=None): - return self._run_sync_cleanup( - self.asyncore.get_linkmode, ifname, ifindex - ) - - def get_stringset(self, ifname=None, ifindex=None): - return self._run_sync_cleanup( - self.asyncore.get_stringset, ifname, ifindex - ) - - def get_linkstate(self, ifname=None, ifindex=None): - return self._run_sync_cleanup( - self.asyncore.get_linkstate, ifname, ifindex - ) - - def get_wol(self, ifname=None, ifindex=None): - return self._run_sync_cleanup(self.asyncore.get_wol, ifname, ifindex) - - def get_rings(self, ifname=None, ifindex=None): - return self._run_sync_cleanup(self.asyncore.get_rings, ifname, ifindex) - - def set_rings(self, rings, ifname=None, ifindex=None): - return self._run_sync_cleanup( - self.asyncore.set_rings, rings, ifname, ifindex - ) + self.bind(ETHTOOL_GENL_NAME, ethtool_rings_msg) + return self._do_request(rings, msg_flags=NLM_F_REQUEST | NLM_F_ACK) diff --git a/pyroute2/netlink/generic/ipvs.py b/pyroute2/netlink/generic/ipvs.py deleted file mode 100644 index 6d5fa7ab9..000000000 --- a/pyroute2/netlink/generic/ipvs.py +++ /dev/null @@ -1,133 +0,0 @@ -from pyroute2.netlink import genlmsg, nla -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) - -GENL_NAME = "IPVS" -GENL_VERSION = 0x1 - -IPVS_CMD_UNSPEC = 0 - -IPVS_CMD_NEW_SERVICE = 1 -IPVS_CMD_SET_SERVICE = 2 -IPVS_CMD_DEL_SERVICE = 3 -IPVS_CMD_GET_SERVICE = 4 - -IPVS_CMD_NEW_DEST = 5 -IPVS_CMD_SET_DEST = 6 -IPVS_CMD_DEL_DEST = 7 -IPVS_CMD_GET_DEST = 8 - -IPVS_CMD_NEW_DAEMON = 9 -IPVS_CMD_DEL_DAEMON = 10 -IPVS_CMD_GET_DAEMON = 11 - -IPVS_CMD_SET_CONFIG = 12 -IPVS_CMD_GET_CONFIG = 13 - -IPVS_CMD_SET_INFO = 14 -IPVS_CMD_GET_INFO = 15 - -IPVS_CMD_ZERO = 16 -IPVS_CMD_FLUSH = 17 - - -class ipvsstats: - class stats(nla): - nla_map = ( - ("IPVS_STATS_ATTR_UNSPEC", "none"), - ("IPVS_STATS_ATTR_CONNS", "uint32"), - ("IPVS_STATS_ATTR_INPKTS", "uint32"), - ("IPVS_STATS_ATTR_OUTPKTS", "uint32"), - ("IPVS_STATS_ATTR_INBYTES", "uint64"), - ("IPVS_STATS_ATTR_OUTBYTES", "uint64"), - ("IPVS_STATS_ATTR_CPS", "uint32"), - ("IPVS_STATS_ATTR_INPPS", "uint32"), - ("IPVS_STATS_ATTR_OUTPPS", "uint32"), - ("IPVS_STATS_ATTR_INBPS", "uint32"), - ("IPVS_STATS_ATTR_OUTBPS", "uint32"), - ) - - class stats64(nla): - nla_map = ( - ("IPVS_STATS_ATTR_UNSPEC", "none"), - ("IPVS_STATS_ATTR_CONNS", "uint64"), - ("IPVS_STATS_ATTR_INPKTS", "uint64"), - ("IPVS_STATS_ATTR_OUTPKTS", "uint64"), - ("IPVS_STATS_ATTR_INBYTES", "uint64"), - ("IPVS_STATS_ATTR_OUTBYTES", "uint64"), - ("IPVS_STATS_ATTR_CPS", "uint64"), - ("IPVS_STATS_ATTR_INPPS", "uint64"), - ("IPVS_STATS_ATTR_OUTPPS", "uint64"), - ("IPVS_STATS_ATTR_INBPS", "uint64"), - ("IPVS_STATS_ATTR_OUTBPS", "uint64"), - ) - - -class ipvsmsg(genlmsg): - prefix = "IPVS_CMD_ATTR_" - nla_map = ( - ("IPVS_CMD_ATTR_UNSPEC", "none"), - ("IPVS_CMD_ATTR_SERVICE", "service"), - ("IPVS_CMD_ATTR_DEST", "dest"), - ("IPVS_CMD_ATTR_DAEMON", "hex"), - ("IPVS_CMD_ATTR_TIMEOUT_TCP", "hex"), - ("IPVS_CMD_ATTR_TIMEOUT_TCP_FIN", "hex"), - ("IPVS_CMD_ATTR_TIMEOUT_UDP", "hex"), - ) - - class service(nla, ipvsstats): - prefix = "IPVS_SVC_ATTR_" - nla_map = ( - ("IPVS_SVC_ATTR_UNSPEC", "none"), - ("IPVS_SVC_ATTR_AF", "uint16"), - ("IPVS_SVC_ATTR_PROTOCOL", "uint16"), - ("IPVS_SVC_ATTR_ADDR", "target(nla,IPVS_SVC_ATTR_AF)"), - ("IPVS_SVC_ATTR_PORT", "be16"), - ("IPVS_SVC_ATTR_FWMARK", "uint32"), - ("IPVS_SVC_ATTR_SCHED_NAME", "asciiz"), - ("IPVS_SVC_ATTR_FLAGS", "flags"), - ("IPVS_SVC_ATTR_TIMEOUT", "uint32"), - ("IPVS_SVC_ATTR_NETMASK", "ip4addr"), - ("IPVS_SVC_ATTR_STATS", "stats"), - ("IPVS_SVC_ATTR_PE_NAME", "asciiz"), - ("IPVS_SVC_ATTR_STATS64", "stats64"), - ) - - class flags(nla): - fields = (("flags", "I"), ("mask", "I")) - - class dest(nla, ipvsstats): - prefix = "IPVS_DEST_ATTR_" - nla_map = ( - ("IPVS_DEST_ATTR_UNSPEC", "none"), - ("IPVS_DEST_ATTR_ADDR", "target(nla,IPVS_DEST_ATTR_ADDR_FAMILY)"), - ("IPVS_DEST_ATTR_PORT", "be16"), - ("IPVS_DEST_ATTR_FWD_METHOD", "uint32"), - ("IPVS_DEST_ATTR_WEIGHT", "uint32"), - ("IPVS_DEST_ATTR_U_THRESH", "uint32"), - ("IPVS_DEST_ATTR_L_THRESH", "uint32"), - ("IPVS_DEST_ATTR_ACTIVE_CONNS", "uint32"), - ("IPVS_DEST_ATTR_INACT_CONNS", "uint32"), - ("IPVS_DEST_ATTR_PERSIST_CONNS", "uint32"), - ("IPVS_DEST_ATTR_STATS", "stats"), - ("IPVS_DEST_ATTR_ADDR_FAMILY", "uint16"), - ("IPVS_DEST_ATTR_STATS64", "stats64"), - ("IPVS_DEST_ATTR_TUN_TYPE", "uint8"), - ("IPVS_DEST_ATTR_TUN_PORT", "uint16"), - ("IPVS_DEST_ATTR_TUN_FLAGS", "uint16"), - ) - - -class AsyncIPVSSocket(AsyncGenericNetlinkSocket): - - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - await super().setup_endpoint() - await self.bind(GENL_NAME, ipvsmsg) - - -class IPVSSocket(GenericNetlinkSocket): - async_class = AsyncIPVSSocket diff --git a/pyroute2/netlink/generic/l2tp.py b/pyroute2/netlink/generic/l2tp.py index d906971d2..6d7b08b31 100644 --- a/pyroute2/netlink/generic/l2tp.py +++ b/pyroute2/netlink/generic/l2tp.py @@ -1,14 +1,12 @@ from pyroute2.netlink import ( NLA_F_NESTED, + NLM_F_ACK, NLM_F_DUMP, NLM_F_REQUEST, genlmsg, nla, ) -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) +from pyroute2.netlink.generic import GenericNetlinkSocket # Defines from uapi/linux/l2tp.h L2TP_GENL_NAME = "l2tp" @@ -146,15 +144,15 @@ class l2tp_stats(nla): ) -class AsyncL2tp(AsyncGenericNetlinkSocket): +class L2tp(GenericNetlinkSocket): + def __init__(self, *args, **kwargs): + GenericNetlinkSocket.__init__(self, *args, **kwargs) + self.bind(L2TP_GENL_NAME, l2tpmsg) - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - await super().setup_endpoint() - await self.bind(L2TP_GENL_NAME, l2tpmsg) + def _do_request(self, msg, msg_flags=NLM_F_REQUEST | NLM_F_ACK): + return self.nlm_request(msg, msg_type=self.prid, msg_flags=msg_flags) - async def _send_tunnel( + def _send_tunnel( self, cmd, tunnel_id, @@ -251,9 +249,9 @@ async def _send_tunnel( if debug is not None: msg["attrs"].append(["L2TP_ATTR_DEBUG", debug]) - return await self._do_request(msg) + return self._do_request(msg) - async def create_tunnel( + def create_tunnel( self, tunnel_id, peer_tunnel_id, @@ -308,7 +306,7 @@ async def create_tunnel( "selected" ) - return await self._send_tunnel( + return self._send_tunnel( cmd=L2TP_CMD_TUNNEL_CREATE, tunnel_id=tunnel_id, peer_tunnel_id=peer_tunnel_id, @@ -325,7 +323,7 @@ async def create_tunnel( debug=debug, ) - async def modify_tunnel(self, tunnel_id, debug): + def modify_tunnel(self, tunnel_id, debug): """ Modify an existing L2TP tunnel :param tunnel_id: local tunnel id @@ -333,11 +331,11 @@ async def modify_tunnel(self, tunnel_id, debug): tunnel :return: netlink response """ - return await self._send_tunnel( + return self._send_tunnel( L2TP_CMD_TUNNEL_MODIFY, tunnel_id=tunnel_id, debug=debug ) - async def delete_tunnel(self, tunnel_id): + def delete_tunnel(self, tunnel_id): """ Delete a tunnel :param tunnel_id: tunnel id of the tunnel to be deleted @@ -348,9 +346,9 @@ async def delete_tunnel(self, tunnel_id): msg["version"] = L2TP_GENL_VERSION msg["attrs"].append(["L2TP_ATTR_CONN_ID", tunnel_id]) - return await self._do_request(msg) + return self._do_request(msg) - async def _send_session( + def _send_session( self, cmd, tunnel_id, @@ -442,9 +440,9 @@ async def _send_session( if cmd == L2TP_CMD_SESSION_CREATE: msg["attrs"].append(["L2TP_ATTR_PW_TYPE", pwtype]) - return await self._do_request(msg) + return self._do_request(msg) - async def create_session( + def create_session( self, tunnel_id, session_id, @@ -477,7 +475,7 @@ async def create_session( :param pwtype: pseudowire type :return: netlink response """ - await self._send_session( + self._send_session( cmd=L2TP_CMD_SESSION_CREATE, tunnel_id=tunnel_id, session_id=session_id, @@ -493,7 +491,7 @@ async def create_session( pwtype=pwtype, ) - async def modify_session( + def modify_session( self, tunnel_id, session_id, @@ -513,7 +511,7 @@ async def modify_session( :param recv_timeout: Reorder timeout :return: netlink response """ - await self._send_session( + self._send_session( cmd=L2TP_CMD_SESSION_MODIFY, tunnel_id=tunnel_id, session_id=session_id, @@ -523,7 +521,7 @@ async def modify_session( recv_timeout=recv_timeout, ) - async def delete_session(self, tunnel_id, session_id): + def delete_session(self, tunnel_id, session_id): """ Delete a session :param tunnel_id: tunnel id in which the session to be deleted is @@ -537,9 +535,9 @@ async def delete_session(self, tunnel_id, session_id): msg["attrs"].append(["L2TP_ATTR_CONN_ID", tunnel_id]) msg["attrs"].append(["L2TP_ATTR_SESSION_ID", session_id]) - return await self._do_request(msg) + return self._do_request(msg) - async def get_tunnel(self, tunnel_id): + def get_tunnel(self, tunnel_id): """ Get one tunnel :param tunnel_id: tunnel id of the tunnel to show @@ -549,9 +547,9 @@ async def get_tunnel(self, tunnel_id): msg["cmd"] = L2TP_CMD_TUNNEL_GET msg["version"] = L2TP_GENL_VERSION msg["attrs"].append(["L2TP_ATTR_CONN_ID", tunnel_id]) - return (await self._do_request(msg, msg_flags=NLM_F_REQUEST))[0] + return self._do_request(msg, msg_flags=NLM_F_REQUEST)[0] - async def dump_tunnels(self, tunnel_id): + def dump_tunnels(self, tunnel_id): """ Dump all tunnels :return: netlink response @@ -559,9 +557,9 @@ async def dump_tunnels(self, tunnel_id): msg = l2tpmsg() msg["cmd"] = L2TP_CMD_TUNNEL_GET msg["version"] = L2TP_GENL_VERSION - return await self._do_dump(msg, msg_flags=NLM_F_REQUEST | NLM_F_DUMP) + return self._do_request(msg, msg_flags=NLM_F_REQUEST | NLM_F_DUMP) - async def get_session(self, tunnel_id, session_id): + def get_session(self, tunnel_id, session_id): """ Get one session :param tunnel_id: tunnel id of the session @@ -573,9 +571,10 @@ async def get_session(self, tunnel_id, session_id): msg["version"] = L2TP_GENL_VERSION msg["attrs"].append(["L2TP_ATTR_CONN_ID", tunnel_id]) msg["attrs"].append(["L2TP_ATTR_SESSION_ID", session_id]) - return (await self._do_request(msg, msg_flags=NLM_F_REQUEST))[0] - async def dump_sessions(self): + return self._do_request(msg, msg_flags=NLM_F_REQUEST)[0] + + def dump_sessions(self): """ Dump all sessions :return: netlink response @@ -583,118 +582,5 @@ async def dump_sessions(self): msg = l2tpmsg() msg["cmd"] = L2TP_CMD_SESSION_GET msg["version"] = L2TP_GENL_VERSION - return await self._do_dump(msg, msg_flags=NLM_F_REQUEST | NLM_F_DUMP) - - -class L2tp(GenericNetlinkSocket): - async_class = AsyncL2tp - - def create_tunnel( - self, - tunnel_id, - peer_tunnel_id, - protocol=3, - remote=None, - local=None, - fd=None, - encap="udp", - udp_sport=None, - udp_dport=None, - udp_csum=None, - udp6_csum_rx=None, - udp6_csum_tx=None, - debug=False, - ): - return self._run_with_cleanup( - self.asyncore.create_tunnel, - tunnel_id, - peer_tunnel_id, - protocol, - remote, - local, - fd, - encap, - udp_sport, - udp_dport, - udp_csum, - udp6_csum_rx, - udp6_csum_tx, - debug, - ) - - def modify_tunnel(self, tunnel_id, debug): - return self._run_with_cleanup( - self.asyncore.modify_tunnel, tunnel_id, debug - ) - - def delete_tunnel(self, tunnel_id): - return self._run_with_cleanup(self.asyncore.delete_tunnel, tunnel_id) - def create_session( - self, - tunnel_id, - session_id, - peer_session_id=None, - ifname=None, - l2spec_type=None, - cookie=None, - peer_cookie=None, - debug=None, - seq=None, - lns_mode=None, - recv_timeout=None, - pwtype=L2TP_PWTYPE_ETH, - ): - return self._run_with_cleanup( - self.asyncore.create_session, - tunnel_id, - session_id, - peer_session_id, - ifname, - l2spec_type, - cookie, - peer_cookie, - debug, - seq, - lns_mode, - recv_timeout, - pwtype, - ) - - def modify_session( - self, - tunnel_id, - session_id, - debug=None, - seq=None, - lns_mode=None, - recv_timeout=None, - ): - return self._run_with_cleanup( - self.asyncore.modify_session, - tunnel_id, - session_id, - debug, - seq, - lns_mode, - recv_timeout, - ) - - def delete_session(self, tunnel_id, session_id): - return self._run_with_cleanup( - self.asyncore.delete_session, tunnel_id, session_id - ) - - def get_tunnel(self, tunnel_id): - return self._run_with_cleanup(self.asyncore.get_tunnel, tunnel_id) - - def dump_tunnels(self): - return self._generate_with_cleanup(self.asyncore.dump_tunnels) - - def get_session(self, tunnel_id, session_id): - return self._run_with_cleanup( - self.asyncore.get_session, tunnel_id, session_id - ) - - def dump_sessions(self): - return self._generate_with_cleanup(self.asyncore.dump_sessions) + return self._do_request(msg, msg_flags=NLM_F_REQUEST | NLM_F_DUMP) diff --git a/pyroute2/netlink/generic/mptcp.py b/pyroute2/netlink/generic/mptcp.py index 2c57b37f9..d968796de 100644 --- a/pyroute2/netlink/generic/mptcp.py +++ b/pyroute2/netlink/generic/mptcp.py @@ -1,10 +1,7 @@ from socket import AF_INET, AF_INET6 from pyroute2.netlink import NLM_F_ACK, NLM_F_DUMP, NLM_F_REQUEST, genlmsg, nla -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) +from pyroute2.netlink.generic import GenericNetlinkSocket MPTCP_GENL_NAME = 'mptcp_pm' @@ -41,15 +38,16 @@ class pm_addr(nla): ) -class AsyncMPTCP(AsyncGenericNetlinkSocket): - - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - await super().setup_endpoint() - await self.bind(MPTCP_GENL_NAME, mptcp_msg) +class MPTCP(GenericNetlinkSocket): + def __init__(self, ext_ack=True): + super(MPTCP, self).__init__(ext_ack=ext_ack) + try: + self.bind(MPTCP_GENL_NAME, mptcp_msg) + except Exception as e: + self.close() + raise e - async def endpoint(self, cmd, **kwarg): + def endpoint(self, cmd, **kwarg): ''' Usage:: @@ -89,9 +87,9 @@ async def endpoint(self, cmd, **kwarg): ) msg['attrs'] = [('MPTCP_PM_ATTR_ADDR', addr_info, 0x8000)] - return await self.nlm_request(msg, msg_type=self.prid, msg_flags=flags) + return self.nlm_request(msg, msg_type=self.prid, msg_flags=flags) - async def limits(self, cmd, **kwarg): + def limits(self, cmd, **kwarg): ''' Usage:: @@ -120,15 +118,4 @@ async def limits(self, cmd, **kwarg): key = 'rcv_add_addrs' msg['attrs'].append((mptcp_msg.name2nla(key), value)) - return await self.nlm_request(msg, msg_type=self.prid, msg_flags=flags) - - -class MPTCP(GenericNetlinkSocket): - - async_class = AsyncMPTCP - - def endpoint(self, cmd, **kwarg): - return self._run_sync_cleanup(self.asyncore.endpoint, cmd, **kwarg) - - def limits(self, cmd, **kwarg): - return self._run_sync_cleanup(self.asyncore.limits, cmd, **kwarg) + return self.nlm_request(msg, msg_type=self.prid, msg_flags=flags) diff --git a/pyroute2/netlink/generic/wireguard.py b/pyroute2/netlink/generic/wireguard.py index 432dfef1c..7575f9f43 100644 --- a/pyroute2/netlink/generic/wireguard.py +++ b/pyroute2/netlink/generic/wireguard.py @@ -55,12 +55,10 @@ struct peer_s { public_key: # Base64 public key - required remove: # Boolean - optional - update_only: # Boolean - optional preshared_key: # Base64 preshared key - optional endpoint_addr: # IPv4 or IPv6 endpoint - optional endpoint_port : # endpoint Port - required only if endpoint_addr persistent_keepalive: # time in seconds to send keep alive - optional - replace_allowed_ips: # Boolean - optional allowed_ips: # list of CIDRs allowed - optional } ''' @@ -73,18 +71,13 @@ from struct import pack, unpack from time import ctime -from pyroute2.netlink import ( - NLA_F_NESTED, - NLM_F_ACK, - NLM_F_DUMP, - NLM_F_REQUEST, - genlmsg, - nla, -) -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) +from pr2modules.netlink import genlmsg +from pr2modules.netlink import nla +from pr2modules.netlink import NLM_F_ACK +from pr2modules.netlink import NLM_F_DUMP +from pr2modules.netlink import NLA_F_NESTED +from pr2modules.netlink import NLM_F_REQUEST +from pr2modules.netlink.generic import GenericNetlinkSocket # Defines from uapi/wireguard.h WG_GENL_NAME = "wireguard" @@ -107,7 +100,7 @@ WGDEVICE_A_PEERS = 8 # WireGuard Device flags -WGDEVICE_F_REPLACE_PEERS = 0x1 +WGDEVICE_F_REPLACE_PEERS = 1 # WireGuard Allowed IP attributes WGALLOWEDIP_A_UNSPEC = 0 @@ -116,9 +109,9 @@ WGALLOWEDIP_A_CIDR_MASK = 3 # WireGuard Peer flags -WGPEER_F_REMOVE_ME = 0x1 -WGPEER_F_REPLACE_ALLOWEDIPS = 0x2 -WGPEER_F_UPDATE_ONLY = 0x4 +WGPEER_F_REMOVE_ME = 0 +WGPEER_F_REPLACE_ALLOWEDIPS = 1 +WGPEER_F_UPDATE_ONLY = 2 # Specific defines WG_MAX_PEERS = 1000 @@ -238,15 +231,12 @@ def encode(self): nla.encode(self) -class AsyncWireGuard(AsyncGenericNetlinkSocket): - - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - await super().setup_endpoint() - await self.bind(WG_GENL_NAME, wgmsg) +class WireGuard(GenericNetlinkSocket): + def __init__(self, *args, **kwargs): + GenericNetlinkSocket.__init__(self, *args, **kwargs) + self.bind(WG_GENL_NAME, wgmsg) - async def info(self, interface=None, ifindex=None): + def info(self, interface=None, ifindex=None): msg = wgmsg() msg['cmd'] = WG_CMD_GET_DEVICE @@ -257,18 +247,18 @@ async def info(self, interface=None, ifindex=None): else: raise ValueError("ifname or ifindex are unset") - return await self.nlm_request( - msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + return self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_DUMP ) - async def set( - self, - interface=None, - ifindex=None, - listen_port=None, - fwmark=None, - private_key=None, - peer=None, + def set( + self, + interface=None, + ifindex=None, + listen_port=None, + fwmark=None, + private_key=None, + peer=None ): msg = wgmsg() @@ -300,7 +290,7 @@ async def set( msg['header']['pid'] = self.pid msg.encode() self.sendto(msg.data, (0, 0)) - (msg,) = [x async for x in self.get()] + msg = self.get()[0] err = msg['header'].get('error', None) if err is not None: if hasattr(err, 'code') and err.code == errno.ENOENT: @@ -331,7 +321,7 @@ def _wg_set_peer(self, msg, peer): # If peer removal is set to True if 'remove' in peer and peer['remove']: - attrs.append(['WGPEER_A_FLAGS', WGPEER_F_REMOVE_ME]) + attrs.append(['WGPEER_A_FLAGS', WGDEVICE_F_REPLACE_PEERS]) msg['attrs'].append(['WGDEVICE_A_PEERS', wg_peer]) return @@ -358,12 +348,7 @@ def _wg_set_peer(self, msg, peer): attrs.append(['WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL', keepalive]) # Set Peer flags - flags = 0 - if 'update_only' in peer and peer['update_only']: - flags |= WGPEER_F_UPDATE_ONLY - if 'replace_allowed_ips' in peer and peer['replace_allowed_ips']: - flags |= WGPEER_F_REPLACE_ALLOWEDIPS - attrs.append(['WGPEER_A_FLAGS', flags]) + attrs.append(['WGPEER_A_FLAGS', WGPEER_F_UPDATE_ONLY]) # Set allowed IPs if 'allowed_ips' in peer: @@ -395,29 +380,3 @@ def _wg_build_allowedips(self, allowed_ips): allowed_ip.append(['WGALLOWEDIP_A_CIDR_MASK', int(mask)]) return ret - - -class WireGuard(GenericNetlinkSocket): - async_class = AsyncWireGuard - - def info(self, interface=None, ifindex=None): - return self._run_sync_cleanup(self.asyncore.info, interface, ifindex) - - def set( - self, - interface=None, - ifindex=None, - listen_port=None, - fwmark=None, - private_key=None, - peer=None, - ): - return self._run_with_cleanup( - self.asyncore.set, - interface, - ifindex, - listen_port, - fwmark, - private_key, - peer, - ) diff --git a/pyroute2/netlink/marshal.py b/pyroute2/netlink/marshal.py deleted file mode 100644 index 1a4b11dbc..000000000 --- a/pyroute2/netlink/marshal.py +++ /dev/null @@ -1,144 +0,0 @@ -import errno -import struct -import threading -from functools import partial -from typing import Callable, Generator, Optional - -from pyroute2.netlink import ( - NLM_F_ACK_TLVS, - NLMSG_DONE, - NLMSG_ERROR, - mtypes, - nlmsg, - nlmsgerr, -) -from pyroute2.netlink.exceptions import ( - NetlinkDecodeError, - NetlinkError, - NetlinkHeaderDecodeError, -) - - -class Marshal: - ''' - Generic marshalling class - ''' - - msg_map = {} - seq_map = None - key_offset = None - key_format = None - key_mask = None - debug = False - default_message_class = nlmsg - error_type = NLMSG_ERROR - - def __init__(self): - self.lock = threading.Lock() - self.msg_map = self.msg_map.copy() - self.seq_map = {} - self.defragmentation = {} - - def parse_one_message( - self, key, flags, sequence_number, data, offset, length - ): - msg = None - error = None - msg_class = self.msg_map.get(key, self.default_message_class) - # ignore length for a while - # get the message - if (key == self.error_type) or ( - key == NLMSG_DONE and flags & NLM_F_ACK_TLVS - ): - msg = nlmsgerr(data, offset=offset) - else: - msg = msg_class(data, offset=offset) - - try: - msg.decode() - except NetlinkHeaderDecodeError as e: - msg = nlmsg() - msg['header']['error'] = e - except NetlinkDecodeError as e: - msg['header']['error'] = e - - if isinstance(msg, nlmsgerr) and msg['error'] != 0: - code = abs(msg['error']) - if code == errno.ENOBUFS: - error = OSError(code, msg.get_attr('NLMSGERR_ATTR_MSG')) - else: - error = NetlinkError(code, msg.get_attr('NLMSGERR_ATTR_MSG')) - enc_type = struct.unpack_from('H', data, offset + 24)[0] - enc_class = self.msg_map.get(enc_type, nlmsg) - enc = enc_class(data, offset=offset + 20) - enc.decode() - msg['header']['errmsg'] = enc - - msg['header']['error'] = error - return msg - - def get_parser(self, key, flags, sequence_number): - return self.seq_map.get( - sequence_number, - partial(self.parse_one_message, key, flags, sequence_number), - ) - - def parse( - self, - data: bytes, - seq: Optional[int] = None, - callback: Optional[Callable] = None, - skip_alien_seq: bool = False, - ) -> Generator[nlmsg, None, None]: - ''' - Parse string data. - - At this moment all transport, except of the native - Netlink is deprecated in this library, so we should - not support any defragmentation on that level - ''' - offset = 0 - # there must be at least one header in the buffer, - # 'IHHII' == 16 bytes - while offset <= len(data) - 16: - # pick type and length - (length, key, flags, sequence_number) = struct.unpack_from( - 'IHHI', data, offset - ) - if skip_alien_seq and sequence_number != seq: - continue - if not 0 < length <= len(data): - break - # support custom parser keys - # see also: pyroute2.netlink.diag.MarshalDiag - if self.key_format is not None: - (key,) = struct.unpack_from( - self.key_format, data, offset + self.key_offset - ) - if self.key_mask is not None: - key &= self.key_mask - - parser = self.get_parser(key, flags, sequence_number) - msg = parser(data, offset, length) - offset += length - if msg is None: - continue - - if callable(callback) and seq == sequence_number: - try: - if callback(msg): - continue - except Exception: - pass - - mtype = msg['header'].get('type', None) - if mtype in (1, 2, 3, 4) and 'event' not in msg: - msg['event'] = mtypes.get(mtype, 'none') - self.fix_message(msg) - yield msg - - def is_enough(self, msg): - return msg['header']['type'] == NLMSG_DONE - - def fix_message(self, msg): - pass diff --git a/pyroute2/netlink/nfnetlink/ipset.py b/pyroute2/netlink/nfnetlink/ipset.py index 04788dab6..7e7f30fb1 100644 --- a/pyroute2/netlink/nfnetlink/ipset.py +++ b/pyroute2/netlink/nfnetlink/ipset.py @@ -86,7 +86,7 @@ class ipset_msg(nfgen_msg): ('IPSET_ATTR_FLAGS', 'be32'), ('IPSET_ATTR_DATA', 'get_data_type'), ('IPSET_ATTR_ADT', 'attr_adt'), - ('IPSET_ATTR_LINENO', 'be32'), + ('IPSET_ATTR_LINENO', 'hex'), ('IPSET_ATTR_PROTOCOL_MIN', 'uint8'), ('IPSET_ATTR_INDEX', 'be16'), ) diff --git a/pyroute2/netlink/nfnetlink/nfctsocket.py b/pyroute2/netlink/nfnetlink/nfctsocket.py index abd6e7621..f9cfbc3d7 100644 --- a/pyroute2/netlink/nfnetlink/nfctsocket.py +++ b/pyroute2/netlink/nfnetlink/nfctsocket.py @@ -17,9 +17,8 @@ NLMSG_ERROR, nla, ) -from pyroute2.netlink.core import SyncAPI from pyroute2.netlink.nfnetlink import NFNL_SUBSYS_CTNETLINK, nfgen_msg -from pyroute2.netlink.nlsocket import AsyncNetlinkSocket +from pyroute2.netlink.nlsocket import NetlinkSocket IPCTNL_MSG_CT_NEW = 0 IPCTNL_MSG_CT_GET = 1 @@ -198,7 +197,6 @@ def terminate_error_msg(msg): class nfct_stats(nfgen_msg): - prefix = 'CTA_STATS_' nla_map = ( ('CTA_STATS_GLOBAL_UNSPEC', 'none'), ('CTA_STATS_GLOBAL_ENTRIES', 'be32'), @@ -207,7 +205,6 @@ class nfct_stats(nfgen_msg): class nfct_stats_cpu(nfgen_msg): - prefix = 'CTA_STATS_' nla_map = ( ('CTA_STATS_UNSPEC', 'none'), ('CTA_STATS_SEARCHED', 'be32'), @@ -255,7 +252,6 @@ class nfct_msg(nfgen_msg): ('CTA_LABELS_MASK', 'cta_labels'), ('CTA_SYNPROXY', 'cta_synproxy'), ('CTA_FILTER', 'cta_filter'), - ('CTA_STATUS_MASK', 'be32'), ) @classmethod @@ -271,7 +267,6 @@ def create_from(cls, **kwargs): return self class cta_tuple(nla): - prefix = 'CTA_TUPLE_' nla_map = ( ('CTA_TUPLE_UNSPEC', 'none'), ('CTA_TUPLE_IP', 'cta_ip'), @@ -279,7 +274,6 @@ class cta_tuple(nla): ) class cta_ip(nla): - prefix = 'CTA_IP_' nla_map = ( ('CTA_IP_UNSPEC', 'none'), ('CTA_IP_V4_SRC', 'ip4addr'), @@ -289,7 +283,6 @@ class cta_ip(nla): ) class cta_proto(nla): - prefix = 'CTA_PROTO_' nla_map = ( ('CTA_PROTO_UNSPEC', 'none'), ('CTA_PROTO_NUM', 'uint8'), @@ -715,7 +708,7 @@ def __repr__(self): return r + '))' -class AsyncNFCTSocket(AsyncNetlinkSocket): +class NFCTSocket(NetlinkSocket): policy = { k | (NFNL_SUBSYS_CTNETLINK << 8): v for k, v in { @@ -731,23 +724,21 @@ class AsyncNFCTSocket(AsyncNetlinkSocket): } def __init__(self, nfgen_family=socket.AF_INET, **kwargs): - super().__init__(family=NETLINK_NETFILTER, **kwargs) + super(NFCTSocket, self).__init__(family=NETLINK_NETFILTER, **kwargs) self.register_policy(self.policy) self._nfgen_family = nfgen_family - async def request(self, msg, msg_type, **kwargs): + def request(self, msg, msg_type, **kwargs): msg['nfgen_family'] = self._nfgen_family msg_type |= NFNL_SUBSYS_CTNETLINK << 8 - return await self.nlm_request(msg, msg_type, **kwargs) + return tuple(self.nlm_request(msg, msg_type, **kwargs)) - async def dump( + def dump( self, mark=None, mark_mask=0xFFFFFFFF, tuple_orig=None, tuple_reply=None, - status=None, - status_mask=None, ): """Dump conntrack entries @@ -755,7 +746,6 @@ async def dump( * mark and mark_mask, for almost all kernel * tuple_orig and tuple_reply, since kernel 5.8 and newer. Warning: tuple_reply has a bug in kernel, fixed only recently. - * status and status_mask, available since kernel 5.15 tuple_orig and tuple_reply are type NFCTAttrTuple. You can give only some attribute for filtering. @@ -769,9 +759,6 @@ async def dump( filter = NFCTAttrTuple(proto=socket.IPPROTO_TCP, dport=443) ct.dump_entries(tuple_orig=filter) - # Get only one way connection (no reply) - ct.dump_entries(status=0, status_mask=IPS_SEEN_REPLY) - Note that NFCTAttrTuple attributes are working like one AND operator. Example:: @@ -797,65 +784,47 @@ async def dump( msg = nfct_msg.create_from( tuple_reply=tuple_reply, cta_filter=cta_filter ) + elif mark: + msg = nfct_msg.create_from(mark=mark, mark_mask=mark_mask) else: - kwargs = {} - if mark: - kwargs['mark'] = mark - kwargs['mark_mask'] = mark_mask - if status is not None: - kwargs['status'] = status - if status_mask is not None: - kwargs['status_mask'] = status_mask - msg = nfct_msg.create_from(**kwargs) - return await self.request( + msg = nfct_msg.create_from() + return self.request( msg, IPCTNL_MSG_CT_GET, msg_flags=NLM_F_REQUEST | NLM_F_DUMP ) - async def stat(self): - return [ - x - async for x in await self.request( - nfct_msg(), - IPCTNL_MSG_CT_GET_STATS_CPU, - msg_flags=NLM_F_REQUEST | NLM_F_DUMP, - ) - ] - - async def count(self): - return [ - x - async for x in await self.request( - nfct_msg(), - IPCTNL_MSG_CT_GET_STATS, - msg_flags=NLM_F_REQUEST | NLM_F_DUMP, - terminate=terminate_single_msg, - ) - ] + def stat(self): + return self.request( + nfct_msg(), + IPCTNL_MSG_CT_GET_STATS_CPU, + msg_flags=NLM_F_REQUEST | NLM_F_DUMP, + ) - async def flush(self, mark=None, mark_mask=None): + def count(self): + return self.request( + nfct_msg(), + IPCTNL_MSG_CT_GET_STATS, + msg_flags=NLM_F_REQUEST | NLM_F_DUMP, + terminate=terminate_single_msg, + ) + + def flush(self, mark=None, mark_mask=None): msg = nfct_msg.create_from(mark=mark, mark_mask=mark_mask) - return [ - x - async for x in await self.request( - msg, - IPCTNL_MSG_CT_DELETE, - msg_flags=NLM_F_REQUEST | NLM_F_ACK, - terminate=terminate_error_msg, - ) - ] - - async def conntrack_max_size(self): - return [ - x - async for x in await self.request( - nfct_msg(), - IPCTNL_MSG_CT_GET_STATS, - msg_flags=NLM_F_REQUEST | NLM_F_DUMP, - terminate=terminate_single_msg, - ) - ] + return self.request( + msg, + IPCTNL_MSG_CT_DELETE, + msg_flags=NLM_F_REQUEST | NLM_F_ACK, + terminate=terminate_error_msg, + ) + + def conntrack_max_size(self): + return self.request( + nfct_msg(), + IPCTNL_MSG_CT_GET_STATS, + msg_flags=NLM_F_REQUEST | NLM_F_DUMP, + terminate=terminate_single_msg, + ) - async def entry(self, cmd, **kwargs): + def entry(self, cmd, **kwargs): """ Get or change a conntrack entry. @@ -899,56 +868,9 @@ async def entry(self, cmd, **kwargs): ): raise ValueError('Deletion requires a tuple at least') - return [ - x - async for x in await self.request( - nfct_msg.create_from(**kwargs), - msg_type, - msg_flags=NLM_F_REQUEST | msg_flags, - terminate=terminate_error_msg, - ) - ] - - -class NFCTSocket(SyncAPI): - def __init__(self, nfgen_family=socket.AF_INET, **kwargs): - self.asyncore = AsyncNFCTSocket(nfgen_family, **kwargs) - - def request(self, msg, msg_type, **kwargs): - return self._run_sync_cleanup( - self.asyncore.request, msg, msg_type, **kwargs - ) - - def dump( - self, - mark=None, - mark_mask=0xFFFFFFFF, - tuple_orig=None, - tuple_reply=None, - status=None, - status_mask=None, - ): - return self._generate_with_cleanup( - self.asyncore.dump, - mark, - mark_mask, - tuple_orig, - tuple_reply, - status, - status_mask, + return self.request( + nfct_msg.create_from(**kwargs), + msg_type, + msg_flags=NLM_F_REQUEST | msg_flags, + terminate=terminate_error_msg, ) - - def stat(self): - return self._run_with_cleanup(self.asyncore.stat) - - def count(self): - return self._run_with_cleanup(self.asyncore.count) - - def flush(self, mark=None, mark_mask=None): - return self._run_with_cleanup(self.asyncore.flush, mark, mark_mask) - - def conntrack_max_size(self): - return self._run_with_cleanup(self.asyncore.conntrack_max_size) - - def entry(self, cmd, **kwarg): - return self._run_with_cleanup(self.asyncore.entry, cmd, **kwarg) diff --git a/pyroute2/netlink/nfnetlink/nftsocket.py b/pyroute2/netlink/nfnetlink/nftsocket.py index 15fe05001..898ae1d0d 100644 --- a/pyroute2/netlink/nfnetlink/nftsocket.py +++ b/pyroute2/netlink/nfnetlink/nftsocket.py @@ -4,7 +4,6 @@ See also: pyroute2.nftables """ -import enum import struct import threading @@ -22,7 +21,7 @@ nlmsg_atoms, ) from pyroute2.netlink.nfnetlink import NFNL_SUBSYS_NFTABLES, nfgen_msg -from pyroute2.netlink.nlsocket import AsyncNetlinkSocket, NetlinkSocket +from pyroute2.netlink.nlsocket import NetlinkSocket NFT_MSG_NEWTABLE = 0 NFT_MSG_GETTABLE = 1 @@ -108,77 +107,6 @@ NFPROTO_IPV6 = 10 -class Cmp(enum.IntEnum): - NFT_CMP_EQ: int = 0 - NFT_CMP_NEQ: int = 1 - NFT_CMP_LT: int = 2 - NFT_CMP_LTE: int = 3 - NFT_CMP_GT: int = 4 - NFT_CMP_GTE: int = 5 - - -class Meta(enum.IntEnum): - NFT_META_LEN: int = 0 - NFT_META_PROTOCOL: int = 1 - NFT_META_PRIORITY: int = 2 - NFT_META_MARK: int = 3 - NFT_META_IIF: int = 4 - NFT_META_OIF: int = 5 - NFT_META_IIFNAME: int = 6 - NFT_META_OIFNAME: int = 7 - NFT_META_IIFTYPE: int = 8 - NFT_META_OIFTYPE: int = 9 - NFT_META_SKUID: int = 10 - NFT_META_SKGID: int = 11 - NFT_META_NFTRACE: int = 12 - NFT_META_RTCLASSID: int = 13 - NFT_META_SECMARK: int = 14 - NFT_META_NFPROTO: int = 15 - NFT_META_L4PROTO: int = 16 - NFT_META_BRI_IIFNAME: int = 17 - NFT_META_BRI_OIFNAME: int = 18 - NFT_META_PKTTYPE: int = 19 - NFT_META_CPU: int = 20 - NFT_META_IIFGROUP: int = 21 - NFT_META_OIFGROUP: int = 22 - NFT_META_CGROUP: int = 23 - NFT_META_PRANDOM: int = 24 - NFT_META_SECPATH: int = 25 - NFT_META_IIFKIND: int = 26 - NFT_META_OIFKIND: int = 27 - NFT_META_BRI_IIFPVID: int = 28 - NFT_META_BRI_IIFVPROTO: int = 29 - NFT_META_TIME_NS: int = 30 - NFT_META_TIME_DAY: int = 31 - NFT_META_TIME_HOUR: int = 32 - NFT_META_SDIF: int = 33 - NFT_META_SDIFNAME: int = 34 - - -class Regs(enum.IntEnum): - NFT_REG_VERDICT: int = 0x00 - NFT_REG_1: int = 0x01 - NFT_REG_2: int = 0x02 - NFT_REG_3: int = 0x03 - NFT_REG_4: int = 0x04 - NFT_REG32_00: int = 0x08 - NFT_REG32_01: int = 0x09 - NFT_REG32_02: int = 0x0A - NFT_REG32_03: int = 0x0B - NFT_REG32_04: int = 0x0C - NFT_REG32_05: int = 0x0D - NFT_REG32_06: int = 0x0E - NFT_REG32_07: int = 0x0F - NFT_REG32_08: int = 0x10 - NFT_REG32_09: int = 0x11 - NFT_REG32_10: int = 0x12 - NFT_REG32_11: int = 0x13 - NFT_REG32_12: int = 0x14 - NFT_REG32_13: int = 0x15 - NFT_REG32_14: int = 0x16 - NFT_REG32_15: int = 0x17 - - class nftnl_udata(nla_base_string): # TLV structures: # nftnl_udata @@ -300,29 +228,7 @@ class nft_gen_msg(nfgen_msg): ) -class CData: - @staticmethod - def decode_from(data, offset=0): - (length,) = struct.unpack_from('>H', data, offset) - (data,) = struct.unpack_from(f'{length - 1}s', data, offset + 2) - return (data.decode('utf-8'), offset + 2 + length) - - @staticmethod - def encode_into(data, offset, value): - encoded = value.encode('utf-8') - length = len(encoded) + 1 - data.extend([0] * (length + 2)) - struct.pack_into(f'>H{length}s', data, offset, length, encoded) - return offset + length + 2 - - -class nft_has_comment: - - class comment(nla): - fields = [('value', CData)] - - -class nft_chain_msg(nfgen_msg, nft_has_comment): +class nft_chain_msg(nfgen_msg): prefix = 'NFTA_CHAIN_' nla_map = ( ('NFTA_CHAIN_UNSPEC', 'none'), @@ -337,7 +243,7 @@ class nft_chain_msg(nfgen_msg, nft_has_comment): ('NFTA_CHAIN_PAD', 'hex'), ('NFTA_CHAIN_FLAGS', 'flags'), ('NFTA_CHAIN_ID', 'be32'), - ('NFTA_CHAIN_USERDATA', 'comment'), + ('NFTA_CHAIN_USERDATA', 'hex'), ) class counters(nla): @@ -375,7 +281,29 @@ class nat_range(nft_flags_be32): class nft_regs(nla): class regs(nft_map_be32): - ops = {x.value: x.name for x in Regs} + ops = { + 0x00: 'NFT_REG_VERDICT', + 0x01: 'NFT_REG_1', + 0x02: 'NFT_REG_2', + 0x03: 'NFT_REG_3', + 0x04: 'NFT_REG_4', + 0x08: 'NFT_REG32_00', + 0x09: 'NFT_REG32_01', + 0x0A: 'NFT_REG32_02', + 0x0B: 'NFT_REG32_03', + 0x0C: 'NFT_REG32_04', + 0x0D: 'NFT_REG32_05', + 0x0E: 'NFT_REG32_06', + 0x0F: 'NFT_REG32_07', + 0x10: 'NFT_REG32_08', + 0x11: 'NFT_REG32_09', + 0x12: 'NFT_REG32_10', + 0x13: 'NFT_REG32_11', + 0x14: 'NFT_REG32_12', + 0x15: 'NFT_REG32_13', + 0x16: 'NFT_REG32_14', + 0x17: 'NFT_REG32_15', + } class nft_data(nla): @@ -461,7 +389,14 @@ class nft_cmp(nft_data, nft_regs): ) class ops(nft_map_be32): - ops = {x.value: x.name for x in Cmp} + ops = { + 0: 'NFT_CMP_EQ', + 1: 'NFT_CMP_NEQ', + 2: 'NFT_CMP_LT', + 3: 'NFT_CMP_LTE', + 4: 'NFT_CMP_GT', + 5: 'NFT_CMP_GTE', + } class nft_match(nla): nla_map = ( @@ -701,7 +636,43 @@ class nft_meta(nft_regs): ) class meta_key(nft_map_be32): - ops = {x.value: x.name for x in Meta} + ops = { + 0: 'NFT_META_LEN', + 1: 'NFT_META_PROTOCOL', + 2: 'NFT_META_PRIORITY', + 3: 'NFT_META_MARK', + 4: 'NFT_META_IIF', + 5: 'NFT_META_OIF', + 6: 'NFT_META_IIFNAME', + 7: 'NFT_META_OIFNAME', + 8: 'NFT_META_IIFTYPE', + 9: 'NFT_META_OIFTYPE', + 10: 'NFT_META_SKUID', + 11: 'NFT_META_SKGID', + 12: 'NFT_META_NFTRACE', + 13: 'NFT_META_RTCLASSID', + 14: 'NFT_META_SECMARK', + 15: 'NFT_META_NFPROTO', + 16: 'NFT_META_L4PROTO', + 17: 'NFT_META_BRI_IIFNAME', + 18: 'NFT_META_BRI_OIFNAME', + 19: 'NFT_META_PKTTYPE', + 20: 'NFT_META_CPU', + 21: 'NFT_META_IIFGROUP', + 22: 'NFT_META_OIFGROUP', + 23: 'NFT_META_CGROUP', + 24: 'NFT_META_PRANDOM', + 25: 'NFT_META_SECPATH', + 26: 'NFT_META_IIFKIND', + 27: 'NFT_META_OIFKIND', + 28: 'NFT_META_BRI_IIFPVID', + 29: 'NFT_META_BRI_IIFVPROTO', + 30: 'NFT_META_TIME_NS', + 31: 'NFT_META_TIME_DAY', + 32: 'NFT_META_TIME_HOUR', + 33: 'NFT_META_SDIF', + 34: 'NFT_META_SDIFNAME', + } class nft_nat(nft_regs, nat_flags): nla_map = ( @@ -970,7 +941,7 @@ def expr(self, *argv, **kwarg): return expr -class nft_rule_msg(nfgen_msg, nft_contains_expr, nft_has_comment): +class nft_rule_msg(nfgen_msg, nft_contains_expr): prefix = 'NFTA_RULE_' nla_map = ( ('NFTA_RULE_UNSPEC', 'none'), @@ -980,7 +951,7 @@ class nft_rule_msg(nfgen_msg, nft_contains_expr, nft_has_comment): ('NFTA_RULE_EXPRESSIONS', '*nft_expr'), ('NFTA_RULE_COMPAT', 'hex'), ('NFTA_RULE_POSITION', 'be64'), - ('NFTA_RULE_USERDATA', 'comment'), + ('NFTA_RULE_USERDATA', 'hex'), ('NFTA_RULE_PAD', 'hex'), ('NFTA_RULE_ID', 'be32'), ('NFTA_RULE_POSITION_ID', 'be32'), @@ -1059,7 +1030,7 @@ class set_field_attribute(nla): ) -class nft_table_msg(nfgen_msg, nft_contains_expr, nft_has_comment): +class nft_table_msg(nfgen_msg, nft_contains_expr): prefix = 'NFTA_TABLE_' nla_map = ( ('NFTA_TABLE_UNSPEC', 'none'), @@ -1068,7 +1039,7 @@ class nft_table_msg(nfgen_msg, nft_contains_expr, nft_has_comment): ('NFTA_TABLE_USE', 'be32'), ('NFTA_TABLE_HANDLE', 'be64'), ('NFTA_TABLE_PAD', 'hex'), - ('NFTA_TABLE_USERDATA', 'comment'), + ('NFTA_TABLE_USERDATA', 'hex'), ) @@ -1164,48 +1135,6 @@ class flowtable_hook(nft_device): class NFTSocket(NetlinkSocket): - def __init__(self, version=1, attr_revision=0, nfgen_family=2): - self.asyncore = AsyncNFTSocket(version, attr_revision, nfgen_family) - self.asyncore.local.keep_event_loop = True - self.asyncore.event_loop.run_until_complete( - self.asyncore.setup_endpoint() - ) - - def begin(self): - return self.asyncore.begin() - - def commit(self): - return self.asyncore.commit() - - def request_get( - self, - msg, - msg_type, - msg_flags=NLM_F_REQUEST | NLM_F_DUMP, - terminate=None, - ): - async def collect_data(): - return [ - x - async for x in await self.asyncore.request_get( - msg, msg_type, msg_flags, terminate - ) - ] - - return self._run_with_cleanup(collect_data) - - def request_put(self, msg, msg_type, msg_flags=NLM_F_REQUEST): - return self._run_with_cleanup( - self.asyncore.request_put, msg, msg_type, msg_flags - ) - - def _command(self, msg_class, commands, cmd, kwarg): - return self._run_with_cleanup( - self.asyncore._command, msg_class, commands, cmd, kwarg - ) - - -class AsyncNFTSocket(AsyncNetlinkSocket): ''' NFNetlink socket (family=NETLINK_NETFILTER). @@ -1236,7 +1165,7 @@ class AsyncNFTSocket(AsyncNetlinkSocket): } def __init__(self, version=1, attr_revision=0, nfgen_family=2): - super().__init__(family=NETLINK_NETFILTER) + super(NFTSocket, self).__init__(family=NETLINK_NETFILTER) policy = dict( [ (x | (NFNL_SUBSYS_NFTABLES << 8), y) @@ -1285,7 +1214,7 @@ def commit(self): self.addr_pool.free(seqnum, ban=10) del self._ts.data - async def request_get( + def request_get( self, msg, msg_type, @@ -1297,14 +1226,16 @@ async def request_get( the request and get an answer. ''' msg['nfgen_family'] = self._nfgen_family - return await self.nlm_request( - msg, - msg_type | (NFNL_SUBSYS_NFTABLES << 8), - msg_flags, - terminate=terminate, + return tuple( + self.nlm_request( + msg, + msg_type | (NFNL_SUBSYS_NFTABLES << 8), + msg_flags, + terminate=terminate, + ) ) - async def request_put(self, msg, msg_type, msg_flags=NLM_F_REQUEST): + def request_put(self, msg, msg_type, msg_flags=NLM_F_REQUEST): ''' Read-write requests. ''' @@ -1318,7 +1249,7 @@ async def request_put(self, msg, msg_type, msg_flags=NLM_F_REQUEST): if one_shot: self.commit() - async def _command(self, msg_class, commands, cmd, kwarg): + def _command(self, msg_class, commands, cmd, kwarg): flags = kwarg.pop('flags', NLM_F_ACK) cmd_name = cmd cmd_flags = { @@ -1349,7 +1280,7 @@ async def _command(self, msg_class, commands, cmd, kwarg): msg['attrs'].append([nla, value]) msg['header']['type'] = (NFNL_SUBSYS_NFTABLES << 8) | cmd msg['header']['flags'] = flags | NLM_F_REQUEST - msg['nfgen_family'] = kwarg.get('nfgen_family') or self._nfgen_family + msg['nfgen_family'] = self._nfgen_family if cmd_name != 'get': trans_start = nfgen_msg() @@ -1363,12 +1294,7 @@ async def _command(self, msg_class, commands, cmd, kwarg): trans_end['header']['flags'] = NLM_F_REQUEST messages = [trans_start, msg, trans_end] - return [ - x - async for x in self.nlm_request_batch( - messages, noraise=(flags & NLM_F_ACK) == 0 - ) - ] + self.nlm_request_batch(messages, noraise=(flags & NLM_F_ACK) == 0) # Only throw an error when the request fails. For now, # do not return anything. else: diff --git a/pyroute2/netlink/nl80211/__init__.py b/pyroute2/netlink/nl80211/__init__.py index 883ae9496..1b847f5bc 100644 --- a/pyroute2/netlink/nl80211/__init__.py +++ b/pyroute2/netlink/nl80211/__init__.py @@ -7,14 +7,13 @@ import datetime import struct - -from pyroute2.common import map_namespace -from pyroute2.netlink import genlmsg, nla, nla_base -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) -from pyroute2.netlink.nlsocket import Marshal +import datetime +from pr2modules.common import map_namespace +from pr2modules.netlink import genlmsg +from pr2modules.netlink.generic import GenericNetlinkSocket +from pr2modules.netlink.nlsocket import Marshal +from pr2modules.netlink import nla +from pr2modules.netlink import nla_base # Define from uapi/linux/nl80211.h NL80211_GENL_NAME = "nl80211" @@ -269,9 +268,9 @@ class nl80211cmd(genlmsg): ('NL80211_ATTR_DTIM_PERIOD', 'hex'), ('NL80211_ATTR_BEACON_HEAD', 'hex'), ('NL80211_ATTR_BEACON_TAIL', 'hex'), - ('NL80211_ATTR_STA_AID', 'uint16'), + ('NL80211_ATTR_STA_AID', 'hex'), ('NL80211_ATTR_STA_FLAGS', 'hex'), - ('NL80211_ATTR_STA_LISTEN_INTERVAL', 'uint16'), + ('NL80211_ATTR_STA_LISTEN_INTERVAL', 'hex'), ('NL80211_ATTR_STA_SUPPORTED_RATES', 'hex'), ('NL80211_ATTR_STA_VLAN', 'hex'), ('NL80211_ATTR_STA_INFO', 'STAInfo'), @@ -424,7 +423,7 @@ class nl80211cmd(genlmsg): ('NL80211_ATTR_RADAR_EVENT', 'hex'), ('NL80211_ATTR_EXT_CAPA', 'array(uint8)'), ('NL80211_ATTR_EXT_CAPA_MASK', 'array(uint8)'), - ('NL80211_ATTR_STA_CAPABILITY', 'uint16'), + ('NL80211_ATTR_STA_CAPABILITY', 'hex'), ('NL80211_ATTR_STA_EXT_CAPABILITY', 'hex'), ('NL80211_ATTR_PROTOCOL_FEATURES', 'hex'), ('NL80211_ATTR_SPLIT_WIPHY_DUMP', 'hex'), @@ -1612,17 +1611,13 @@ def fix_message(self, msg): pass -class AsyncNL80211(AsyncGenericNetlinkSocket): - marshal_class = MarshalNl80211 - - async def bind(self, groups=0, **kwarg): - await super().bind( - NL80211_GENL_NAME, nl80211cmd, groups, None, **kwarg - ) - - class NL80211(GenericNetlinkSocket): - async_class = AsyncNL80211 + + def __init__(self, *args, **kwargs): + GenericNetlinkSocket.__init__(self, *args, **kwargs) + self.marshal = MarshalNl80211() def bind(self, groups=0, **kwarg): - return self._run_with_cleanup(self.asyncore.bind, groups, **kwarg) + GenericNetlinkSocket.bind( + self, NL80211_GENL_NAME, nl80211cmd, groups, None, **kwarg + ) diff --git a/pyroute2/netlink/nlsocket.py b/pyroute2/netlink/nlsocket.py index 0f306d6e6..f63ac19ec 100644 --- a/pyroute2/netlink/nlsocket.py +++ b/pyroute2/netlink/nlsocket.py @@ -80,18 +80,30 @@ ------- ''' -import asyncio +import collections import errno import logging import os import random +import select import struct -from dataclasses import dataclass -from socket import SO_RCVBUF, SO_SNDBUF, SOCK_DGRAM, SOL_SOCKET -from typing import Optional +import threading +import time +import traceback +import warnings +from functools import partial +from socket import ( + MSG_DONTWAIT, + MSG_PEEK, + MSG_TRUNC, + SO_RCVBUF, + SO_SNDBUF, + SOCK_DGRAM, + SOL_SOCKET, +) -from pyroute2 import config, netns -from pyroute2.common import AddrPool, basestring, msg_done +from pyroute2 import config +from pyroute2.common import DEFAULT_RCVBUF, AddrPool from pyroute2.config import AF_NETLINK from pyroute2.netlink import ( NETLINK_ADD_MEMBERSHIP, @@ -101,30 +113,35 @@ NETLINK_GET_STRICT_CHK, NETLINK_LISTEN_ALL_NSID, NLM_F_ACK, - NLM_F_APPEND, - NLM_F_ATOMIC, - NLM_F_CREATE, + NLM_F_ACK_TLVS, NLM_F_DUMP, - NLM_F_ECHO, - NLM_F_EXCL, - NLM_F_REPLACE, + NLM_F_DUMP_INTR, + NLM_F_MULTI, NLM_F_REQUEST, - NLM_F_ROOT, + NLMSG_DONE, + NLMSG_ERROR, SOL_NETLINK, + mtypes, + nlmsg, + nlmsgerr, ) -from pyroute2.netlink.core import ( - AsyncCoreSocket, - CoreConfig, - CoreDatagramProtocol, - CoreSocketSpec, - SyncAPI, +from pyroute2.netlink.exceptions import ( + ChaoticException, + NetlinkDecodeError, + NetlinkDumpInterrupted, + NetlinkError, + NetlinkHeaderDecodeError, ) -from pyroute2.netlink.exceptions import ChaoticException, NetlinkError -from pyroute2.netlink.marshal import Marshal -from pyroute2.netlink.rt_files import NlProtosFile -from pyroute2.requests.main import RequestFilter + +try: + from Queue import Queue +except ImportError: + from queue import Queue log = logging.getLogger(__name__) +Stats = collections.namedtuple('Stats', ('qsize', 'delta', 'delay')) + +NL_BUFSIZE = 32768 class CompileContext: @@ -142,6 +159,121 @@ def close(self): self.netlink_socket.compiled = None +class Marshal: + ''' + Generic marshalling class + ''' + + msg_map = {} + seq_map = None + key_offset = None + key_format = None + key_mask = None + debug = False + default_message_class = nlmsg + error_type = NLMSG_ERROR + + def __init__(self): + self.lock = threading.Lock() + self.msg_map = self.msg_map.copy() + self.seq_map = {} + self.defragmentation = {} + + def parse_one_message( + self, key, flags, sequence_number, data, offset, length + ): + msg = None + error = None + msg_class = self.msg_map.get(key, self.default_message_class) + # ignore length for a while + # get the message + if (key == self.error_type) or ( + key == NLMSG_DONE and flags & NLM_F_ACK_TLVS + ): + msg = nlmsgerr(data, offset=offset) + else: + msg = msg_class(data, offset=offset) + + try: + msg.decode() + except NetlinkHeaderDecodeError as e: + msg = nlmsg() + msg['header']['error'] = e + except NetlinkDecodeError as e: + msg['header']['error'] = e + + if isinstance(msg, nlmsgerr) and msg['error'] != 0: + error = NetlinkError( + abs(msg['error']), msg.get_attr('NLMSGERR_ATTR_MSG') + ) + enc_type = struct.unpack_from('H', data, offset + 24)[0] + enc_class = self.msg_map.get(enc_type, nlmsg) + enc = enc_class(data, offset=offset + 20) + enc.decode() + msg['header']['errmsg'] = enc + + msg['header']['error'] = error + return msg + + def get_parser(self, key, flags, sequence_number): + return self.seq_map.get( + sequence_number, + partial(self.parse_one_message, key, flags, sequence_number), + ) + + def parse(self, data, seq=None, callback=None, skip_alien_seq=False): + ''' + Parse string data. + + At this moment all transport, except of the native + Netlink is deprecated in this library, so we should + not support any defragmentation on that level + ''' + offset = 0 + + # there must be at least one header in the buffer, + # 'IHHII' == 16 bytes + while offset <= len(data) - 16: + # pick type and length + (length, key, flags, sequence_number) = struct.unpack_from( + 'IHHI', data, offset + ) + if skip_alien_seq and sequence_number != seq: + continue + if not 0 < length <= len(data): + break + # support custom parser keys + # see also: pyroute2.netlink.diag.MarshalDiag + if self.key_format is not None: + (key,) = struct.unpack_from( + self.key_format, data, offset + self.key_offset + ) + if self.key_mask is not None: + key &= self.key_mask + + parser = self.get_parser(key, flags, sequence_number) + msg = parser(data, offset, length) + offset += length + if msg is None: + continue + + if callable(callback) and seq == sequence_number: + try: + if callback(msg): + continue + except Exception: + pass + + mtype = msg['header'].get('type', None) + if mtype in (1, 2, 3, 4) and 'event' not in msg: + msg['event'] = mtypes.get(mtype, 'none') + self.fix_message(msg) + yield msg + + def fix_message(self, msg): + pass + + # 8<----------------------------------------------------------- # Singleton, containing possible modifiers to the NetlinkSocket # bind() call. @@ -155,269 +287,113 @@ def close(self): # 8<----------------------------------------------------------- -@dataclass -class NetlinkConfig(CoreConfig): - family: int = NETLINK_GENERIC - port: Optional[int] = None - pid: Optional[int] = None - fileno: Optional[int] = None - rcvbuf: int = 1048576 - sndbuf: int = 1048576 - all_ns: bool = False - ext_ack: bool = False - strict_check: bool = False - nlm_echo: bool = False - - -class NetlinkSocketSpecFilter(RequestFilter): - def set_target(self, context, value): - if 'target' in context: - return {'target': context['target']} - return {'target': value} - - def set_netns(self, context, value): - if 'target' in context: - return {'netns': value} - return {'target': value, 'netns': value} - - def set_pid(self, context, value): - if value is None: - return {'pid': os.getpid() & 0x3FFFFF, 'port': context['port']} - elif value == 0: - return {'pid': os.getpid(), 'port': 0} - else: - return {'pid': value, 'port': 0} +class LockProxy: + def __init__(self, factory, key): + self.factory = factory + self.refcount = 0 + self.key = key + self.internal = threading.Lock() + self.lock = factory.klass() - def set_port(self, context, value): - if isinstance(value, int): - return {'port': value, 'epid': context['pid'] + (value << 22)} + def acquire(self, *argv, **kwarg): + with self.internal: + self.refcount += 1 + return self.lock.acquire() + def release(self): + with self.internal: + self.refcount -= 1 + if (self.refcount == 0) and (self.key != 0): + try: + del self.factory.locks[self.key] + except KeyError: + pass + return self.lock.release() -class NetlinkSocketSpec(CoreSocketSpec): - defaults = { - 'pid': 0, - 'epid': 0, - 'port': 0, - 'closed': False, - 'uname': config.uname, - 'use_socket': False, - 'event_loop': 'none', - 'transport_mode': 'none', - } - status_filters = [NetlinkSocketSpecFilter] + def __enter__(self): + self.acquire() + def __exit__(self, exc_type, exc_value, traceback): + self.release() -class AsyncNetlinkSocket(AsyncCoreSocket): - ''' - Netlink socket - ''' - marshal_class = Marshal +class LockFactory: + def __init__(self, klass=threading.RLock): + self.klass = klass + self.locks = {0: LockProxy(self, 0)} - def __init__( - self, - family=NETLINK_GENERIC, - port=None, - pid=None, - fileno=None, - sndbuf=1048576, - rcvbuf=1048576, - rcvsize=16384, - all_ns=False, - async_qsize=None, - nlm_generator=None, - target='localhost', - ext_ack=False, - strict_check=False, - groups=0, - nlm_echo=False, - use_socket=None, - netns=None, - flags=os.O_CREAT, - libc=None, - use_event_loop=None, - telemetry=None, - ): + def __enter__(self): + self.locks[0].acquire() - if isinstance(family, str): - family = NlProtosFile().get_rt_id(family) + def __exit__(self, exc_type, exc_value, traceback): + self.locks[0].release() - # 8<----------------------------------------- - self.spec = NetlinkSocketSpec( - NetlinkConfig( - family=family, - port=port, - pid=pid, - fileno=fileno, - sndbuf=sndbuf, - rcvbuf=rcvbuf, - rcvsize=rcvsize, - all_ns=all_ns, - target=target, - ext_ack=ext_ack, - strict_check=strict_check, - groups=groups, - nlm_echo=nlm_echo, - tag_field='sequence_number', - netns=netns, - flags=flags, - use_libc=libc is not None, - use_socket=use_socket is not None, - use_event_loop=use_event_loop is not None, - telemetry=telemetry, - ) - ) - # TODO: merge capabilities to self.status - self.capabilities = { - 'create_bridge': config.kernel > [3, 2, 0], - 'create_bond': config.kernel > [3, 2, 0], - 'create_dummy': True, - 'provide_master': config.kernel[0] > 2, - } - super().__init__( - libc=libc, use_socket=use_socket, use_event_loop=use_event_loop - ) - self.marshal = self.marshal_class() - self.request_proxy = None - self.batch = None - - async def setup_endpoint(self): - if getattr(self.local, 'transport', None) is not None: - return - self.local.transport, self.local.protocol = ( - await self.event_loop.create_datagram_endpoint( - lambda: CoreDatagramProtocol( - self.connection_lost, - self.enqueue, - self._error_event, - self.status, - ), - sock=self.socket, - ) - ) + def __getitem__(self, key): + if key is None: + key = 0 + if key not in self.locks: + self.locks[key] = LockProxy(self, key) + return self.locks[key] - def setup_socket(self): - if self.status['use_socket']: - return self.use_socket - sock = netns.create_socket( - netns=self.spec['netns'], - family=AF_NETLINK, - socket_type=SOCK_DGRAM, - proto=self.spec['family'], # netlink family = socket proto - fileno=self.spec['fileno'], - flags=self.spec['flags'], - libc=self.libc, - ) - sock.setsockopt(SOL_SOCKET, SO_SNDBUF, self.status['sndbuf']) - sock.setsockopt(SOL_SOCKET, SO_RCVBUF, self.status['rcvbuf']) - if self.status['ext_ack']: - sock.setsockopt(SOL_NETLINK, NETLINK_EXT_ACK, 1) - if self.status['all_ns']: - sock.setsockopt(SOL_NETLINK, NETLINK_LISTEN_ALL_NSID, 1) - if self.status['strict_check']: - sock.setsockopt(SOL_NETLINK, NETLINK_GET_STRICT_CHK, 1) - return sock + def __delitem__(self, key): + del self.locks[key] - @property - def uname(self): - return self.status['uname'] - @property - def groups(self): - return self.status['groups'] +class EngineBase: + def __init__(self, socket): + self.socket = socket + self.get_timeout = 30 + self.get_timeout_exception = None + self.change_master = threading.Event() + self.read_lock = threading.Lock() + self.qsize = 0 @property - def pid(self): - return self.status['pid'] + def marshal(self): + return self.socket.marshal @property - def port(self): - return self.status['port'] + def backlog(self): + return self.socket.backlog @property - def epid(self): - return self.status['epid'] + def backlog_lock(self): + return self.socket.backlog_lock @property - def target(self): - return self.status['target'] + def error_deque(self): + return self.socket.error_deque - def _sync_bind(self, groups=0, pid=None, **kwarg): - self.spec['groups'] = groups - # if we have pre-defined port, use it strictly - self.spec['pid'] = pid - if pid is None: - for port in range(20, 200): - try: - self.spec['port'] = port - self.socket.bind( - (self.status['epid'], self.status['groups']) - ) - break - except Exception as e: - # create a new underlying socket -- on kernel 4 - # one failed bind() makes the socket useless - log.debug(e) - else: - raise KeyError('no free address available') - - async def bind(self, groups=0, pid=None, **kwarg): - ''' - Bind the socket to given multicast groups, using - given pid. + @property + def lock(self): + return self.socket.lock - - If pid is None, use automatic port allocation - - If pid == 0, use process' pid - - If pid == , use the value instead of pid - ''' - await self.setup_endpoint() - self._sync_bind(groups, pid, **kwarg) + @property + def buffer_queue(self): + return self.socket.buffer_queue - def add_membership(self, group): - self.socket.setsockopt(SOL_NETLINK, NETLINK_ADD_MEMBERSHIP, group) + @property + def epid(self): + return self.socket.epid - def drop_membership(self, group): - self.socket.setsockopt(SOL_NETLINK, NETLINK_DROP_MEMBERSHIP, group) + @property + def target(self): + return self.socket.target - def enqueue(self, data, addr): - # calculate msg_seq - tag = struct.unpack_from('I', data, 8)[0] - return self.msg_queue.put_nowait(tag, data) + @property + def callbacks(self): + return self.socket.callbacks - def compile(self): - return CompileContext(self) - def make_request_type(self, command, command_map): - if isinstance(command, basestring): - return (lambda x: (x[0], self.make_request_flags(x[1])))( - command_map[command] - ) - elif isinstance(command, int): - return command, self.make_request_flags('create') - elif isinstance(command, (list, tuple)): - return command - else: - raise TypeError('allowed command types: int, str, list, tuple') - - def make_request_flags(self, mode): - flags = { - 'dump': NLM_F_REQUEST | NLM_F_DUMP, - 'get': NLM_F_REQUEST | NLM_F_ACK, - 'req': NLM_F_REQUEST | NLM_F_ACK, - 'put': NLM_F_REQUEST | NLM_F_CREATE, - } - flags['create'] = flags['req'] | NLM_F_CREATE | NLM_F_EXCL - flags['append'] = flags['req'] | NLM_F_CREATE | NLM_F_APPEND - flags['change'] = flags['req'] | NLM_F_REPLACE - flags['replace'] = flags['change'] | NLM_F_CREATE - - return flags[mode] | ( - NLM_F_ECHO - if (self.status['nlm_echo'] and mode not in ('get', 'dump')) - else 0 - ) +class EngineThreadSafe(EngineBase): + ''' + Thread-safe engine for netlink sockets. It buffers all + incoming messages regardless sequence numbers, and returns + only messages with requested numbers. This is done using + synchronization primitives in a quite complicated manner. + ''' - async def put( + def put( self, msg, msg_type, @@ -426,251 +402,381 @@ async def put( msg_seq=0, msg_pid=None, ): - request = NetlinkRequest( - self, - msg, - msg_type=msg_type, - msg_flags=msg_flags, - msg_seq=msg_seq, - msg_pid=msg_pid, - ) - await request.send() - return request + ''' + Construct a message from a dictionary and send it to + the socket. Parameters: + + - msg -- the message in the dictionary format + - msg_type -- the message type + - msg_flags -- the message flags to use in the request + - addr -- `sendto()` addr, default `(0, 0)` + - msg_seq -- sequence number to use + - msg_pid -- pid to use, if `None` -- use os.getpid() + + Example:: + + s = IPRSocket() + s.bind() + s.put({'index': 1}, RTM_GETLINK) + s.get() + s.close() + + Please notice, that the return value of `s.get()` can be + not the result of `s.put()`, but any broadcast message. + To fix that, use `msg_seq` -- the response must contain the + same `msg['header']['sequence_number']` value. + ''' + if msg_seq != 0: + self.lock[msg_seq].acquire() + try: + if msg_seq not in self.backlog: + self.backlog[msg_seq] = [] + if not isinstance(msg, nlmsg): + msg_class = self.marshal.msg_map[msg_type] + msg = msg_class(msg) + if msg_pid is None: + msg_pid = self.epid or os.getpid() + msg['header']['type'] = msg_type + msg['header']['flags'] = msg_flags + msg['header']['sequence_number'] = msg_seq + msg['header']['pid'] = msg_pid + self.socket.sendto_gate(msg, addr) + except: + raise + finally: + if msg_seq != 0: + self.lock[msg_seq].release() - async def nlm_request_batch(self, msgs, noraise=False): - expected_responses = [] - data = b'' - for message in msgs: - request = NetlinkRequest(self, message) - await request.prepare() - data += request.msg.data - if (request.msg['header']['flags'] & NLM_F_ACK) or ( - request.msg['header']['flags'] & NLM_F_DUMP - ): - expected_responses.append(request) - else: - request.cleanup() + def get( + self, + bufsize=DEFAULT_RCVBUF, + msg_seq=0, + terminate=None, + callback=None, + noraise=False, + ): + ''' + Get parsed messages list. If `msg_seq` is given, return + only messages with that `msg['header']['sequence_number']`, + saving all other messages into `self.backlog`. + + The routine is thread-safe. - self.send(data) - for request in expected_responses: + The `bufsize` parameter can be: + + - -1: bufsize will be calculated from the first 4 bytes of + the network data + - 0: bufsize will be calculated from SO_RCVBUF sockopt + - int >= 0: just a bufsize + + If `noraise` is true, error messages will be treated as any + other message. + ''' + ctime = time.time() + + with self.lock[msg_seq]: + if bufsize == -1: + # get bufsize from the network data + bufsize = struct.unpack("I", self.recv(4, MSG_PEEK))[0] + elif bufsize == 0: + # get bufsize from SO_RCVBUF + bufsize = self.getsockopt(SOL_SOCKET, SO_RCVBUF) // 2 + + tmsg = None + enough = False + backlog_acquired = False try: - async for msg in request.response(): - yield msg - except NetlinkError: - if not noraise: - raise + while not enough: + # 8<----------------------------------------------------------- + # + # This stage changes the backlog, so use mutex to + # prevent side changes + self.backlog_lock.acquire() + backlog_acquired = True + ## + # Stage 1. BEGIN + # + # 8<----------------------------------------------------------- + # + # Check backlog and return already collected + # messages. + # + if msg_seq == -1 and any(self.backlog.values()): + for seq, backlog in self.backlog.items(): + if backlog: + for msg in backlog: + yield msg + self.backlog[seq] = [] + enough = True + break + elif msg_seq == 0 and self.backlog[0]: + # Zero queue. + # + # Load the backlog, if there is valid + # content in it + for msg in self.backlog[0]: + yield msg + self.backlog[0] = [] + # And just exit + break + elif msg_seq > 0 and len(self.backlog.get(msg_seq, [])): + # Any other msg_seq. + # + # Collect messages up to the terminator. + # Terminator conditions: + # * NLMSG_ERROR != 0 + # * NLMSG_DONE + # * terminate() function (if defined) + # * not NLM_F_MULTI + # + # Please note, that if terminator not occured, + # more `recv()` rounds CAN be required. + for msg in tuple(self.backlog[msg_seq]): + # Drop the message from the backlog, if any + self.backlog[msg_seq].remove(msg) + + # If there is an error, raise exception + if ( + msg['header']['error'] is not None + and not noraise + ): + # reschedule all the remaining messages, + # including errors and acks, into a + # separate deque + self.error_deque.extend(self.backlog[msg_seq]) + # flush the backlog for this msg_seq + del self.backlog[msg_seq] + # The loop is done + raise msg['header']['error'] + + # If it is the terminator message, say "enough" + # and requeue all the rest into Zero queue + if terminate is not None: + tmsg = terminate(msg) + if isinstance(tmsg, nlmsg): + yield msg + if (msg['header']['type'] == NLMSG_DONE) or tmsg: + # The loop is done + enough = True + + # If it is just a normal message, append it to + # the response + if not enough: + # finish the loop on single messages + if not msg['header']['flags'] & NLM_F_MULTI: + enough = True + yield msg + + # Enough is enough, requeue the rest and delete + # our backlog + if enough: + self.backlog[0].extend(self.backlog[msg_seq]) + del self.backlog[msg_seq] + break + + # Next iteration + self.backlog_lock.release() + backlog_acquired = False + else: + # Stage 1. END + # + # 8<------------------------------------------------------- + # + # Stage 2. BEGIN + # + # 8<------------------------------------------------------- + # + # Receive the data from the socket and put the messages + # into the backlog + # + self.backlog_lock.release() + backlog_acquired = False + ## + # + # Control the timeout. We should not be within the + # function more than TIMEOUT seconds. All the locks + # MUST be released here. + # + if (msg_seq != 0) and ( + time.time() - ctime > self.get_timeout + ): + # requeue already received for that msg_seq + self.backlog[0].extend(self.backlog[msg_seq]) + del self.backlog[msg_seq] + # throw an exception + if self.get_timeout_exception: + raise self.get_timeout_exception() + else: + return + # + if self.read_lock.acquire(False): + try: + self.change_master.clear() + # If the socket is free to read from, occupy + # it and wait for the data + # + # This is a time consuming process, so all the + # locks, except the read lock must be released + data = self.socket.recv(bufsize) + # Parse data + msgs = tuple( + self.socket.marshal.parse( + data, msg_seq, callback + ) + ) + # Reset ctime -- timeout should be measured + # for every turn separately + ctime = time.time() + # + current = self.buffer_queue.qsize() + delta = current - self.qsize + delay = 0 + if delta > 10: + delay = min( + 3, max(0.01, float(current) / 60000) + ) + message = ( + "Packet burst: " + "delta=%s qsize=%s delay=%s" + % (delta, current, delay) + ) + if delay < 1: + log.debug(message) + else: + log.warning(message) + time.sleep(delay) + self.qsize = current + + # We've got the data, lock the backlog again + with self.backlog_lock: + for msg in msgs: + msg['header']['target'] = self.target + msg['header']['stats'] = Stats( + current, delta, delay + ) + seq = msg['header']['sequence_number'] + if seq not in self.backlog: + if ( + msg['header']['type'] + == NLMSG_ERROR + ): + # Drop orphaned NLMSG_ERROR + # messages + continue + seq = 0 + # 8<----------------------------------- + # Callbacks section + for cr in self.callbacks: + try: + if cr[0](msg): + cr[1](msg, *cr[2]) + except: + # FIXME + # + # Usually such code formatting + # means that the method should + # be refactored to avoid such + # indentation. + # + # Plz do something with it. + # + lw = log.warning + lw("Callback fail: %s" % (cr)) + lw(traceback.format_exc()) + # 8<----------------------------------- + self.backlog[seq].append(msg) + + # Now wake up other threads + self.change_master.set() + finally: + # Finally, release the read lock: all data + # processed + self.read_lock.release() + else: + # If the socket is occupied and there is still no + # data for us, wait for the next master change or + # for a timeout + self.change_master.wait(1) + # 8<------------------------------------------------------- + # + # Stage 2. END + # + # 8<------------------------------------------------------- + finally: + if backlog_acquired: + self.backlog_lock.release() + + +class EngineThreadUnsafe(EngineBase): + ''' + Thread unsafe nlsocket base class. Does not implement any locks + on message processing. Discards any message if the sequence number + does not match. + ''' - async def nlm_request( + def put( self, msg, msg_type, - msg_flags=NLM_F_REQUEST | NLM_F_DUMP, - terminate=None, - callback=None, - parser=None, + msg_flags=NLM_F_REQUEST, + addr=(0, 0), + msg_seq=0, + msg_pid=None, ): - request = NetlinkRequest( - self, msg, terminate=terminate, callback=callback - ) - request.msg['header']['type'] = msg_type - request.msg['header']['flags'] = msg_flags - await request.send() - return request.response() - - -class NetlinkRequest: - # request flags - flags = { - 'dump': NLM_F_REQUEST | NLM_F_DUMP, - 'root': NLM_F_REQUEST | NLM_F_ROOT | NLM_F_ATOMIC, - 'get': NLM_F_REQUEST | NLM_F_ACK, - 'req': NLM_F_REQUEST | NLM_F_ACK, - } - flags['create'] = flags['req'] | NLM_F_CREATE | NLM_F_EXCL - flags['append'] = flags['req'] | NLM_F_CREATE | NLM_F_APPEND - flags['change'] = flags['req'] | NLM_F_REPLACE - flags['replace'] = flags['change'] | NLM_F_CREATE - - def __init__( + if not isinstance(msg, nlmsg): + msg_class = self.marshal.msg_map[msg_type] + msg = msg_class(msg) + if msg_pid is None: + msg_pid = self.epid or os.getpid() + msg['header']['type'] = msg_type + msg['header']['flags'] = msg_flags + msg['header']['sequence_number'] = msg_seq + msg['header']['pid'] = msg_pid + self.sendto_gate(msg, addr) + + def get( self, - sock, - msg, - command=None, - command_map=None, - dump_filter=None, - request_filter=None, + bufsize=DEFAULT_RCVBUF, + msg_seq=0, terminate=None, callback=None, - parser=None, - msg_type=None, - msg_flags=None, - msg_seq=None, - msg_pid=None, + noraise=False, ): - self.sock = sock - self.addr_pool = sock.addr_pool - self.status = sock.status - self.epid = sock.epid if msg_pid is None else msg_pid - self.marshal = sock.marshal - self.parser = parser - # if not isinstance(msg, nlmsg): - # msg_class = self.marshal.msg_map[msg_type] - # msg = msg_class(msg) - self.msg_seq = self.addr_pool.alloc() if msg_seq is None else msg_seq - - # prio 3: message object - # prio 2: direct msg_type & msg_flags arguments - # prio 1: command map - if command_map is not None: - msg_type, msg_flags = self.calculate_request_type( - command, command_map, self.status['nlm_echo'] - ) - if msg_type is not None: - msg['header']['type'] = msg_type - if msg_flags is not None: - msg['header']['flags'] = msg_flags - - # if there is no type & flags yet, set defaults - # FIXME: collect usecases - if msg['header'].get('type') is None: - msg['header']['type'] = 0 - if msg['header'].get('flags') is None: - msg['header']['flags'] = NLM_F_REQUEST | NLM_F_DUMP - - msg['header']['sequence_number'] = self.msg_seq - msg['header']['pid'] = self.epid or os.getpid() - msg.reset() - # set fields - if request_filter is not None: - for field in msg.fields: - msg[field[0]] = request_filter.get_value( - field[0], default=0, mode='field' - ) - # attach NLAs - for key, value in request_filter.items(): - nla = type(msg).name2nla(key) - if msg.valid_nla(nla) and value is not None: - msg['attrs'].append([nla, value]) - # extend with custom NLAs - if 'attrs' in request_filter: - msg['attrs'].extend(request_filter['attrs']) - self.msg = msg - self.dump_filter = dump_filter - self.terminate = terminate - self.callback = callback - self.command = command - - @classmethod - def calculate_request_type(cls, command, command_map, echo=False): - if isinstance(command, basestring): - return (lambda x: (x[0], cls.calculate_request_flags(x[1], echo)))( - command_map[command] + if bufsize == -1: + # get bufsize from the network data + bufsize = struct.unpack("I", self.recv(4, MSG_PEEK))[0] + elif bufsize == 0: + # get bufsize from SO_RCVBUF + bufsize = self.getsockopt(SOL_SOCKET, SO_RCVBUF) // 2 + enough = False + while not enough: + data = self.recv(bufsize) + *messages, last = tuple( + self.marshal.parse(data, msg_seq, callback) ) - elif isinstance(command, int): - return command, cls.calculate_request_flags('create', echo) - elif isinstance(command, (list, tuple)): - return command - else: - raise TypeError('allowed command types: int, str, list, tuple') - - @classmethod - def calculate_request_flags(cls, mode, echo): - return cls.flags[mode] | ( - NLM_F_ECHO if (echo and mode not in ('get', 'dump')) else 0 - ) + for msg in messages: + msg['header']['target'] = self.target + msg['header']['stats'] = Stats(0, 0, 0) + yield msg - @staticmethod - def match_one_message(dump_filter, msg): - if hasattr(dump_filter, '__call__'): - return dump_filter(msg) - elif isinstance(dump_filter, dict): - matches = [] - for key in dump_filter: - # get the attribute - if not isinstance(key, (str, tuple)): - continue - value = msg.get(key) - if value is not None and callable(dump_filter[key]): - matches.append(dump_filter[key](value)) - else: - matches.append(dump_filter[key] == value) - return all(matches) - - async def prepare(self): - await self.sock.setup_endpoint() - self.msg.encode() - self.sock.msg_queue.ensure_tag(self.msg_seq) - if self.parser is not None: - self.marshal.seq_map[self.msg_seq] = self.parser - - def cleanup(self): - self.addr_pool.free(self.msg_seq, ban=0xFF) - self.sock.msg_queue.free_tag(self.msg_seq) - if self.msg_seq in self.marshal.seq_map: - self.marshal.seq_map.pop(self.msg_seq) - - async def proxy(self): - if self.sock.batch is not None: - self.sock.batch += self.msg.data - await self.sock.msg_queue.put(self.msg_seq, msg_done(self.msg)) - return True - if self.sock.request_proxy is None: - return False - ret = self.sock.request_proxy.handle(self.msg) - if ret == b'': - return False - await self.sock.msg_queue.put(self.msg_seq, ret) - return True - - async def send(self): - await self.prepare() - if await self.proxy(): - return len(self.msg.data) - count = 0 - exc = RuntimeError('Max attempts sending message') - for count in range(30): - try: - return self.sock.send(self.msg.data) - except NetlinkError as e: - if e.code != errno.EBUSY: - exc = e - break - log.warning(f'Error 16, retry {count}') - await asyncio.sleep(0.3) - except Exception as e: - exc = e + if last['header']['type'] == NLMSG_DONE: break - self.cleanup() - raise exc - - async def response(self): - coro = self.sock.get( - msg_seq=self.msg_seq, - terminate=self.terminate, - callback=self.callback, - ) - try: - async for msg in coro: - if ( - self.dump_filter is not None - and not self.match_one_message(self.dump_filter, msg) - ): - continue - for cr in self.sock.callbacks: - try: - if cr[0](msg): - cr[1](msg, *cr[2]) - except Exception: - log.warning("Callback fail: %{cr}") - yield msg - finally: - await coro.aclose() - self.cleanup() + + if ( + (msg_seq == 0) + or (not last['header']['flags'] & NLM_F_MULTI) + or (callable(terminate) and terminate(last)) + ): + enough = True + yield last -class NetlinkSocket(SyncAPI): +class NetlinkSocketBase: + ''' + Generic netlink socket. + ''' - async_class = AsyncNetlinkSocket + input_from_buffer_queue = False def __init__( self, @@ -680,61 +786,113 @@ def __init__( fileno=None, sndbuf=1048576, rcvbuf=1048576, - rcvsize=16384, all_ns=False, async_qsize=None, - nlm_generator=True, + nlm_generator=None, target='localhost', ext_ack=False, strict_check=False, groups=0, nlm_echo=False, - netns=None, - flags=os.O_CREAT, - libc=None, - use_socket=None, - use_event_loop=None, - telemetry=None, ): - self.asyncore = self.async_class( - family=family, - port=port, - pid=pid, - fileno=fileno, - sndbuf=sndbuf, - rcvbuf=rcvbuf, - rcvsize=rcvsize, - all_ns=all_ns, - target=target, - ext_ack=ext_ack, - strict_check=strict_check, - groups=groups, - nlm_echo=nlm_echo, - netns=netns, - flags=flags, - libc=libc, - use_socket=use_socket, - use_event_loop=use_event_loop, - telemetry=telemetry, - ) - self.asyncore.local.keep_event_loop = True - self.asyncore.status['event_loop'] = 'new' - self.asyncore.status['nlm_generator'] = nlm_generator - # FIXME: temporary override from a class attribute - if hasattr(self, 'class_gen_sync'): - self.asyncore.status['nlm_generator'] = self.class_gen_sync - self.asyncore.event_loop.run_until_complete( - self.asyncore.setup_endpoint() - ) - if self.asyncore.socket.fileno() == -1: - raise OSError(9, 'Bad file descriptor') + # 8<----------------------------------------- + self.config = { + 'family': family, + 'port': port, + 'pid': pid, + 'fileno': fileno, + 'sndbuf': sndbuf, + 'rcvbuf': rcvbuf, + 'all_ns': all_ns, + 'async_qsize': async_qsize, + 'target': target, + 'nlm_generator': nlm_generator, + 'ext_ack': ext_ack, + 'strict_check': strict_check, + 'groups': groups, + 'nlm_echo': nlm_echo, + } + # 8<----------------------------------------- + self.addr_pool = AddrPool(minaddr=0x000000FF, maxaddr=0x0000FFFF) + self.epid = None + self.port = 0 + self.fixed = True + self.family = family + self._fileno = fileno + self._sndbuf = sndbuf + self._rcvbuf = rcvbuf + self._use_peek = True + self.backlog = {0: []} + self.error_deque = collections.deque(maxlen=1000) + self.callbacks = [] # [(predicate, callback, args), ...] + self.buffer_thread = None + self.closed = False + self.compiled = None + self.uname = config.uname + self.target = target + self.groups = groups + self.capabilities = { + 'create_bridge': config.kernel > [3, 2, 0], + 'create_bond': config.kernel > [3, 2, 0], + 'create_dummy': True, + 'provide_master': config.kernel[0] > 2, + } + self.backlog_lock = threading.Lock() + self.sys_lock = threading.RLock() + self.lock = LockFactory() + self._sock = None + self._ctrl_read, self._ctrl_write = os.pipe() + if async_qsize is None: + async_qsize = config.async_qsize + self.async_qsize = async_qsize + if nlm_generator is None: + nlm_generator = config.nlm_generator + self.nlm_generator = nlm_generator + self.buffer_queue = Queue(maxsize=async_qsize) + self.log = [] + self.all_ns = all_ns + self.ext_ack = ext_ack + self.strict_check = strict_check + if pid is None: + self.pid = os.getpid() & 0x3FFFFF + self.port = port + self.fixed = self.port is not None + elif pid == 0: + self.pid = os.getpid() + else: + self.pid = pid + # 8<----------------------------------------- + self.marshal = Marshal() + # 8<----------------------------------------- + if not nlm_generator: + + def nlm_request(*argv, **kwarg): + return tuple(self._genlm_request(*argv, **kwarg)) + + def get(*argv, **kwarg): + return tuple(self._genlm_get(*argv, **kwarg)) + + self._genlm_request = self.nlm_request + self._genlm_get = self.get + + self.nlm_request = nlm_request + self.get = get + + def nlm_request_batch(*argv, **kwarg): + return tuple(self._genlm_request_batch(*argv, **kwarg)) + + self._genlm_request_batch = self.nlm_request_batch + self.nlm_request_batch = nlm_request_batch - def bind(self, *argv, **kwarg): - with self.lock: - self.asyncore._check_tid(tag='bind', level=logging.WARN) - self.asyncore.local.keep_event_loop = True - self._run_with_cleanup(self.asyncore.bind, *argv, **kwarg) - self.asyncore._register_loop_ref() + # Set defaults + self.post_init() + self.engine = EngineThreadSafe(self) + + def post_init(self): + pass + + def clone(self): + return type(self)(**self.config) def put( self, @@ -742,23 +900,293 @@ def put( msg_type, msg_flags=NLM_F_REQUEST, addr=(0, 0), - msg_seq=None, + msg_seq=0, msg_pid=None, ): - if msg is None: - msg_class = self.marshal.msg_map[msg_type] - msg = msg_class() - return self._run_with_cleanup( - self.asyncore.put, msg, msg_type, msg_flags, addr, msg_seq, msg_pid + return self.engine.put( + msg, msg_type, msg_flags, addr, msg_seq, msg_pid ) - def nlm_request_batch(self, msgs, noraise=False): - async def collect_data(): - return [ - x async for x in self.asyncore.nlm_request_batch(msgs, noraise) - ] + def get( + self, + bufsize=DEFAULT_RCVBUF, + msg_seq=0, + terminate=None, + callback=None, + noraise=False, + ): + return self.engine.get(bufsize, msg_seq, terminate, callback, noraise) + + def close(self, code=errno.ECONNRESET): + if code > 0 and self.input_from_buffer_queue: + self.buffer_queue.put( + struct.pack('IHHQIQQ', 28, 2, 0, 0, code, 0, 0) + ) + try: + os.close(self._ctrl_write) + os.close(self._ctrl_read) + except OSError: + # ignore the case when it is closed already + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def release(self): + warnings.warn('deprecated, use close() instead', DeprecationWarning) + self.close() + + def register_callback(self, callback, predicate=lambda x: True, args=None): + ''' + Register a callback to run on a message arrival. + + Callback is the function that will be called with the + message as the first argument. Predicate is the optional + callable object, that returns True or False. Upon True, + the callback will be called. Upon False it will not. + Args is a list or tuple of arguments. + + Simplest example, assume ipr is the IPRoute() instance:: + + # create a simplest callback that will print messages + def cb(msg): + print(msg) + + # register callback for any message: + ipr.register_callback(cb) + + More complex example, with filtering:: + + # Set object's attribute after the message key + def cb(msg, obj): + obj.some_attr = msg["some key"] + + # Register the callback only for the loopback device, index 1: + ipr.register_callback(cb, + lambda x: x.get('index', None) == 1, + (self, )) + + Please note: you do **not** need to register the default 0 queue + to invoke callbacks on broadcast messages. Callbacks are + iterated **before** messages get enqueued. + ''' + if args is None: + args = [] + self.callbacks.append((predicate, callback, args)) + + def unregister_callback(self, callback): + ''' + Remove the first reference to the function from the callback + register + ''' + cb = tuple(self.callbacks) + for cr in cb: + if cr[1] == callback: + self.callbacks.pop(cb.index(cr)) + return + + def register_policy(self, policy, msg_class=None): + ''' + Register netlink encoding/decoding policy. Can + be specified in two ways: + `nlsocket.register_policy(MSG_ID, msg_class)` + to register one particular rule, or + `nlsocket.register_policy({MSG_ID1: msg_class})` + to register several rules at once. + E.g.:: + + policy = {RTM_NEWLINK: ifinfmsg, + RTM_DELLINK: ifinfmsg, + RTM_NEWADDR: ifaddrmsg, + RTM_DELADDR: ifaddrmsg} + nlsocket.register_policy(policy) + + One can call `register_policy()` as many times, + as one want to -- it will just extend the current + policy scheme, not replace it. + ''' + if isinstance(policy, int) and msg_class is not None: + policy = {policy: msg_class} + + if not isinstance(policy, dict): + raise TypeError('wrong policy type') + for key in policy: + self.marshal.msg_map[key] = policy[key] + + return self.marshal.msg_map + + def unregister_policy(self, policy): + ''' + Unregister policy. Policy can be: + + - int -- then it will just remove one policy + - list or tuple of ints -- remove all given + - dict -- remove policies by keys from dict + + In the last case the routine will ignore dict values, + it is implemented so just to make it compatible with + `get_policy_map()` return value. + ''' + if isinstance(policy, int): + policy = [policy] + elif isinstance(policy, dict): + policy = list(policy) + + if not isinstance(policy, (tuple, list, set)): + raise TypeError('wrong policy type') + + for key in policy: + del self.marshal.msg_map[key] + + return self.marshal.msg_map + + def get_policy_map(self, policy=None): + ''' + Return policy for a given message type or for all + message types. Policy parameter can be either int, + or a list of ints. Always return dictionary. + ''' + if policy is None: + return self.marshal.msg_map + + if isinstance(policy, int): + policy = [policy] + + if not isinstance(policy, (list, tuple, set)): + raise TypeError('wrong policy type') + + ret = {} + for key in policy: + ret[key] = self.marshal.msg_map[key] + + return ret + + def _peek_bufsize(self, socket_descriptor): + data = bytearray() + try: + bufsize, _ = socket_descriptor.recvfrom_into( + data, 0, MSG_DONTWAIT | MSG_PEEK | MSG_TRUNC + ) + except BlockingIOError: + self._use_peek = False + bufsize = socket_descriptor.getsockopt(SOL_SOCKET, SO_RCVBUF) // 2 + return bufsize + + def sendto(self, *argv, **kwarg): + return self._sendto(*argv, **kwarg) + + def recv(self, bufsize, flags=0): + if self.input_from_buffer_queue: + data_in = self.buffer_queue.get() + if isinstance(data_in, Exception): + raise data_in + return data_in + return self._sock.recv( + self._peek_bufsize(self._sock) if self._use_peek else bufsize, + flags, + ) - return self._run_with_cleanup(collect_data) + def recv_into(self, data, *argv, **kwarg): + if self.input_from_buffer_queue: + data_in = self.buffer_queue.get() + if isinstance(data, Exception): + raise data_in + data[:] = data_in + return len(data_in) + return self._sock.recv_into(data, *argv, **kwarg) + + def buffer_thread_routine(self): + poll = select.poll() + poll.register(self._sock, select.POLLIN | select.POLLPRI) + poll.register(self._ctrl_read, select.POLLIN | select.POLLPRI) + sockfd = self._sock.fileno() + while True: + events = poll.poll() + for fd, event in events: + if fd == sockfd: + try: + data = bytearray(64000) + self._sock.recv_into(data, 64000) + self.buffer_queue.put_nowait(data) + except Exception as e: + self.buffer_queue.put(e) + return + else: + return + + def compile(self): + return CompileContext(self) + + def _send_batch(self, msgs, addr=(0, 0)): + with self.backlog_lock: + for msg in msgs: + self.backlog[msg['header']['sequence_number']] = [] + # We have locked the message locks in the caller already. + data = bytearray() + for msg in msgs: + if not isinstance(msg, nlmsg): + msg_class = self.marshal.msg_map[msg['header']['type']] + msg = msg_class(msg) + msg.reset() + msg.encode() + data += msg.data + if self.compiled is not None: + return self.compiled.append(data) + self._sock.sendto(data, addr) + + def sendto_gate(self, msg, addr): + msg.reset() + msg.encode() + if self.compiled is not None: + return self.compiled.append(msg.data) + return self._sock.sendto(msg.data, addr) + + def nlm_request_batch(self, msgs, noraise=False): + """ + This function is for messages which are expected to have side effects. + Do not blindly retry in case of errors as this might duplicate them. + """ + expected_responses = [] + acquired = 0 + seqs = self.addr_pool.alloc_multi(len(msgs)) + try: + for seq in seqs: + self.lock[seq].acquire() + acquired += 1 + for seq, msg in zip(seqs, msgs): + msg['header']['sequence_number'] = seq + if 'pid' not in msg['header']: + msg['header']['pid'] = self.epid or os.getpid() + if (msg['header']['flags'] & NLM_F_ACK) or ( + msg['header']['flags'] & NLM_F_DUMP + ): + expected_responses.append(seq) + self._send_batch(msgs) + if self.compiled is not None: + for data in self.compiled: + yield data + else: + for seq in expected_responses: + for msg in self.get(msg_seq=seq, noraise=noraise): + if msg['header']['flags'] & NLM_F_DUMP_INTR: + # Leave error handling to the caller + raise NetlinkDumpInterrupted() + yield msg + finally: + # Release locks in reverse order. + for seq in seqs[acquired - 1 :: -1]: + self.lock[seq].release() + + with self.backlog_lock: + for seq in seqs: + # Clear the backlog. We may have raised an error + # causing the backlog to not be consumed entirely. + if seq in self.backlog: + del self.backlog[seq] + self.addr_pool.free(seq, ban=0xFF) def nlm_request( self, @@ -769,30 +1197,238 @@ def nlm_request( callback=None, parser=None, ): - ret = self._generate_with_cleanup( - self.asyncore.nlm_request, - msg, - msg_type, - msg_flags, - terminate, - callback, - parser, - ) - if self.status['nlm_generator']: - return ret - return tuple(ret) + msg_seq = self.addr_pool.alloc() + defer = None + if callable(parser): + self.marshal.seq_map[msg_seq] = parser + with self.lock[msg_seq]: + retry_count = 0 + try: + while True: + try: + self.put(msg, msg_type, msg_flags, msg_seq=msg_seq) + if self.compiled is not None: + for data in self.compiled: + yield data + else: + for msg in self.get( + msg_seq=msg_seq, + terminate=terminate, + callback=callback, + ): + # analyze the response for effects to be + # deferred + if ( + defer is None + and msg['header']['flags'] + & NLM_F_DUMP_INTR + ): + defer = NetlinkDumpInterrupted() + yield msg + break + except NetlinkError as e: + if e.code != errno.EBUSY: + raise + if retry_count >= 30: + raise + log.warning('Error 16, retry {}.'.format(retry_count)) + time.sleep(0.055) + retry_count += 1 + continue + except Exception: + raise + finally: + # Ban this msg_seq for 0xff rounds + # + # It's a long story. Modern kernels for RTM_SET.* + # operations always return NLMSG_ERROR(0) == success, + # even not setting NLM_F_MULTI flag on other response + # messages and thus w/o any NLMSG_DONE. So, how to detect + # the response end? One can not rely on NLMSG_ERROR on + # old kernels, but we have to support them too. Ty, we + # just ban msg_seq for several rounds, and NLMSG_ERROR, + # being received, will become orphaned and just dropped. + # + # Hack, but true. + self.addr_pool.free(msg_seq, ban=0xFF) + if msg_seq in self.marshal.seq_map: + self.marshal.seq_map.pop(msg_seq) + if defer is not None: + raise defer - def get(self, msg_seq=0, terminate=None, callback=None, noraise=False): - async def collect_data(): - return [ - i - async for i in self.asyncore.get( - msg_seq, terminate, callback, noraise - ) - ] +class BatchAddrPool: + def alloc(self, *argv, **kwarg): + return 0 + + def free(self, *argv, **kwarg): + pass + + +class BatchBacklogQueue(list): + def append(self, *argv, **kwarg): + pass + + def pop(self, *argv, **kwarg): + pass + + +class BatchBacklog(dict): + def __getitem__(self, key): + return BatchBacklogQueue() + + def __setitem__(self, key, value): + pass + + def __delitem__(self, key): + pass + + +class BatchSocket(NetlinkSocketBase): + def post_init(self): + self.backlog = BatchBacklog() + self.addr_pool = BatchAddrPool() + self._sock = None + self.reset() + + def reset(self): + self.batch = bytearray() - return self._run_with_cleanup(collect_data) + def nlm_request( + self, + msg, + msg_type, + msg_flags=NLM_F_REQUEST | NLM_F_DUMP, + terminate=None, + callback=None, + ): + msg_seq = self.addr_pool.alloc() + msg_pid = self.epid or os.getpid() + + msg['header']['type'] = msg_type + msg['header']['flags'] = msg_flags + msg['header']['sequence_number'] = msg_seq + msg['header']['pid'] = msg_pid + msg.data = self.batch + msg.offset = len(self.batch) + msg.encode() + return [] + + def get(self, *argv, **kwarg): + pass + + +class NetlinkSocket(NetlinkSocketBase): + def post_init(self): + # recreate the underlying socket + with self.sys_lock: + if self._sock is not None: + self._sock.close() + self._sock = config.SocketBase( + AF_NETLINK, SOCK_DGRAM, self.family, self._fileno + ) + self.setsockopt(SOL_SOCKET, SO_SNDBUF, self._sndbuf) + self.setsockopt(SOL_SOCKET, SO_RCVBUF, self._rcvbuf) + if self.ext_ack: + self.setsockopt(SOL_NETLINK, NETLINK_EXT_ACK, 1) + if self.all_ns: + self.setsockopt(SOL_NETLINK, NETLINK_LISTEN_ALL_NSID, 1) + if self.strict_check: + self.setsockopt(SOL_NETLINK, NETLINK_GET_STRICT_CHK, 1) + + def __getattr__(self, attr): + if attr in ( + 'getsockname', + 'getsockopt', + 'makefile', + 'setsockopt', + 'setblocking', + 'settimeout', + 'gettimeout', + 'shutdown', + 'recvfrom', + 'recvfrom_into', + 'fileno', + ): + return getattr(self._sock, attr) + elif attr in ('_sendto', '_recv', '_recv_into'): + return getattr(self._sock, attr.lstrip("_")) + + raise AttributeError(attr) + + def bind(self, groups=0, pid=None, **kwarg): + ''' + Bind the socket to given multicast groups, using + given pid. + + - If pid is None, use automatic port allocation + - If pid == 0, use process' pid + - If pid == , use the value instead of pid + ''' + if pid is not None: + self.port = 0 + self.fixed = True + self.pid = pid or os.getpid() + + if 'async' in kwarg: + # FIXME + # raise deprecation error after 0.5.3 + # + log.warning( + 'use "async_cache" instead of "async", ' + '"async" is a keyword from Python 3.7' + ) + async_cache = kwarg.get('async_cache') or kwarg.get('async') + + self.groups = groups + # if we have pre-defined port, use it strictly + if self.fixed: + self.epid = self.pid + (self.port << 22) + self._sock.bind((self.epid, self.groups)) + else: + for port in range(1024): + try: + self.port = port + self.epid = self.pid + (self.port << 22) + self._sock.bind((self.epid, self.groups)) + break + except Exception: + # create a new underlying socket -- on kernel 4 + # one failed bind() makes the socket useless + self.post_init() + else: + raise KeyError('no free address available') + # all is OK till now, so start async recv, if we need + if async_cache: + self.buffer_thread = threading.Thread( + name="Netlink async cache", target=self.buffer_thread_routine + ) + self.input_from_buffer_queue = True + self.buffer_thread.daemon = True + self.buffer_thread.start() + + def add_membership(self, group): + self.setsockopt(SOL_NETLINK, NETLINK_ADD_MEMBERSHIP, group) + + def drop_membership(self, group): + self.setsockopt(SOL_NETLINK, NETLINK_DROP_MEMBERSHIP, group) + + def close(self, code=errno.ECONNRESET): + ''' + Correctly close the socket and free all resources. + ''' + with self.sys_lock: + if self.closed: + return + self.closed = True + + if self.buffer_thread: + os.write(self._ctrl_write, b'exit') + self.buffer_thread.join() + super(NetlinkSocket, self).close(code=code) + + # Common shutdown procedure + self._sock.close() class ChaoticNetlinkSocket(NetlinkSocket): diff --git a/pyroute2/netlink/proxy.py b/pyroute2/netlink/proxy.py new file mode 100644 index 000000000..31fc0a992 --- /dev/null +++ b/pyroute2/netlink/proxy.py @@ -0,0 +1,76 @@ +''' +Netlink proxy engine +''' + +import errno +import logging +import struct +import threading +import traceback + +from pyroute2.netlink.exceptions import NetlinkError + +log = logging.getLogger(__name__) + + +class NetlinkProxy(object): + ''' + Proxy schemes:: + + User -> NetlinkProxy -> Kernel + | + <---------+ + + User <- NetlinkProxy <- Kernel + + ''' + + def __init__(self, policy='forward', nl=None, lock=None): + self.nl = nl + self.lock = lock or threading.Lock() + self.pmap = {} + self.policy = policy + + def handle(self, msg): + # + # match the packet + # + ptype = msg['header']['type'] + plugin = self.pmap.get(ptype, None) + if plugin is not None: + with self.lock: + try: + ret = plugin(msg, self.nl) + if ret is None: + # + # The packet is terminated in the plugin, + # return the NLMSG_ERR == 0 + # + # FIXME: optimize + # + newmsg = struct.pack('IHH', 40, 2, 0) + newmsg += msg.data[8:16] + newmsg += struct.pack('I', 0) + # nlmsgerr struct alignment + newmsg += b'\0' * 20 + return {'verdict': self.policy, 'data': newmsg} + else: + return ret + + except Exception as e: + log.error(''.join(traceback.format_stack())) + log.error(traceback.format_exc()) + # errmsg + if isinstance(e, (OSError, IOError)): + code = e.errno + elif isinstance(e, NetlinkError): + code = e.code + else: + code = errno.ECOMM + newmsg = struct.pack('HH', 2, 0) + newmsg += msg.data[8:16] + newmsg += struct.pack('I', code) + newmsg += msg.data + newmsg = struct.pack('I', len(newmsg) + 4) + newmsg + return {'verdict': 'error', 'data': newmsg} + return None diff --git a/pyroute2/netlink/rt_files.py b/pyroute2/netlink/rt_files.py deleted file mode 100644 index 5f3aaa967..000000000 --- a/pyroute2/netlink/rt_files.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Rt files parser - -iproute2 got lot of "map" files, called rt_xxx for most of them, -this module is an helper for all files -""" - -from dataclasses import dataclass, field, fields -from pathlib import Path -from typing import Union - - -@dataclass -class IPRouteRtFile: - filename: str - - id2name: dict[int, Union[str, int]] = field(default_factory=dict) - name2id: dict[str, int] = field(default_factory=dict) - - # like iproute2 stop at first existing directory - DIRECTORIES = (Path("/etc/iproute2/"), Path("/usr/share/iproute2/")) - - def __post_init__(self): - self.load_files() - - @classmethod - def get_rt_filename(cls): - """Helper to get filename when dataclass is not instancied""" - for rt_field in fields(cls): - if rt_field.name == 'filename': - return rt_field.default - raise KeyError(f"no filename in {cls}") - - def _iter_files(self, filepath): - d_folder = Path(f'{filepath}.d') - if filepath.exists(): - yield filepath - if d_folder.exists(): - yield from (p for p in d_folder.iterdir() if p.suffix == '.conf') - - def iter_files(self): - next_folder = True - for folder in self.DIRECTORIES: - for filepath in self._iter_files(folder / self.filename): - next_folder = False - yield filepath - if not next_folder: - return - - def load_files(self): - self.id2name = {} - self.name2id = {} - - for filename in self.iter_files(): - with filename.open(encoding='utf-8') as fp: - for line in fp.readlines(): - line = line.strip() - if not line or line[0] == '#': - continue - rt_id_as_str, rt_name = line.split() - - if rt_id_as_str.startswith("0x"): - rt_id = int(rt_id_as_str[2:], 16) - elif ':' in rt_id_as_str: - # tc handle as class_id string - (major, minor) = [ - int(x if x else '0', 16) - for x in rt_id_as_str.split(':') - ] - rt_id = (major << 16) | minor - else: - rt_id = int(rt_id_as_str) - - if rt_id in self.id2name: - continue # Accept only one rt_name by rt_id - self.id2name[rt_id] = rt_name - self.name2id[rt_name] = rt_id - - def get_rt_id( - self, rt_name: Union[str, int], default: Union[int, None] = None - ) -> Union[int, None]: - """Return id from the name. - if rt_name is an int or digits() return it as int - """ - if isinstance(rt_name, int): - return rt_name - if rt_name.isdigit(): - return int(rt_name) - if default is None: - return self.name2id[rt_name] - return self.name2id.get(rt_name, default) - - def get_rt_name( - self, rt_id: Union[str, int], default: Union[str, None] = None - ) -> Union[int, str, None]: - """Return name from the id. - name not found return id as str - if the id is already a string return it - """ - if isinstance(rt_id, str): - return rt_id - if default is None: - return self.id2name.get(rt_id, rt_id) - return self.id2name.get(rt_id, default) - - def __iter__(self): - yield from self.id2name.items() - - -@dataclass -class EmatchMapFile(IPRouteRtFile): - filename: str = 'ematch_map' - - -@dataclass -class NlProtosFile(IPRouteRtFile): - filename: str = 'nl_protos' - - -@dataclass -class RtAddrProtosFile(IPRouteRtFile): - filename: str = 'rt_addrprotos' - - -@dataclass -class RtDsfieldFile(IPRouteRtFile): - filename: str = 'rt_dsfield' - - -@dataclass -class RtGroupFile(IPRouteRtFile): - filename: str = 'rt_group' - - -@dataclass -class RtProtosFile(IPRouteRtFile): - filename: str = 'rt_protos' - - -@dataclass -class RtRealmsFile(IPRouteRtFile): - filename: str = 'rt_realms' - - -@dataclass -class RtScopesFile(IPRouteRtFile): - filename: str = 'rt_scopes' - - -@dataclass -class RtTablesFile(IPRouteRtFile): - filename: str = 'rt_tables' - - -@dataclass -class TcClsFile(IPRouteRtFile): - filename: str = 'tc_cls' - - -def main(cls_list): - for cls in cls_list.values(): - try: - assert issubclass(cls, IPRouteRtFile) and cls is not IPRouteRtFile - except (TypeError, AssertionError): - continue - - cls = cls() - print(f"==== Show rt maps for: {cls.filename} ===") - print( - "get_rt_id('default') ==", cls.get_rt_id("default", "pas trouvé") - ) - for id, name in cls: - print(f"{id} {name}") - - -if __name__ == "__main__": - main(locals()) diff --git a/pyroute2/netlink/rtnl/__init__.py b/pyroute2/netlink/rtnl/__init__.py index 2f47686f8..0859046d8 100644 --- a/pyroute2/netlink/rtnl/__init__.py +++ b/pyroute2/netlink/rtnl/__init__.py @@ -37,10 +37,6 @@ from pyroute2.common import map_namespace -# Useful RTNL related constants from the kernel sources: -IFNAMSIZ = 16 -ALTIFNAMSIZ = 128 - # RTnetlink multicast group flags (for use with bind()) RTMGRP_NONE = 0x0 RTMGRP_LINK = 0x1 @@ -159,9 +155,6 @@ RTM_NEWNETNS = 500 RTM_DELNETNS = 501 RTM_GETNETNS = 502 -RTM_NEWPROBE = 504 -RTM_DELPROBE = 505 -RTM_GETPROBE = 506 (RTM_NAMES, RTM_VALUES) = map_namespace('RTM_', globals()) TC_H_INGRESS = 0xFFFFFFF1 diff --git a/pyroute2/netlink/rtnl/ifaddrmsg.py b/pyroute2/netlink/rtnl/ifaddrmsg.py index 229d78da8..3e1e379b5 100644 --- a/pyroute2/netlink/rtnl/ifaddrmsg.py +++ b/pyroute2/netlink/rtnl/ifaddrmsg.py @@ -73,9 +73,6 @@ class ifaddrmsg(nlmsg): ('IFA_CACHEINFO', 'cacheinfo'), ('IFA_MULTICAST', 'ipaddr'), ('IFA_FLAGS', 'uint32'), - ('IFA_RT_PRIORITY', 'uint32'), - ('IFA_TARGET_NETNSID', 'asciiz'), - ('IFA_PROTO', 'uint8'), ) class cacheinfo(nla): diff --git a/pyroute2/netlink/rtnl/ifinfmsg/__init__.py b/pyroute2/netlink/rtnl/ifinfmsg/__init__.py index cf23fe1db..e4f9e5744 100644 --- a/pyroute2/netlink/rtnl/ifinfmsg/__init__.py +++ b/pyroute2/netlink/rtnl/ifinfmsg/__init__.py @@ -346,14 +346,6 @@ class protinfo_bridge(nla): ('IFLA_BRPORT_BACKUP_PORT', 'uint32'), ('IFLA_BRPORT_MRP_RING_OPEN', 'uint8'), ('IFLA_BRPORT_MRP_IN_OPEN', 'uint8'), - ('IFLA_BRPORT_MCAST_EHT_HOSTS_LIMIT', 'uint32'), - ('IFLA_BRPORT_MCAST_EHT_HOSTS_CNT', 'uint32'), - ('IFLA_BRPORT_LOCKED', 'uint8'), - ('IFLA_BRPORT_MAB', 'uint8'), - ('IFLA_BRPORT_MCAST_N_GROUPS', 'uint32'), - ('IFLA_BRPORT_MCAST_MAX_GROUPS', 'uint32'), - ('IFLA_BRPORT_NEIGH_VLAN_SUPPRESS', 'uint8'), - ('IFLA_BRPORT_BACKUP_NHID', 'uint32'), ) class br_id(ifla_bridge_id): @@ -621,7 +613,7 @@ class xdp_fd(nlmsg_atoms.int32): sql_type = None class xdp_flags(nla): - fields = [('value', 'I')] + fields = [('value', '>H')] sql_type = 'INTEGER' def encode(self): @@ -810,7 +802,7 @@ class ifmap(nla): @staticmethod def protinfo(self, *argv, **kwarg): proto_map = {AF_BRIDGE: protinfo_bridge} - return proto_map[self['family']] + return proto_map.get(self['family'], self.hex) class ifinfo(nla): prefix = 'IFLA_INFO_' diff --git a/pyroute2/netlink/rtnl/ifinfmsg/compat.py b/pyroute2/netlink/rtnl/ifinfmsg/compat.py new file mode 100644 index 000000000..c8140d4e2 --- /dev/null +++ b/pyroute2/netlink/rtnl/ifinfmsg/compat.py @@ -0,0 +1,382 @@ +import json +import os +import subprocess + +from pyroute2.common import map_enoent +from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg +from pyroute2.netlink.rtnl.ifinfmsg.sync import sync +from pyroute2.netlink.rtnl.ifinfmsg.tuntap import manage_tuntap +from pyroute2.netlink.rtnl.marshal import MarshalRtnl + +# it's simpler to double constants here, than to change all the +# module layout; but it is a subject of the future refactoring +RTM_NEWLINK = 16 +RTM_DELLINK = 17 +# + +_BONDING_MASTERS = '/sys/class/net/bonding_masters' +_BONDING_SLAVES = '/sys/class/net/%s/bonding/slaves' +_BRIDGE_MASTER = '/sys/class/net/%s/brport/bridge/ifindex' +_BONDING_MASTER = '/sys/class/net/%s/master/ifindex' +IFNAMSIZ = 16 + + +def compat_fix_attrs(msg, nl): + kind = None + ifname = msg.get_attr('IFLA_IFNAME') + + # fix master + if not nl.capabilities['provide_master']: + master = compat_get_master(ifname) + if master is not None: + msg['attrs'].append(['IFLA_MASTER', master]) + + # fix linkinfo & kind + li = msg.get_attr('IFLA_LINKINFO') + if li is not None: + kind = li.get_attr('IFLA_INFO_KIND') + if kind is None: + kind = get_interface_type(ifname) + li['attrs'].append(['IFLA_INFO_KIND', kind]) + elif 'attrs' in msg: + kind = get_interface_type(ifname) + msg['attrs'].append( + ['IFLA_LINKINFO', {'attrs': [['IFLA_INFO_KIND', kind]]}] + ) + else: + return + + li = msg.get_attr('IFLA_LINKINFO') + # fetch specific interface data + + if (kind in ('bridge', 'bond')) and [ + x for x in li['attrs'] if x[0] == 'IFLA_INFO_DATA' + ]: + if kind == 'bridge': + t = '/sys/class/net/%s/bridge/%s' + ifdata = ifinfmsg.ifinfo.bridge_data + elif kind == 'bond': + t = '/sys/class/net/%s/bonding/%s' + ifdata = ifinfmsg.ifinfo.bond_data + + commands = [] + for cmd, _ in ifdata.nla_map: + try: + with open(t % (ifname, ifdata.nla2name(cmd)), 'r') as f: + value = f.read() + if cmd == 'IFLA_BOND_MODE': + value = value.split()[1] + commands.append([cmd, int(value)]) + except: + pass + if commands: + li['attrs'].append(['IFLA_INFO_DATA', {'attrs': commands}]) + + +def proxy_linkinfo(data, nl): + marshal = MarshalRtnl() + inbox = marshal.parse(data) + data = b'' + for msg in inbox: + if msg['event'] == 'NLMSG_ERROR': + data += msg.data + continue + # Sysfs operations can require root permissions, + # but the script can be run under a normal user + # Bug-Url: https://github.com/svinota/pyroute2/issues/113 + try: + compat_fix_attrs(msg, nl) + except OSError: + # We can safely ignore here any OSError. + # In the worst case, we just return what we have got + # from the kernel via netlink + pass + + msg.reset() + msg.encode() + data += msg.data + + return {'verdict': 'forward', 'data': data} + + +def proxy_setlink(imsg, nl): + def get_interface(index): + msg = nl.get_links(index)[0] + try: + kind = msg.get_attr('IFLA_LINKINFO').get_attr('IFLA_INFO_KIND') + except AttributeError: + kind = 'unknown' + return { + 'ifname': msg.get_attr('IFLA_IFNAME'), + 'master': msg.get_attr('IFLA_MASTER'), + 'kind': kind, + } + + msg = ifinfmsg(imsg.data) + msg.decode() + forward = True + + kind = None + infodata = None + + ifname = ( + msg.get_attr('IFLA_IFNAME') or get_interface(msg['index'])['ifname'] + ) + linkinfo = msg.get_attr('IFLA_LINKINFO') + if linkinfo: + kind = linkinfo.get_attr('IFLA_INFO_KIND') + infodata = linkinfo.get_attr('IFLA_INFO_DATA') + + if kind in ('bond', 'bridge') and infodata is not None: + code = 0 + # + if kind == 'bond': + func = compat_set_bond + elif kind == 'bridge': + func = compat_set_bridge + # + for cmd, value in infodata.get('attrs', []): + cmd = infodata.nla2name(cmd) + code = func(ifname, cmd, value) or code + # + if code: + err = OSError() + err.errno = code + raise err + + # is it a port setup? + master = msg.get_attr('IFLA_MASTER') + if master is not None: + if master == 0: + # port delete + # 1. get the current master + iface = get_interface(msg['index']) + master = get_interface(iface['master']) + cmd = 'del' + else: + # port add + # 1. get the master + master = get_interface(master) + cmd = 'add' + + # 2. manage the port + forward_map = { + 'team': manage_team_port, + 'bridge': compat_bridge_port, + 'bond': compat_bond_port, + } + if master['kind'] in forward_map: + func = forward_map[master['kind']] + forward = func(cmd, master['ifname'], ifname, nl) + + if forward is not None: + return {'verdict': 'forward', 'data': imsg.data} + + +def proxy_dellink(imsg, nl): + orig_msg = ifinfmsg(imsg.data) + orig_msg.decode() + + # get full interface description + msg = nl.get_links(orig_msg['index'])[0] + msg['header']['type'] = orig_msg['header']['type'] + + # get the interface kind + kind = None + li = msg.get_attr('IFLA_LINKINFO') + if li is not None: + kind = li.get_attr('IFLA_INFO_KIND') + + # team interfaces can be stopped by a normal RTM_DELLINK + if kind == 'bond' and not nl.capabilities['create_bond']: + return compat_del_bond(msg) + elif kind == 'bridge' and not nl.capabilities['create_bridge']: + return compat_del_bridge(msg) + + return {'verdict': 'forward', 'data': imsg.data} + + +def proxy_newlink(imsg, nl): + msg = ifinfmsg(imsg.data) + msg.decode() + kind = None + + # get the interface kind + linkinfo = msg.get_attr('IFLA_LINKINFO') + if linkinfo is not None: + kind = [x[1] for x in linkinfo['attrs'] if x[0] == 'IFLA_INFO_KIND'] + if kind: + kind = kind[0] + + if kind == 'tuntap': + return manage_tuntap(msg) + elif kind == 'team': + return manage_team(msg) + elif kind == 'bond' and not nl.capabilities['create_bond']: + return compat_create_bond(msg) + elif kind == 'bridge' and not nl.capabilities['create_bridge']: + return compat_create_bridge(msg) + + return {'verdict': 'forward', 'data': imsg.data} + + +@map_enoent +@sync +def manage_team(msg): + if msg['header']['type'] != RTM_NEWLINK: + raise ValueError('wrong command type') + + config = { + 'device': msg.get_attr('IFLA_IFNAME'), + 'runner': {'name': 'activebackup'}, + 'link_watch': {'name': 'ethtool'}, + } + + with open(os.devnull, 'w') as fnull: + subprocess.check_call( + ['teamd', '-d', '-n', '-c', json.dumps(config)], + stdout=fnull, + stderr=fnull, + ) + + +@map_enoent +def manage_team_port(cmd, master, ifname, nl): + with open(os.devnull, 'w') as fnull: + subprocess.check_call( + [ + 'teamdctl', + master, + 'port', + 'remove' if cmd == 'del' else 'add', + ifname, + ], + stdout=fnull, + stderr=fnull, + ) + + +@sync +def compat_create_bridge(msg): + name = msg.get_attr('IFLA_IFNAME') + with open(os.devnull, 'w') as fnull: + subprocess.check_call( + ['brctl', 'addbr', name], stdout=fnull, stderr=fnull + ) + + +@sync +def compat_create_bond(msg): + name = msg.get_attr('IFLA_IFNAME') + with open(_BONDING_MASTERS, 'w') as f: + f.write('+%s' % (name)) + + +def compat_set_bond(name, cmd, value): + # FIXME: join with bridge + # FIXME: use internal IO, not bash + t = 'echo %s >/sys/class/net/%s/bonding/%s' + with open(os.devnull, 'w') as fnull: + return subprocess.call( + ['bash', '-c', t % (value, name, cmd)], stdout=fnull, stderr=fnull + ) + + +def compat_set_bridge(name, cmd, value): + t = 'echo %s >/sys/class/net/%s/bridge/%s' + with open(os.devnull, 'w') as fnull: + return subprocess.call( + ['bash', '-c', t % (value, name, cmd)], stdout=fnull, stderr=fnull + ) + + +@sync +def compat_del_bridge(msg): + name = msg.get_attr('IFLA_IFNAME') + with open(os.devnull, 'w') as fnull: + subprocess.check_call(['ip', 'link', 'set', 'dev', name, 'down']) + subprocess.check_call( + ['brctl', 'delbr', name], stdout=fnull, stderr=fnull + ) + + +@sync +def compat_del_bond(msg): + name = msg.get_attr('IFLA_IFNAME') + subprocess.check_call(['ip', 'link', 'set', 'dev', name, 'down']) + with open(_BONDING_MASTERS, 'w') as f: + f.write('-%s' % (name)) + + +def compat_bridge_port(cmd, master, port, nl): + if nl.capabilities['create_bridge']: + return True + with open(os.devnull, 'w') as fnull: + subprocess.check_call( + ['brctl', '%sif' % (cmd), master, port], stdout=fnull, stderr=fnull + ) + + +def compat_bond_port(cmd, master, port, nl): + if nl.capabilities['create_bond']: + return True + remap = {'add': '+', 'del': '-'} + cmd = remap[cmd] + with open(_BONDING_SLAVES % (master), 'w') as f: + f.write('%s%s' % (cmd, port)) + + +def compat_get_master(name): + f = None + + for i in (_BRIDGE_MASTER, _BONDING_MASTER): + try: + try: + f = open(i % (name)) + except UnicodeEncodeError: + # a special case with python3 on Ubuntu 14 + f = open(i % (name.encode('utf-8'))) + break + except IOError: + pass + + if f is not None: + master = int(f.read()) + f.close() + return master + + +def get_interface_type(name): + ''' + Utility function to get interface type. + + Unfortunately, we can not rely on RTNL or even ioctl(). + RHEL doesn't support interface type in RTNL and doesn't + provide extended (private) interface flags via ioctl(). + + Args: + * name (str): interface name + + Returns: + * False -- sysfs info unavailable + * None -- type not known + * str -- interface type: + - 'bond' + - 'bridge' + ''' + # FIXME: support all interface types? Right now it is + # not needed + try: + ifattrs = os.listdir('/sys/class/net/%s/' % (name)) + except OSError as e: + if e.errno == 2: + return 'unknown' + else: + raise + + if 'bonding' in ifattrs: + return 'bond' + elif 'bridge' in ifattrs: + return 'bridge' + else: + return 'unknown' diff --git a/pyroute2/netlink/rtnl/ifinfmsg/proxy.py b/pyroute2/netlink/rtnl/ifinfmsg/proxy.py new file mode 100644 index 000000000..2dc1e4693 --- /dev/null +++ b/pyroute2/netlink/rtnl/ifinfmsg/proxy.py @@ -0,0 +1,123 @@ +import json +import os +import subprocess + +from pyroute2.common import map_enoent +from pyroute2.netlink.rtnl.ifinfmsg import RTM_NEWLINK +from pyroute2.netlink.rtnl.ifinfmsg.sync import sync +from pyroute2.netlink.rtnl.ifinfmsg.tuntap import manage_tun, manage_tuntap + +_BONDING_MASTERS = '/sys/class/net/bonding_masters' +_BONDING_SLAVES = '/sys/class/net/%s/bonding/slaves' +_BRIDGE_MASTER = '/sys/class/net/%s/brport/bridge/ifindex' +_BONDING_MASTER = '/sys/class/net/%s/master/ifindex' +IFNAMSIZ = 16 + + +def proxy_setlink(msg, nl): + def get_interface(index): + msg = nl.get_links(index)[0] + try: + kind = msg.get_attr('IFLA_LINKINFO').get_attr('IFLA_INFO_KIND') + except AttributeError: + kind = 'unknown' + return { + 'ifname': msg.get_attr('IFLA_IFNAME'), + 'master': msg.get_attr('IFLA_MASTER'), + 'kind': kind, + } + + forward = True + + # is it a port setup? + master = msg.get_attr('IFLA_MASTER') + if master is not None: + if master == 0: + # port delete + # 1. get the current master + iface = get_interface(msg['index']) + master = get_interface(iface['master']) + cmd = 'del' + else: + # port add + # 1. get the master + master = get_interface(master) + cmd = 'add' + + ifname = ( + msg.get_attr('IFLA_IFNAME') + or get_interface(msg['index'])['ifname'] + ) + + # 2. manage the port + forward_map = {'team': manage_team_port} + if master['kind'] in forward_map: + func = forward_map[master['kind']] + forward = func(cmd, master['ifname'], ifname, nl) + + if forward is not None: + return {'verdict': 'forward', 'data': msg.data} + + +def proxy_newlink(msg, nl): + kind = None + + # get the interface kind + linkinfo = msg.get_attr('IFLA_LINKINFO') + if linkinfo is not None: + kind = [x[1] for x in linkinfo['attrs'] if x[0] == 'IFLA_INFO_KIND'] + if kind: + kind = kind[0] + + if kind == 'tuntap': + return manage_tuntap(msg) + elif kind == 'tun': + return manage_tun(msg) + elif kind == 'team': + return manage_team(msg) + + return {'verdict': 'forward', 'data': msg.data} + + +@map_enoent +@sync +def manage_team(msg): + if msg['header']['type'] != RTM_NEWLINK: + raise ValueError('wrong command type') + + try: + linkinfo = msg.get_attr('IFLA_LINKINFO') + infodata = linkinfo.get_attr('IFLA_INFO_DATA') + config = infodata.get_attr('IFLA_TEAM_CONFIG') + config = json.loads(config) + except AttributeError: + config = { + 'runner': {'name': 'activebackup'}, + 'link_watch': {'name': 'ethtool'}, + } + + # fix device + config['device'] = msg.get_attr('IFLA_IFNAME') + + with open(os.devnull, 'w') as fnull: + subprocess.check_call( + ['teamd', '-d', '-n', '-c', json.dumps(config)], + stdout=fnull, + stderr=fnull, + ) + + +@map_enoent +def manage_team_port(cmd, master, ifname, nl): + with open(os.devnull, 'w') as fnull: + subprocess.check_call( + [ + 'teamdctl', + master, + 'port', + 'remove' if cmd == 'del' else 'add', + ifname, + ], + stdout=fnull, + stderr=fnull, + ) diff --git a/pyroute2/netlink/rtnl/ifinfmsg/sync.py b/pyroute2/netlink/rtnl/ifinfmsg/sync.py new file mode 100644 index 000000000..feff27175 --- /dev/null +++ b/pyroute2/netlink/rtnl/ifinfmsg/sync.py @@ -0,0 +1,70 @@ +import os +import select +import threading + +from pyroute2.netlink.rtnl import RTM_VALUES +from pyroute2.netlink.rtnl.riprsocket import RawIPRSocket + + +def sync(f): + ''' + A decorator to wrap up external utility calls. + + A decorated function receives a netlink message + as a parameter, and then: + + 1. Starts a monitoring thread + 2. Performs the external call + 3. Waits for a netlink event specified by `msg` + 4. Joins the monitoring thread + + If the wrapped function raises an exception, the + monitoring thread will be forced to stop via the + control channel pipe. The exception will be then + forwarded. + ''' + + def monitor(event, ifname, cmd): + with RawIPRSocket() as ipr: + poll = select.poll() + poll.register(ipr, select.POLLIN | select.POLLPRI) + poll.register(cmd, select.POLLIN | select.POLLPRI) + ipr.bind() + while True: + events = poll.poll() + for fd, event in events: + if fd == ipr.fileno(): + msgs = ipr.get() + for msg in msgs: + if ( + msg.get('event') == event + and msg.get_attr('IFLA_IFNAME') == ifname + ): + return + else: + return + + def decorated(msg): + rcmd, cmd = os.pipe() + t = threading.Thread( + target=monitor, + args=( + RTM_VALUES[msg['header']['type']], + msg.get_attr('IFLA_IFNAME'), + rcmd, + ), + ) + t.start() + ret = None + try: + ret = f(msg) + except Exception: + raise + finally: + os.write(cmd, b'q') + t.join() + os.close(rcmd) + os.close(cmd) + return ret + + return decorated diff --git a/pyroute2/netlink/rtnl/ifinfmsg/tuntap.py b/pyroute2/netlink/rtnl/ifinfmsg/tuntap.py index e77e4c494..1ade786e4 100644 --- a/pyroute2/netlink/rtnl/ifinfmsg/tuntap.py +++ b/pyroute2/netlink/rtnl/ifinfmsg/tuntap.py @@ -2,10 +2,8 @@ import os import struct from fcntl import ioctl -from typing import Optional from pyroute2 import config -from pyroute2.netlink import nlmsg from pyroute2.netlink.exceptions import NetlinkError from pyroute2.netlink.rtnl.ifinfmsg import ( IFT_MULTI_QUEUE, @@ -16,6 +14,7 @@ IFT_VNET_HDR, RTM_NEWLINK, ) +from pyroute2.netlink.rtnl.ifinfmsg.sync import sync IFNAMSIZ = 16 @@ -30,7 +29,6 @@ 'aarch64', 'loongarch64', ) -TUNSETIFF: Optional[int] = None if config.machine in PLATFORMS: TUNSETIFF = 0x400454CA TUNSETPERSIST = 0x400454CB @@ -41,9 +39,12 @@ TUNSETPERSIST = 0x800454CB TUNSETOWNER = 0x800454CC TUNSETGROUP = 0x800454CE +else: + TUNSETIFF = None -def manage_tun(msg: nlmsg) -> bytes: +@sync +def manage_tun(msg): if TUNSETIFF is None: raise NetlinkError(errno.EOPNOTSUPP, 'Arch not supported') @@ -89,10 +90,10 @@ def manage_tun(msg: nlmsg) -> bytes: raise finally: os.close(fd) - return b'' -def manage_tuntap(msg: nlmsg) -> bytes: +@sync +def manage_tuntap(msg): if TUNSETIFF is None: raise NetlinkError(errno.EOPNOTSUPP, 'Arch not supported') @@ -141,4 +142,3 @@ def manage_tuntap(msg: nlmsg) -> bytes: raise finally: os.close(fd) - return b'' diff --git a/pyroute2/netlink/rtnl/iprsocket.py b/pyroute2/netlink/rtnl/iprsocket.py index d7c618464..5dc3b4877 100644 --- a/pyroute2/netlink/rtnl/iprsocket.py +++ b/pyroute2/netlink/rtnl/iprsocket.py @@ -1,296 +1,99 @@ import errno -import os -import struct -from collections.abc import Callable -from typing import Optional -from unittest import mock +import sys +import types -from pyroute2 import config -from pyroute2.common import msg_done -from pyroute2.iproute.ipmock import IPEngine -from pyroute2.netlink import NETLINK_ROUTE, NLM_F_REQUEST, nlmsg, rtnl -from pyroute2.netlink.exceptions import NetlinkError +from pyroute2.common import DEFAULT_RCVBUF, AddrPool, Namespace +from pyroute2.netlink import NETLINK_ROUTE, rtnl from pyroute2.netlink.nlsocket import ( - AsyncNetlinkSocket, + BatchSocket, ChaoticNetlinkSocket, - NetlinkRequest, NetlinkSocket, ) -from pyroute2.netlink.rtnl.ifinfmsg.tuntap import manage_tun, manage_tuntap +from pyroute2.netlink.proxy import NetlinkProxy from pyroute2.netlink.rtnl.marshal import MarshalRtnl -from pyroute2.netlink.rtnl.probe_msg import proxy_newprobe -from pyroute2.netns import setns -from pyroute2.process import ChildProcess, ChildProcessReturnValue +if sys.platform.startswith('linux'): + from pyroute2.netlink.rtnl.ifinfmsg.proxy import ( + proxy_newlink, + proxy_setlink, + ) -def _run_in_netns( - netns: str, target: Callable[[nlmsg], bytes], msg: nlmsg -) -> ChildProcessReturnValue: - setns(netns) - return ChildProcessReturnValue(target(msg), []) +class IPRSocketBase(object): + def __init__(self, *argv, **kwarg): + if 'family' in kwarg: + kwarg.pop('family') + super(IPRSocketBase, self).__init__(NETLINK_ROUTE, *argv[1:], **kwarg) + self.marshal = MarshalRtnl() + if self.groups == 0: + self.groups = rtnl.RTMGRP_DEFAULTS + self._s_channel = None + if sys.platform.startswith('linux'): + send_ns = Namespace( + self, + {'addr_pool': AddrPool(0x10000, 0x1FFFF), 'monitor': False}, + ) + self._sproxy = NetlinkProxy(policy='return', nl=send_ns) + self._sproxy.pmap = { + rtnl.RTM_NEWLINK: proxy_newlink, + rtnl.RTM_SETLINK: proxy_setlink, + } + + def bind(self, groups=None, **kwarg): + super(IPRSocketBase, self).bind( + groups if groups is not None else self.groups, **kwarg + ) -class IPRouteProxy: - route: dict[ - int, - tuple[tuple[Callable[[nlmsg], bool], Callable[[nlmsg], bytes]], ...], - ] = { - rtnl.RTM_NEWLINK: ( - (lambda x: x.get(('linkinfo', 'kind')) == 'tuntap', manage_tuntap), - (lambda x: x.get(('linkinfo', 'kind')) == 'tun', manage_tun), - ), - rtnl.RTM_NEWPROBE: ((lambda x: True, proxy_newprobe),), - } - - def __init__(self, netns: Optional[str] = None): - self.netns = netns - - def handle(self, msg: nlmsg) -> bytes: - ret: bytes = b'' - key = msg['header']['type'] - if key not in self.route: - return ret - for predicate, target in self.route[key]: - if predicate(msg): - try: - if self.netns is not None: - with ChildProcess( - target=_run_in_netns, - args=[self.netns, target, msg], - ) as proc: - ret = proc.get_data(timeout=4) - else: - ret = target(msg) - return ret or msg_done(msg) - except Exception as e: - # errmsg - if isinstance(e, (OSError, IOError)): - code = e.errno or errno.ENODATA - elif isinstance(e, NetlinkError): - code = e.code - else: - code = errno.ECOMM - newmsg = struct.pack('HH', 2, 0) - newmsg += msg.data[8:16] - newmsg += struct.pack('I', code) - newmsg += msg.data - newmsg = struct.pack('I', len(newmsg) + 4) + newmsg - return newmsg - return b'' - - -class AsyncIPRSocket(AsyncNetlinkSocket): - '''A low-level class to provide RTNL socket. - - This is a low-level class designed to provide an RTNL - asyncio-controlled socket. It does not include high-level - methods like those found in AsyncIPRoute. Instead, it provides - only common netlink methods such as `get()` and `put()`. For - more details, refer to the `AsyncNetlinkSocket` documentation. - - .. testcode:: - :hide: - - from pyroute2 import AsyncIPRSocket - - iprsock = AsyncIPRSocket() - assert callable(iprsock.get) - assert callable(iprsock.put) - assert callable(iprsock.nlm_request) - assert callable(iprsock.bind) - - Since the underlying socket is controlled by asyncio, it is - not possible to use it in poll/select loops. If you want - such API, consider using synchronous `IPRSocket`. - - .. warning:: - - Your code must process incoming messages quickly enough to - prevent the RCVBUF from overflowing. If the RCVBUF overflows, - all subsequent socket operations will raise an OSError: - - .. code:: - - >>> [ x async for x in ipr.get() ] - Traceback (most recent call last): - File ".../python3.13/futures/_base.py", line 456, in result - return self.__get_result() - ~~~~~~~~~~~~~~~~~^^ - ... - OSError: [Errno 105] No buffer space available - - - If this exception occurs, the only solution is to close the - socket and create a new one. - - This class does not handle protocol-level error propagation; it - only provides socket-level error handling. It is the user's - responsibility to catch and manage protocol-level errors: - - .. testsetup:: as0 - - from pyroute2.netlink import nlmsgerr, NLMSG_ERROR - msg = nlmsgerr() - msg['header']['type'] = NLMSG_ERROR - msg['error'] = 42 + def sendto_gate(self, msg, addr): msg.reset() msg.encode() - msg.decode() - - .. testcode:: as0 - - if msg.get(('header', 'type')) == NLMSG_ERROR: - # prints error code and the request that - # triggered the error - print( - msg.get('error'), - msg.get('msg'), - ) + if self.compiled is not None: + return self.compiled.append(msg.data) + ret = self._sproxy.handle(msg) + if ret is not None: + if ret['verdict'] == 'forward': + return self._sendto(ret['data'], addr) + elif ret['verdict'] in ('return', 'error'): + if self._s_channel is not None: + return self._s_channel.send(ret['data']) + else: + msgs = self.marshal.parse(ret['data']) + for msg in msgs: + seq = msg['header']['sequence_number'] + if seq in self.backlog: + self.backlog[seq].append(msg) + else: + self.backlog[seq] = [msg] + return len(ret['data']) + else: + ValueError('Incorrect verdict') + + return self._sendto(msg.data, addr) + + +class IPBatchSocket(IPRSocketBase, BatchSocket): + pass - .. testoutput:: as0 - :hide: - 42 None +class ChaoticIPRSocket(IPRSocketBase, ChaoticNetlinkSocket): + pass +class IPRSocket(IPRSocketBase, NetlinkSocket): ''' + The simplest class, that connects together the netlink parser and + a generic Python socket implementation. Provides method get() to + receive the next message from netlink socket and parse it. It is + just simple socket-like class, it implements no buffering or + like that. It spawns no additional threads, leaving this up to + developers. - def __init__( - self, - port=None, - pid=None, - fileno=None, - sndbuf=1048576, - rcvbuf=1048576, - rcvsize=16384, - all_ns=False, - async_qsize=None, - nlm_generator=None, - target='localhost', - ext_ack=False, - strict_check=False, - groups=rtnl.RTMGRP_DEFAULTS, - nlm_echo=False, - netns=None, - netns_path=None, - flags=os.O_CREAT, - libc=None, - use_socket=None, - use_event_loop=None, - telemetry=None, - ): - if config.mock_netlink: - use_socket = IPEngine() - if netns is not None: - use_socket.netns = netns - use_socket.flags = flags - use_socket.initdb() - self.marshal = MarshalRtnl() - super().__init__( - family=NETLINK_ROUTE, - port=port, - pid=pid, - fileno=fileno, - sndbuf=sndbuf, - rcvbuf=rcvbuf, - rcvsize=rcvsize, - all_ns=all_ns, - async_qsize=async_qsize, - nlm_generator=nlm_generator, - target=target, - ext_ack=ext_ack, - strict_check=strict_check, - groups=groups, - nlm_echo=nlm_echo, - use_socket=use_socket, - netns=netns, - flags=flags, - libc=libc, - use_event_loop=use_event_loop, - telemetry=telemetry, - ) - if not config.mock_netlink: - self.request_proxy = IPRouteProxy(netns) - self.status['netns_path'] = netns_path or config.netns_path - - async def bind(self, groups=None, **kwarg): - return await super().bind( - groups if groups is not None else self.status['groups'], **kwarg - ) - - -class NotLocal: - event_loop = None - msg_queue = mock.Mock() - + Please note, that netlink is an asynchronous protocol with + non-guaranteed delivery. You should be fast enough to get all the + messages in time. If the message flow rate is higher than the + speed you parse them with, exceeding messages will be dropped. -class IPRSocket(NetlinkSocket): - '''Synchronous select-compatible netlink socket. - - `IPRSocket` is the synchronous counterpart to `AsyncIPRSocket`. - A key feature of `IPRSocket` is that the underlying netlink - socket operates out of asyncio control, allowing it to be - used in poll/select loops. - - .. warning:: - - Your code must process incoming messages quickly enough to - prevent the RCVBUF from overflowing. If the RCVBUF overflows, - all subsequent socket operations will raise an OSError: - - .. code:: - - >>> iprsock.get() - Traceback (most recent call last): - File "", line 1, in - iprsock.get() - ~~~~~~~~^^ - File ".../pyroute2/netlink/rtnl/iprsocket.py", line 276, in get - data = self.socket.recv(16384) - OSError: [Errno 105] No buffer space available - >>> - - If this exception occurs, the only solution is to close the - socket and create a new one. - - Some usage examples: - - .. testcode:: - - import select - - from pyroute2 import IPRSocket - from pyroute2.netlink import NLM_F_DUMP, NLM_F_REQUEST - from pyroute2.netlink.rtnl import RTM_GETLINK - from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg - - with IPRSocket() as iprsock: - iprsock.put( - ifinfmsg(), - msg_type=RTM_GETLINK, - msg_flags=NLM_F_REQUEST | NLM_F_DUMP - ) - - ret = [] - - while True: - rl, wl, xl = select.select([iprsock], [], [], 0) - if not len(rl): - break - ret.extend(iprsock.get()) - - for link in ret: - if link.get('event') == 'RTM_NEWLINK': - print( - link.get('ifname'), - link.get('state'), - link.get('address'), - ) - - .. testoutput:: - - lo up 00:00:00:00:00:00 - eth0 up 52:54:00:72:58:b2 + *Usage* Threadless RT netlink monitoring with blocking I/O calls: @@ -300,118 +103,67 @@ class IPRSocket(NetlinkSocket): >>> s.bind() >>> pprint(s.get()) [{'attrs': [('RTA_TABLE', 254), + ('RTA_DST', '2a00:1450:4009:808::1002'), + ('RTA_GATEWAY', 'fe80:52:0:2282::1fe'), ('RTA_OIF', 2), - ('RTA_GATEWAY', '192.168.122.1')], - 'dst_len': 0, - 'event': 'RTM_NEWROUTE', - 'family': 2, - 'flags': 0, + ('RTA_PRIORITY', 0), + ('RTA_CACHEINFO', {'rta_clntref': 0, + 'rta_error': 0, + 'rta_expires': 0, + 'rta_id': 0, + 'rta_lastuse': 5926, + 'rta_ts': 0, + 'rta_tsage': 0, + 'rta_used': 1})], + 'dst_len': 128, + 'event': 'RTM_DELROUTE', + 'family': 10, + 'flags': 512, 'header': {'error': None, - 'flags': 2, - 'length': 52, - 'pid': 325359, - 'sequence_number': 255, - 'type': 24}, - 'proto': 2, + 'flags': 0, + 'length': 128, + 'pid': 0, + 'sequence_number': 0, + 'type': 25}, + 'proto': 9, 'scope': 0, 'src_len': 0, 'table': 254, 'tos': 0, - 'type': 2}] + 'type': 1}] >>> - - Like `AsyncIPRSocket`, it does not perform response reassembly, - protocol-level error propagation, or packet buffering. ''' - def __init__( - self, - port=None, - pid=None, - fileno=None, - sndbuf=1048576, - rcvbuf=1048576, - rcvsize=16384, - all_ns=False, - async_qsize=None, - nlm_generator=None, - target='localhost', - ext_ack=False, - strict_check=False, - groups=rtnl.RTMGRP_DEFAULTS, - nlm_echo=False, - netns=None, - netns_path=None, - flags=os.O_CREAT, - libc=None, - use_socket=None, - use_event_loop=None, - ): - self.asyncore = AsyncIPRSocket( - port=port, - pid=pid, - fileno=fileno, - sndbuf=sndbuf, - rcvbuf=rcvbuf, - rcvsize=rcvsize, - all_ns=all_ns, - target=target, - ext_ack=ext_ack, - strict_check=strict_check, - groups=groups, - nlm_echo=nlm_echo, - netns=netns, - netns_path=netns_path, - flags=flags, - libc=libc, - use_socket=use_socket, - use_event_loop=use_event_loop, - ) - self.asyncore.local = NotLocal() - - @property - def addr_pool(self): - return self.asyncore.addr_pool - - @property - def socket(self): - return self.asyncore.socket - - @property - def fileno(self): - return self.asyncore.local.socket.fileno - - def bind(self, groups=None, pid=None, **kwarg): - self.asyncore._sync_bind(groups, pid, **kwarg) - - def put( - self, - msg, - msg_type, - msg_flags=NLM_F_REQUEST, - addr=(0, 0), - msg_seq=0, - msg_pid=None, - ): - if msg is None: - msg_class = self.marshal.msg_map[msg_type] - msg = msg_class() - - request = NetlinkRequest( - self, - msg, - msg_type=msg_type, - msg_flags=msg_flags, - msg_seq=msg_seq, - msg_pid=msg_pid, - ) - request.msg.encode() - return request.sock.send(request.msg.data) - - def get(self, msg_seq=0, terminate=None, callback=None, noraise=False): - data = self.socket.recv(16384) - return [x for x in self.marshal.parse(data)] - + _brd_socket = None + + def bind(self, *argv, **kwarg): + if kwarg.pop('clone_socket', False): + self._brd_socket = self.clone() + + def get( + self, + bufsize=DEFAULT_RCVBUF, + msg_seq=0, + terminate=None, + callback=None, + ): + if msg_seq == 0: + return self._brd_socket.get( + bufsize, msg_seq, terminate, callback + ) + else: + return super(IPRSocket, self).get( + bufsize, msg_seq, terminate, callback + ) -class ChaoticIPRSocket(AsyncIPRSocket, ChaoticNetlinkSocket): - pass + def close(self, code=errno.ECONNRESET): + with self.sys_lock: + self._brd_socket.close() + return super(IPRSocket, self).close(code=code) + + self.get = types.MethodType(get, self) + self.close = types.MethodType(close, self) + kwarg['recursive'] = True + return self._brd_socket.bind(*argv, **kwarg) + else: + return super(IPRSocket, self).bind(*argv, **kwarg) diff --git a/pyroute2/netlink/rtnl/marshal.py b/pyroute2/netlink/rtnl/marshal.py index 40431dc54..448b8bfea 100644 --- a/pyroute2/netlink/rtnl/marshal.py +++ b/pyroute2/netlink/rtnl/marshal.py @@ -7,7 +7,6 @@ from pyroute2.netlink.rtnl.ndmsg import ndmsg from pyroute2.netlink.rtnl.ndtmsg import ndtmsg from pyroute2.netlink.rtnl.nsidmsg import nsidmsg -from pyroute2.netlink.rtnl.probe_msg import probe_msg from pyroute2.netlink.rtnl.rtmsg import rtmsg from pyroute2.netlink.rtnl.tcmsg import tcmsg @@ -49,7 +48,6 @@ class MarshalRtnl(Marshal): rtnl.RTM_GETSTATS: ifstatsmsg, rtnl.RTM_NEWLINKPROP: ifinfmsg, rtnl.RTM_DELLINKPROP: ifinfmsg, - rtnl.RTM_NEWPROBE: probe_msg, } def fix_message(self, msg): diff --git a/pyroute2/netlink/rtnl/probe_msg.py b/pyroute2/netlink/rtnl/probe_msg.py deleted file mode 100644 index 71245026e..000000000 --- a/pyroute2/netlink/rtnl/probe_msg.py +++ /dev/null @@ -1,133 +0,0 @@ -import errno -import json -import shutil -import socket -import ssl -import subprocess - -from pyroute2.netlink import nlmsg -from pyroute2.netlink.exceptions import NetlinkError - - -class probe_msg(nlmsg): - ''' - Fake message type to represent network probe info. - - This is a prototype, the NLA layout is subject to change without - notification. - ''' - - __slots__ = () - prefix = 'PROBE_' - - fields = (('family', 'B'), ('proto', 'B'), ('port', 'H'), ('dst_len', 'I')) - - nla_map = ( - ('PROBE_UNSPEC', 'none'), - ('PROBE_KIND', 'asciiz'), - ('PROBE_STDOUT', 'asciiz'), - ('PROBE_STDERR', 'asciiz'), - ('PROBE_SRC', 'asciiz'), - ('PROBE_DST', 'asciiz'), - ('PROBE_NUM', 'uint8'), - ('PROBE_TIMEOUT', 'uint8'), - ('PROBE_HOSTNAME', 'asciiz'), - ('PROBE_SSL_VERIFY', 'uint8'), - ('PROBE_SSL_VERSION', 'asciiz'), - ('PROBE_SSL_CERT_JSON', 'asciiz'), - ('PROBE_SSL_CERT_DER', 'cdata'), - ) - - -def probe_ping(msg): - num = msg.get('num') - timeout = msg.get('timeout') - dst = msg.get('dst') - kind = msg.get('kind') - args = [shutil.which(kind), '-c', f'{num}', '-W', f'{timeout}', f'{dst}'] - if args[0] is None: - raise NetlinkError(errno.ENOENT, 'probe not found') - - process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - try: - out, err = process.communicate(timeout=timeout) - if out: - msg['attrs'].append(['PROBE_STDOUT', out]) - if err: - msg['attrs'].append(['PROBE_STDERR', err]) - except subprocess.TimeoutExpired: - process.terminate() - raise NetlinkError(errno.ETIMEDOUT, 'timeout expired') - finally: - process.stdout.close() - process.stderr.close() - return_code = process.wait() - if return_code != 0: - raise NetlinkError(errno.EHOSTUNREACH, 'probe failed') - - -def probe_tcp(msg, close=True): - timeout = msg.get('timeout') - dst = msg.get('dst') - port = msg.get('port') - connection = None - try: - connection = socket.create_connection((dst, port), timeout=timeout) - except ConnectionRefusedError: - raise NetlinkError(errno.ECONNREFUSED, 'connection refused') - except TimeoutError: - raise NetlinkError(errno.ETIMEDOUT, 'timeout expired') - except Exception: - raise NetlinkError(errno.ECOMM, 'probe failed') - finally: - if close and connection is not None: - connection.close() - return connection - - -def probe_ssl(msg): - hostname = msg.get('hostname') or msg.get('dst') - context = ssl.create_default_context() - context.verify_mode = msg.get('ssl_verify', ssl.CERT_REQUIRED) - with probe_tcp(msg, close=False) as connection: - try: - with context.wrap_socket( - connection, server_hostname=hostname - ) as ssl_wrap: - version = ssl_wrap.version() - peer_cert_json = ssl_wrap.getpeercert(binary_form=False) - peer_cert_der = ssl_wrap.getpeercert(binary_form=True) - if peer_cert_json is not None: - msg['attrs'].append( - ['PROBE_SSL_CERT_JSON', json.dumps(peer_cert_json)] - ) - if peer_cert_der is not None: - msg['attrs'].append(['PROBE_SSL_CERT_DER', peer_cert_der]) - if version is not None: - msg['attrs'].append(['PROBE_SSL_VERSION', version]) - except ssl.SSLError as e: - code = errno.EPROTO - if e.reason == 'UNSUPPORTED_PROTOCOL': - code = errno.EPROTONOSUPPORT - elif e.reason == 'CERTIFICATE_VERIFY_FAILED': - code = errno.EACCES - raise NetlinkError(code, e.strerror) - except Exception: - raise NetlinkError(errno.ECOMM, 'probe failed') - - -def proxy_newprobe(msg): - kind = msg.get('kind') - if kind.startswith('ping'): - probe_ping(msg) - elif kind == 'tcp': - probe_tcp(msg) - elif kind == 'ssl': - probe_ssl(msg) - else: - raise NetlinkError(errno.ENOTSUP, 'probe type not supported') - msg.reset() - msg.encode() - return msg.data diff --git a/pyroute2/netlink/rtnl/riprsocket.py b/pyroute2/netlink/rtnl/riprsocket.py new file mode 100644 index 000000000..fea8987de --- /dev/null +++ b/pyroute2/netlink/rtnl/riprsocket.py @@ -0,0 +1,16 @@ +from pyroute2.netlink import NETLINK_ROUTE, rtnl +from pyroute2.netlink.nlsocket import NetlinkSocket +from pyroute2.netlink.rtnl.marshal import MarshalRtnl + + +class RawIPRSocketBase(object): + def __init__(self, fileno=None): + super(RawIPRSocketBase, self).__init__(NETLINK_ROUTE, fileno=fileno) + self.marshal = MarshalRtnl() + + def bind(self, groups=rtnl.RTMGRP_DEFAULTS, **kwarg): + super(RawIPRSocketBase, self).bind(groups, **kwarg) + + +class RawIPRSocket(RawIPRSocketBase, NetlinkSocket): + pass diff --git a/pyroute2/netlink/rtnl/rtmsg.py b/pyroute2/netlink/rtnl/rtmsg.py index 9f8d3e085..509af2f2b 100644 --- a/pyroute2/netlink/rtnl/rtmsg.py +++ b/pyroute2/netlink/rtnl/rtmsg.py @@ -3,7 +3,6 @@ from pyroute2.common import AF_MPLS, hexdump, map_namespace from pyroute2.netlink import nla, nla_string, nlmsg -from pyroute2.netlink.rtnl import rt_proto, rt_scope, rt_type RTNH_F_DEAD = 1 RTNH_F_PERVASIVE = 2 @@ -100,15 +99,6 @@ class rtmsg_base(nlflags): 'RTA_NEWDST': "NOT NULL DEFAULT ''", } - defaults = { - 'family': AF_INET, - 'scope': rt_scope['universe'], - 'type': rt_type['unspec'], - 'proto': rt_proto['unspec'], - } - - field_policies = {('field', 'table'): lambda x: 252 if x > 255 else x} - fields = ( ('family', 'B'), ('dst_len', 'B'), diff --git a/pyroute2/netlink/rtnl/tcmsg/act_bpf.py b/pyroute2/netlink/rtnl/tcmsg/act_bpf.py index fbcccbf8e..511f3f42f 100644 --- a/pyroute2/netlink/rtnl/tcmsg/act_bpf.py +++ b/pyroute2/netlink/rtnl/tcmsg/act_bpf.py @@ -2,11 +2,6 @@ from pyroute2.netlink.rtnl.tcmsg.common import tc_actions -def fix_request(request): - if 'rate' in request: - del request['rate'] - - class options(nla): nla_map = ( ('TCA_ACT_BPF_UNSPEC', 'none'), diff --git a/pyroute2/netlink/rtnl/tcmsg/cls_basic.py b/pyroute2/netlink/rtnl/tcmsg/cls_basic.py index 99086ad27..eaa6f5b7a 100644 --- a/pyroute2/netlink/rtnl/tcmsg/cls_basic.py +++ b/pyroute2/netlink/rtnl/tcmsg/cls_basic.py @@ -158,12 +158,10 @@ ) -def fix_request(request): - if 'rate' in request: - del request['rate'] - request['info'] = htons( - request.get('protocol', protocols.ETH_P_ALL) & 0xFFFF - ) | ((request.get('prio', 0) << 16) & 0xFFFF0000) +def fix_msg(msg, kwarg): + msg['info'] = htons( + kwarg.get('protocol', protocols.ETH_P_ALL) & 0xFFFF + ) | ((kwarg.get('prio', 0) << 16) & 0xFFFF0000) def get_parameters(kwarg): diff --git a/pyroute2/netlink/rtnl/tcmsg/cls_flow.py b/pyroute2/netlink/rtnl/tcmsg/cls_flow.py index a136e521b..f6d17f403 100644 --- a/pyroute2/netlink/rtnl/tcmsg/cls_flow.py +++ b/pyroute2/netlink/rtnl/tcmsg/cls_flow.py @@ -77,12 +77,10 @@ from pyroute2.netlink.rtnl.tcmsg.common_act import get_tca_action, tca_act_prio -def fix_request(request): - if 'rate' in request: - del request['rate'] - request['info'] = htons( - request.get('protocol', protocols.ETH_P_ALL) & 0xFFFF - ) | ((request.get('prio', 0) << 16) & 0xFFFF0000) +def fix_msg(msg, kwarg): + msg['info'] = htons( + kwarg.get('protocol', protocols.ETH_P_ALL) & 0xFFFF + ) | ((kwarg.get('prio', 0) << 16) & 0xFFFF0000) def get_parameters(kwarg): diff --git a/pyroute2/netlink/rtnl/tcmsg/cls_fw.py b/pyroute2/netlink/rtnl/tcmsg/cls_fw.py index d64d4fc5b..482fbb32d 100644 --- a/pyroute2/netlink/rtnl/tcmsg/cls_fw.py +++ b/pyroute2/netlink/rtnl/tcmsg/cls_fw.py @@ -9,12 +9,10 @@ from pyroute2.netlink.rtnl.tcmsg.common_act import get_tca_action, tca_act_prio -def fix_request(request): - if 'rate' in request: - del request['rate'] - request['info'] = htons( - request.get('protocol', protocols.ETH_P_ALL) & 0xFFFF - ) | ((request.get('prio', 0) << 16) & 0xFFFF0000) +def fix_msg(msg, kwarg): + msg['info'] = htons( + kwarg.get('protocol', protocols.ETH_P_ALL) & 0xFFFF + ) | ((kwarg.get('prio', 0) << 16) & 0xFFFF0000) def get_parameters(kwarg): diff --git a/pyroute2/netlink/rtnl/tcmsg/cls_matchall.py b/pyroute2/netlink/rtnl/tcmsg/cls_matchall.py index a78024b46..e69f41422 100644 --- a/pyroute2/netlink/rtnl/tcmsg/cls_matchall.py +++ b/pyroute2/netlink/rtnl/tcmsg/cls_matchall.py @@ -5,12 +5,10 @@ from pyroute2.netlink.rtnl.tcmsg.common_act import get_tca_action, tca_act_prio -def fix_request(request): - if 'rate' in request: - del request['rate'] - request['info'] = htons( - request.get('protocol', protocols.ETH_P_ALL) & 0xFFFF - ) | ((request.get('prio', 0) << 16) & 0xFFFF0000) +def fix_msg(msg, kwarg): + msg['info'] = htons( + kwarg.get('protocol', protocols.ETH_P_ALL) & 0xFFFF + ) | ((kwarg.get('prio', 0) << 16) & 0xFFFF0000) def get_parameters(kwarg): diff --git a/pyroute2/netlink/rtnl/tcmsg/cls_u32.py b/pyroute2/netlink/rtnl/tcmsg/cls_u32.py index c3fb1c0bd..6e354e469 100644 --- a/pyroute2/netlink/rtnl/tcmsg/cls_u32.py +++ b/pyroute2/netlink/rtnl/tcmsg/cls_u32.py @@ -66,11 +66,9 @@ from pyroute2.netlink.rtnl.tcmsg.common_act import get_tca_action, tca_act_prio -def fix_request(request): - if 'rate' in request: - del request['rate'] - request['info'] = htons(request.get('protocol', 0) & 0xFFFF) | ( - (request.get('prio', 0) << 16) & 0xFFFF0000 +def fix_msg(msg, kwarg): + msg['info'] = htons(kwarg.get('protocol', 0) & 0xFFFF) | ( + (kwarg.get('prio', 0) << 16) & 0xFFFF0000 ) diff --git a/pyroute2/netlink/rtnl/tcmsg/common_ematch.py b/pyroute2/netlink/rtnl/tcmsg/common_ematch.py index eb67b1288..1e8bf3683 100644 --- a/pyroute2/netlink/rtnl/tcmsg/common_ematch.py +++ b/pyroute2/netlink/rtnl/tcmsg/common_ematch.py @@ -1,4 +1,3 @@ -from pyroute2.netlink.rt_files import EmatchMapFile from pyroute2.netlink.rtnl.tcmsg import em_cmp, em_ipset, em_meta plugins = { @@ -82,10 +81,7 @@ def get_tcf_ematches(kwarg): cur_match = kwarg['match'][i] # Translate string kind into numeric kind - try: - kind = plugins_translate[cur_match['kind']] - except KeyError: - kind = EmatchMapFile().get_rt_id(cur_match['kind']) + kind = plugins_translate[cur_match['kind']] match['kind'] = kind data = plugins[kind].data() data.setvalue(cur_match) diff --git a/pyroute2/netlink/rtnl/tcmsg/sched_bpf.py b/pyroute2/netlink/rtnl/tcmsg/sched_bpf.py index 15adfba98..92ec405ee 100644 --- a/pyroute2/netlink/rtnl/tcmsg/sched_bpf.py +++ b/pyroute2/netlink/rtnl/tcmsg/sched_bpf.py @@ -20,13 +20,11 @@ TCA_BPF_FLAG_ACT_DIRECT = 1 -def fix_request(request): - if 'rate' in request: - del request['rate'] - if 'info' not in request: - request['info'] = htons( - request.pop('protocol', ETH_P_ALL) & 0xFFFF - ) | ((request.pop('prio', 0) << 16) & 0xFFFF0000) +def fix_msg(msg, kwarg): + if 'info' not in kwarg: + msg['info'] = htons(kwarg.pop('protocol', ETH_P_ALL) & 0xFFFF) | ( + (kwarg.pop('prio', 0) << 16) & 0xFFFF0000 + ) def get_parameters(kwarg): diff --git a/pyroute2/netlink/rtnl/tcmsg/sched_cake.py b/pyroute2/netlink/rtnl/tcmsg/sched_cake.py index ba5e6c9c8..6f5eff668 100644 --- a/pyroute2/netlink/rtnl/tcmsg/sched_cake.py +++ b/pyroute2/netlink/rtnl/tcmsg/sched_cake.py @@ -102,11 +102,9 @@ TCA_CAKE_MAX_TINS = 8 -def fix_request(request): - if 'rate' in request: - del request['rate'] - if 'parent' not in request or request['parent'] == 0: - request['parent'] = TC_H_ROOT +def fix_msg(msg, kwarg): + if 'parent' not in kwarg: + msg['parent'] = TC_H_ROOT def convert_bandwidth(value): diff --git a/pyroute2/netlink/rtnl/tcmsg/sched_clsact.py b/pyroute2/netlink/rtnl/tcmsg/sched_clsact.py index 08fb024ee..15c95fc40 100644 --- a/pyroute2/netlink/rtnl/tcmsg/sched_clsact.py +++ b/pyroute2/netlink/rtnl/tcmsg/sched_clsact.py @@ -29,7 +29,5 @@ parent = TC_H_CLSACT -def fix_request(request): - if 'rate' in request: - del request['rate'] - request['handle'] = 0xFFFF0000 +def fix_msg(msg, kwarg): + msg['handle'] = 0xFFFF0000 diff --git a/pyroute2/netlink/rtnl/tcmsg/sched_htb.py b/pyroute2/netlink/rtnl/tcmsg/sched_htb.py index 58209b035..373831f7d 100644 --- a/pyroute2/netlink/rtnl/tcmsg/sched_htb.py +++ b/pyroute2/netlink/rtnl/tcmsg/sched_htb.py @@ -81,8 +81,6 @@ def get_class_parameters(kwarg): quantum = kwarg.get('quantum', 0) rate = get_rate(kwarg.get('rate', None)) ceil = get_rate(kwarg.get('ceil', 0)) or rate - rate64 = 0 - ceil64 = 0 burst = ( kwarg.get('burst', None) @@ -106,13 +104,7 @@ def get_class_parameters(kwarg): cburst = ceil / get_hz() + mtu cburst = calc_xmittime(ceil, cburst) - if rate is not None and rate >= 1 << 32: - rate64 = rate - rate = (1 << 32) - 1 - if ceil is not None and ceil >= 1 << 32: - ceil64 = ceil - ceil = (1 << 32) - 1 - ret = { + return { 'attrs': [ [ 'TCA_HTB_PARMS', @@ -133,11 +125,6 @@ def get_class_parameters(kwarg): ['TCA_HTB_CTAB', True], ] } - if rate64 > 0: - ret['attrs'].append(['TCA_HTB_RATE64', rate64]) - if ceil64 > 0: - ret['attrs'].append(['TCA_HTB_CEIL64', ceil64]) - return ret def get_parameters(kwarg): @@ -161,14 +148,10 @@ def get_parameters(kwarg): } -def fix_request(request): - if not request: +def fix_msg(msg, kwarg): + if not kwarg: opts = get_parameters({}) - request['attrs'].append(['TCA_OPTIONS', opts]) - if 'options' not in request: - request['options'] = get_parameters({}) - if 'rate' in request: - del request['rate'] + msg['attrs'].append(['TCA_OPTIONS', opts]) # The tokens and ctokens are badly defined in the kernel structure @@ -212,11 +195,6 @@ class options(nla_plus_rtab): ('TCA_HTB_INIT', 'htb_glob'), ('TCA_HTB_CTAB', 'ctab'), ('TCA_HTB_RTAB', 'rtab'), - ('TCA_HTB_DIRECT_QLEN', 'uint32'), - ('TCA_HTB_RATE64', 'uint64'), - ('TCA_HTB_CEIL64', 'uint64'), - ('TCA_HTB_PAD', 'hex'), - ('TCA_HTB_OFFLOAD', 'hex'), ) class htb_glob(nla): diff --git a/pyroute2/netlink/rtnl/tcmsg/sched_ingress.py b/pyroute2/netlink/rtnl/tcmsg/sched_ingress.py index 185cb2af1..95eb64498 100644 --- a/pyroute2/netlink/rtnl/tcmsg/sched_ingress.py +++ b/pyroute2/netlink/rtnl/tcmsg/sched_ingress.py @@ -4,10 +4,8 @@ parent = TC_H_INGRESS -def fix_request(request): - if 'rate' in request: - del request['rate'] - request['handle'] = 0xFFFF0000 +def fix_msg(msg, kwarg): + msg['handle'] = 0xFFFF0000 class options(nla): diff --git a/pyroute2/netlink/rtnl/tcmsg/sched_tbf.py b/pyroute2/netlink/rtnl/tcmsg/sched_tbf.py index 16081a4bf..bc2f537cd 100644 --- a/pyroute2/netlink/rtnl/tcmsg/sched_tbf.py +++ b/pyroute2/netlink/rtnl/tcmsg/sched_tbf.py @@ -7,11 +7,6 @@ parent = TC_H_ROOT -def fix_request(request): - if 'rate' in request: - del request['rate'] - - def get_parameters(kwarg): parms = get_rate_parameters(kwarg) # fill parameters diff --git a/pyroute2/netlink/rtnl/tcmsg/sched_template.py b/pyroute2/netlink/rtnl/tcmsg/sched_template.py index 013fc1b3d..7610028b2 100644 --- a/pyroute2/netlink/rtnl/tcmsg/sched_template.py +++ b/pyroute2/netlink/rtnl/tcmsg/sched_template.py @@ -16,10 +16,10 @@ parent = TC_H_ROOT -def fix_request(request): +def fix_msg(kwarg, msg): ''' This method it called for all types -- classes, - qdiscs and filters. Can be used to fix some request + qdiscs and filters. Can be used to fix some `msg` fields. ''' pass diff --git a/pyroute2/netlink/taskstats/__init__.py b/pyroute2/netlink/taskstats/__init__.py index c7bf1b7eb..40ffc577a 100644 --- a/pyroute2/netlink/taskstats/__init__.py +++ b/pyroute2/netlink/taskstats/__init__.py @@ -14,13 +14,8 @@ usable. ''' -import struct - from pyroute2.netlink import NLM_F_REQUEST, genlmsg, nla, nla_struct -from pyroute2.netlink.generic import ( - AsyncGenericNetlinkSocket, - GenericNetlinkSocket, -) +from pyroute2.netlink.generic import GenericNetlinkSocket TASKSTATS_CMD_UNSPEC = 0 # Reserved TASKSTATS_CMD_GET = 1 # user->kernel request/get-response @@ -37,7 +32,53 @@ class tcmd(genlmsg): ) -class TStatsDecoder: +class tstats(nla_struct): + fields = ( + ('version', 'H'), # 2 + ('ac_exitcode', 'I'), # 4 + ('ac_flag', 'B'), # 1 + ('ac_nice', 'B'), # 1 --- 10 + ('cpu_count', 'Q'), # 8 + ('cpu_delay_total', 'Q'), # 8 + ('blkio_count', 'Q'), # 8 + ('blkio_delay_total', 'Q'), # 8 + ('swapin_count', 'Q'), # 8 + ('swapin_delay_total', 'Q'), # 8 + ('cpu_run_real_total', 'Q'), # 8 + ('cpu_run_virtual_total', 'Q'), # 8 + ('ac_comm', '32s'), # 32 +++ 112 + ('ac_sched', 'B'), # 1 + ('__ac_pad', '3x'), # 3 + # (the ac_uid field is aligned(8), so we add more padding) + ('__implicit_pad', '4x'), # 4 + ('ac_uid', 'I'), # 4 +++ 120 + ('ac_gid', 'I'), # 4 + ('ac_pid', 'I'), # 4 + ('ac_ppid', 'I'), # 4 + ('ac_btime', 'I'), # 4 +++ 136 + ('ac_etime', 'Q'), # 8 +++ 144 + ('ac_utime', 'Q'), # 8 + ('ac_stime', 'Q'), # 8 + ('ac_minflt', 'Q'), # 8 + ('ac_majflt', 'Q'), # 8 + ('coremem', 'Q'), # 8 + ('virtmem', 'Q'), # 8 + ('hiwater_rss', 'Q'), # 8 + ('hiwater_vm', 'Q'), # 8 + ('read_char', 'Q'), # 8 + ('write_char', 'Q'), # 8 + ('read_syscalls', 'Q'), # 8 + ('write_syscalls', 'Q'), # 8 + ('read_bytes', 'Q'), # ... + ('write_bytes', 'Q'), + ('cancelled_write_bytes', 'Q'), + ('nvcsw', 'Q'), + ('nivcsw', 'Q'), + ('ac_utimescaled', 'Q'), + ('ac_stimescaled', 'Q'), + ('cpu_scaled_run_real_total', 'Q'), + ) + def decode(self): nla_struct.decode(self) command = self['ac_comm'] @@ -46,178 +87,30 @@ def decode(self): self['ac_comm'] = command[: command.find('\0')] -class TStatsBase: - class tstats_vB(nla_struct, TStatsDecoder): - fields = ( - ('version', 'H'), # 2 - ('ac_exitcode', 'I'), # 4 - ('ac_flag', 'B'), # 1 - ('ac_nice', 'B'), # 1 --- 10 - ('cpu_count', 'Q'), # 8 - ('cpu_delay_total', 'Q'), # 8 - ('blkio_count', 'Q'), # 8 - ('blkio_delay_total', 'Q'), # 8 - ('swapin_count', 'Q'), # 8 - ('swapin_delay_total', 'Q'), # 8 - ('cpu_run_real_total', 'Q'), # 8 - ('cpu_run_virtual_total', 'Q'), # 8 - ('ac_comm', '32s'), # 32 +++ 112 - ('ac_sched', 'B'), # 1 - ('__ac_pad', '3x'), # 3 - # (the ac_uid field is aligned(8), so we add more padding) - ('__implicit_pad', '4x'), # 4 - ('ac_uid', 'I'), # 4 +++ 120 - ('ac_gid', 'I'), # 4 - ('ac_pid', 'I'), # 4 - ('ac_ppid', 'I'), # 4 - ('ac_btime', 'I'), # 4 +++ 136 - ('ac_etime', 'Q'), # 8 +++ 144 - ('ac_utime', 'Q'), # 8 - ('ac_stime', 'Q'), # 8 - ('ac_minflt', 'Q'), # 8 - ('ac_majflt', 'Q'), # 8 - ('coremem', 'Q'), # 8 - ('virtmem', 'Q'), # 8 - ('hiwater_rss', 'Q'), # 8 - ('hiwater_vm', 'Q'), # 8 - ('read_char', 'Q'), # 8 - ('write_char', 'Q'), # 8 - ('read_syscalls', 'Q'), # 8 - ('write_syscalls', 'Q'), # 8 - ('read_bytes', 'Q'), # ... - ('write_bytes', 'Q'), - ('cancelled_write_bytes', 'Q'), - ('nvcsw', 'Q'), - ('nivcsw', 'Q'), - ('ac_utimescaled', 'Q'), - ('ac_stimescaled', 'Q'), - ('cpu_scaled_run_real_total', 'Q'), - ) - - class tstats_v15(nla_struct, TStatsDecoder): - fields = ( - ('version', 'H'), - ('ac_exitcode', 'I'), - ('ac_flag', 'B'), - ('ac_nice', 'B'), - # - ('cpu_count', 'Q'), - ('cpu_delay_total', 'Q'), - ('cpu_delay_max', 'Q'), - ('cpu_delay_min', 'Q'), - # - ('blkio_count', 'Q'), - ('blkio_delay_total', 'Q'), - ('blkio_delay_max', 'Q'), - ('blkio_delay_min', 'Q'), - # - ('swapin_count', 'Q'), - ('swapin_delay_total', 'Q'), - ('swapin_delay_max', 'Q'), - ('swapin_delay_min', 'Q'), - # - ('cpu_run_real_total', 'Q'), - ('cpu_run_virtual_total', 'Q'), - # - ('ac_comm', '32s'), - ('ac_sched', 'B'), - ('__ac_pad', '3x'), - # (the ac_uid field is aligned(8), so we add more padding) - ('__implicit_pad', '4x'), - ('ac_uid', 'I'), - ('ac_gid', 'I'), - ('ac_pid', 'I'), - ('ac_ppid', 'I'), - ('ac_btime', 'I'), - ('ac_etime', 'Q'), - ('ac_utime', 'Q'), - ('ac_stime', 'Q'), - ('ac_minflt', 'Q'), - ('ac_majflt', 'Q'), - ('coremem', 'Q'), - ('virtmem', 'Q'), - ('hiwater_rss', 'Q'), - ('hiwater_vm', 'Q'), - ('read_char', 'Q'), - ('write_char', 'Q'), - ('read_syscalls', 'Q'), - ('write_syscalls', 'Q'), - # - ('read_bytes', 'Q'), - ('write_bytes', 'Q'), - ('cancelled_write_bytes', 'Q'), - # - ('nvcsw', 'Q'), - ('nivcsw', 'Q'), - # - ('ac_utimescaled', 'Q'), - ('ac_stimescaled', 'Q'), - ('cpu_scaled_run_real_total', 'Q'), - # - ('freepages_count', 'Q'), - ('freepages_delay_total', 'Q'), - ('freepages_delay_max', 'Q'), - ('freepages_delay_min', 'Q'), - # - ('thrashing_count', 'Q'), - ('thrashing_delay_total', 'Q'), - ('thrashing_delay_max', 'Q'), - ('thrashing_delay_min', 'Q'), - # - ('ac_btime64', 'Q'), - # - ('compact_count', 'Q'), - ('compact_delay_total', 'Q'), - ('compact_delay_max', 'Q'), - ('compact_delay_min', 'Q'), - # - ('ac_tgid', 'I'), - # - ('ac_tgetime', 'Q'), - # - ('ac_exe_dev', 'Q'), - ('ac_exe_inode', 'Q'), - # - ('wpcopy_count', 'Q'), - ('wpcopy_delay_total', 'Q'), - ('wpcopy_delay_max', 'Q'), - ('wpcopy_delay_min', 'Q'), - # - ('irq_count', 'Q'), - ('irq_delay_total', 'Q'), - ('irq_delay_max', 'Q'), - ('irq_delay_min', 'Q'), - ) - - @staticmethod - def versioned_stats(self, *argv, **kwarg): - data = kwarg['data'] - offset = kwarg['offset'] + struct.calcsize('HH') # + NL header - end = offset + struct.calcsize('H') # + uint16, version - (version,) = struct.unpack('H', data[offset:end]) - if version == 15: - return self.tstats_v15 - return self.tstats_vB - - -class taskstatsmsg(genlmsg, TStatsBase): +class taskstatsmsg(genlmsg): nla_map = ( ('TASKSTATS_TYPE_UNSPEC', 'none'), ('TASKSTATS_TYPE_PID', 'uint32'), ('TASKSTATS_TYPE_TGID', 'uint32'), - ('TASKSTATS_TYPE_STATS', 'versioned_stats'), + ('TASKSTATS_TYPE_STATS', 'stats'), ('TASKSTATS_TYPE_AGGR_PID', 'aggr_pid'), ('TASKSTATS_TYPE_AGGR_TGID', 'aggr_tgid'), ) - class aggr_id(nla, TStatsBase): + class stats(tstats): + pass # FIXME: optimize me! + + class aggr_id(nla): nla_map = ( ('TASKSTATS_TYPE_UNSPEC', 'none'), ('TASKSTATS_TYPE_PID', 'uint32'), ('TASKSTATS_TYPE_TGID', 'uint32'), - ('TASKSTATS_TYPE_STATS', 'versioned_stats'), + ('TASKSTATS_TYPE_STATS', 'stats'), ) + class stats(tstats): + pass + class aggr_pid(aggr_id): pass @@ -225,11 +118,11 @@ class aggr_tgid(aggr_id): pass -class AsyncTaskStats(AsyncGenericNetlinkSocket): - async def bind(self): - await super().bind('TASKSTATS', taskstatsmsg) +class TaskStats(GenericNetlinkSocket): + def bind(self): + GenericNetlinkSocket.bind(self, 'TASKSTATS', taskstatsmsg) - async def get_pid_stat(self, pid): + def get_pid_stat(self, pid): ''' Get taskstats for a process. Pid should be an integer. ''' @@ -237,17 +130,17 @@ async def get_pid_stat(self, pid): msg['cmd'] = TASKSTATS_CMD_GET msg['version'] = 1 msg['attrs'].append(['TASKSTATS_CMD_ATTR_PID', pid]) - return await self.nlm_request(msg, self.prid, msg_flags=NLM_F_REQUEST) + return self.nlm_request(msg, self.prid, msg_flags=NLM_F_REQUEST) - async def _register_mask(self, cmd, mask): + def _register_mask(self, cmd, mask): msg = tcmd() msg['cmd'] = TASKSTATS_CMD_GET msg['version'] = 1 msg['attrs'].append([cmd, mask]) # there is no response to this request - await self.put(msg, self.prid, msg_flags=NLM_F_REQUEST) + self.put(msg, self.prid, msg_flags=NLM_F_REQUEST) - async def register_mask(self, mask): + def register_mask(self, mask): ''' Start the accounting for a processors by a mask. Mask is a string, e.g.:: @@ -258,29 +151,10 @@ async def register_mask(self, mask): when it is not used, it is recommended to run deregister_mask() before process exit. ''' - await self._register_mask('TASKSTATS_CMD_ATTR_REGISTER_CPUMASK', mask) + self._register_mask('TASKSTATS_CMD_ATTR_REGISTER_CPUMASK', mask) - async def deregister_mask(self, mask): + def deregister_mask(self, mask): ''' Stop the accounting. ''' - await self._register_mask( - 'TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK', mask - ) - - -class TaskStats(GenericNetlinkSocket): - - async_class = AsyncTaskStats - - def bind(self): - return self._run_with_cleanup(self.asyncore.bind) - - def get_pid_stat(self, pid): - return self._run_sync_cleanup(self.asyncore.get_pid_stat, pid) - - def register_mask(self, mask): - return self._run_with_cleanup(self.asyncore.register_mask, mask) - - def unregister_mask(self, mask): - return self._run_with_cleanup(self.asyncore.unregister_mask, mask) + self._register_mask('TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK', mask) diff --git a/pyroute2/netlink/uevent/__init__.py b/pyroute2/netlink/uevent/__init__.py index e7086b5b3..712a3f56b 100644 --- a/pyroute2/netlink/uevent/__init__.py +++ b/pyroute2/netlink/uevent/__init__.py @@ -1,16 +1,5 @@ -import struct -from typing import Callable, Generator, Optional - from pyroute2.netlink import NETLINK_KOBJECT_UEVENT, nlmsg -from pyroute2.netlink.nlsocket import ( - AsyncNetlinkSocket, - Marshal, - NetlinkSocket, -) - -MONITOR_GROUP_NONE = 0 -MONITOR_GROUP_KERNEL = 1 -MONITOR_GROUP_UDEV = 2 +from pyroute2.netlink.nlsocket import Marshal, NetlinkSocket class ueventmsg(nlmsg): @@ -18,39 +7,14 @@ class ueventmsg(nlmsg): class MarshalUevent(Marshal): - def is_enough(self, _) -> bool: - return False - - def parse( - self, - data: bytes, - seq: Optional[int] = None, - callback: Optional[Callable] = None, - skip_alien_seq: bool = False, - ) -> Generator[ueventmsg, None, None]: + def parse(self, data, seq=None, callback=None): ret = ueventmsg() ret['header']['sequence_number'] = 0 - - # handle udevd messages which have a binary header - if data.startswith(b'libudev\x00'): - # prefix, magic, header_size, properties_off, properties_len, - # filter_subsystem_hash, filter_tag_bloom_hi, - # filter_tag_bloom_low - header_format = '8sIIIIIIII' - header_length = struct.calcsize(header_format) - - if len(data) < header_length: - return - - lines = data[header_length:].split(b'\x00') - ret['header']['message'] = "libudev" - else: - lines = data.split(b'\x00') - ret['header']['message'] = lines[0].decode('utf-8') - + data = data.split(b'\x00') wtf = [] + ret['header']['message'] = data[0].decode('utf-8') ret['header']['unparsed'] = b'' - for line in lines[1:]: + for line in data[1:]: if line.find(b'=') <= 0: wtf.append(line) else: @@ -58,33 +22,17 @@ def parse( ret['header']['unparsed'] = b'\x00'.join(wtf) wtf = [] - try: - key, value = line.decode('utf-8').split('=', 1) - except UnicodeDecodeError: - # should not happen but let's account for it for - # robustness' sake - wtf.append(line) - continue - - ret[key] = value + line = line.decode('utf-8').split('=') + ret[line[0]] = '='.join(line[1:]) del ret['value'] - yield ret - - -class AsyncUeventSocket(AsyncNetlinkSocket): - def __init__(self): - super().__init__(NETLINK_KOBJECT_UEVENT) - self.set_marshal(MarshalUevent()) - - async def bind(self, groups: int = -1) -> None: # type: ignore[override] - return await super().bind(groups=groups) + return [ret] class UeventSocket(NetlinkSocket): def __init__(self): - super().__init__(NETLINK_KOBJECT_UEVENT) - self.set_marshal(MarshalUevent()) + super(UeventSocket, self).__init__(NETLINK_KOBJECT_UEVENT) + self.marshal = MarshalUevent() - def bind(self, groups: int = -1) -> None: - return super().bind(groups=groups) + def bind(self): + return super(UeventSocket, self).bind(groups=-1) diff --git a/pyroute2/netns/__init__.py b/pyroute2/netns/__init__.py index 3edd06b44..29327a576 100644 --- a/pyroute2/netns/__init__.py +++ b/pyroute2/netns/__init__.py @@ -24,17 +24,53 @@ from pyroute2 import IPRoute ipr = IPRoute() ipr.link('add', ifname='v0p0', kind='veth', peer='v0p1') - idx = ipr.link_lookup(ifname='v0p1') + idx = ipr.link_lookup(ifname='v0p1')[0] ipr.link('set', index=idx, net_ns_fd='netns_name') +Create veth and move the peer to a netns with IPDB:: + + from pyroute2 import IPDB + ipdb = IPDB() + ipdb.create(ifname='v0p0', kind='veth', peer='v0p1').commit() + with ipdb.interfaces.v0p1 as i: + i.net_ns_fd = 'netns_name' + +Manage interfaces within a netns +-------------------------------- + +This task can be done with `NetNS` objects. A `NetNS` object +spawns a child and runs it within a netns, providing the same +API as `IPRoute` does:: + + from pyroute2 import NetNS + ns = NetNS('netns_name') + # do some stuff within the netns + ns.close() + +One can even start `IPDB` on the top of `NetNS`:: + + from pyroute2 import NetNS + from pyroute2 import IPDB + ns = NetNS('netns_name') + ipdb = IPDB(nl=ns) + # do some stuff within the netns + ipdb.release() + ns.close() + Spawn a process within a netns ------------------------------ For that purpose one can use `NSPopen` API. It works just as normal `Popen`, but starts a process within a netns. -Network namespace management ----------------------------- +List, set, create, attach and remove netns +------------------------------------------ + +These functions are described below. To use them, import +`netns` module:: + + from pyroute2 import netns + netns.listnetns() Please be aware, that in order to run system calls the library uses `ctypes` module. It can fail on platforms @@ -48,17 +84,14 @@ import ctypes.util import errno import io -import logging import os import os.path -import socket -from typing import Optional +import pickle +import struct +import traceback from pyroute2 import config -from pyroute2.common import USE_DEFAULT_TIMEOUT, basestring, get_time -from pyroute2.process import ChildProcess, ChildProcessReturnValue - -log = logging.getLogger(__name__) +from pyroute2.common import basestring try: file = file @@ -237,19 +270,35 @@ def _create(netns, libc=None, pid=None): raise OSError(ctypes.get_errno(), 'mount failed', netns) -@config.mock_if('mock_netns') def create(netns, libc=None): ''' Create a network namespace. ''' - proc = ChildProcess(target=_create, args=[netns, libc]) - proc.run() - proc.communicate() - proc.stop(kill=True) - proc.close() + rctl, wctl = os.pipe() + pid = os.fork() + if pid == 0: + # child + error = None + try: + _create(netns, libc) + except Exception as e: + error = e + error.tb = traceback.format_exc() + msg = pickle.dumps(error) + os.write(wctl, struct.pack('I', len(msg))) + os.write(wctl, msg) + os._exit(0) + else: + # parent + msglen = struct.unpack('I', os.read(rctl, 4))[0] + error = pickle.loads(os.read(rctl, msglen)) + os.close(rctl) + os.close(wctl) + os.waitpid(pid, 0) + if error is not None: + raise error -@config.mock_if('mock_netns') def attach(netns, pid, libc=None): ''' Attach the network namespace of the process `pid` @@ -258,7 +307,6 @@ def attach(netns, pid, libc=None): _create(netns, libc, pid) -@config.mock_if('mock_netns') def remove(netns, libc=None): ''' Remove a network namespace. @@ -269,8 +317,7 @@ def remove(netns, libc=None): os.unlink(netnspath) -@config.mock_if('mock_netns') -def setns(netns, flags=os.O_CREAT, libc=None, fork=True): +def setns(netns, flags=os.O_CREAT, libc=None): ''' Set netns for the current process. @@ -299,10 +346,7 @@ def setns(netns, flags=os.O_CREAT, libc=None, fork=True): raise OSError(errno.EEXIST, 'netns exists', netns) else: if flags & os.O_CREAT: - if fork: - create(netns, libc=libc) - else: - _create(netns, libc=libc) + create(netns, libc=libc) nsfd = os.open(netnspath, os.O_RDONLY) newfd = True elif isinstance(netns, file): @@ -318,7 +362,6 @@ def setns(netns, flags=os.O_CREAT, libc=None, fork=True): raise OSError(ctypes.get_errno(), 'failed to open netns', netns) -@config.mock_if('mock_netns') def pushns(newns=None, libc=None): ''' Save the current netns in order to return to it later. If newns is @@ -336,7 +379,6 @@ def pushns(newns=None, libc=None): setns(newns, libc=libc) -@config.mock_if('mock_netns') def popns(libc=None): ''' Restore the previously saved netns. @@ -351,7 +393,6 @@ def popns(libc=None): os.close(fd) -@config.mock_if('mock_netns') def dropns(libc=None): ''' Discard the last saved with `pushns()` namespace @@ -362,47 +403,3 @@ def dropns(libc=None): os.close(fd) except Exception: pass - - -def _create_socket_child(nsname, flags, family, socket_type, proto, libc=None): - setns(nsname, flags=flags, libc=libc, fork=False) - sock = socket.socket(family, socket_type, proto) - return ChildProcessReturnValue(b'', [sock]) - - -@config.mock_if('mock_netns') -def create_socket( - netns: Optional[str] = None, - family: int = socket.AF_INET, - socket_type: int = socket.SOCK_STREAM, - proto: int = 0, - fileno: Optional[int] = None, - flags: int = os.O_CREAT, - libc: Optional[ctypes.CDLL] = None, - timeout: int = USE_DEFAULT_TIMEOUT, -) -> socket.socket: - if fileno is not None and netns is not None: - raise TypeError('you can not specify both fileno and netns') - if fileno is not None: - return socket.socket(fileno=fileno) - if netns is None: - return socket.socket(family, socket_type, proto) - if timeout == USE_DEFAULT_TIMEOUT: - timeout = config.default_create_socket_timeout - - start_time = get_time() - while get_time() - start_time < timeout: - with ChildProcess( - target=_create_socket_child, - args=[netns, flags, family, socket_type, proto, libc], - ) as proc: - try: - return socket.socket( - fileno=proc.get_fds( - timeout=config.default_communicate_timeout - )[0] - ) - except TimeoutError: - continue - - raise TimeoutError('could not start netns socket within timeout') diff --git a/pyroute2/nftables/expressions.py b/pyroute2/nftables/expressions.py index 846ffa37d..2894d5083 100644 --- a/pyroute2/nftables/expressions.py +++ b/pyroute2/nftables/expressions.py @@ -2,8 +2,6 @@ import struct from collections import OrderedDict -from pyroute2.netlink.nfnetlink.nftsocket import Cmp, Regs - ## # Utility functions @@ -33,13 +31,9 @@ def genex(name, kwarg): } -def masq(): - return [genex('masq', {})] - - def verdict(code): kwarg = OrderedDict() - kwarg['dreg'] = Regs.NFT_REG_VERDICT + kwarg['dreg'] = 0 # NFT_REG_VERDICT kwarg['data'] = { 'attrs': [ ('NFTA_DATA_VERDICT', {'attrs': [('NFTA_VERDICT_CODE', code)]}) @@ -48,7 +42,7 @@ def verdict(code): return [genex('immediate', kwarg)] -def ipv4addr(src=None, dst=None, op=Cmp.NFT_CMP_EQ): +def ipv4addr(src=None, dst=None): if not src and not dst: raise ValueError('must be at least one of src, dst') @@ -94,10 +88,8 @@ def ipv4addr(src=None, dst=None, op=Cmp.NFT_CMP_EQ): if dst: packed += socket.inet_aton(dst) kwarg = OrderedDict() - kwarg['sreg'] = Regs.NFT_REG_1 # read from NFT_REG_1 - # NFT_CMP_EQ = 0 - # NFT_CMP_NEQ = 1 - kwarg['op'] = op # nftsocket.nft_contains_expr.nft_expr.nft_cmp.ops + kwarg['sreg'] = 1 # read from NFT_REG_1 + kwarg['op'] = 0 # NFT_CMP_EQ kwarg['data'] = {'attrs': [('NFTA_DATA_VALUE', packed)]} ret.append(genex('cmp', kwarg)) return ret diff --git a/pyroute2/nftables/main.py b/pyroute2/nftables/main.py index 283ef63e4..405b90969 100644 --- a/pyroute2/nftables/main.py +++ b/pyroute2/nftables/main.py @@ -1,8 +1,6 @@ ''' ''' -from functools import partial - from pyroute2.netlink.nfnetlink import nfgen_msg from pyroute2.netlink.nfnetlink.nftsocket import ( DATA_TYPE_ID_TO_NAME, @@ -22,7 +20,6 @@ NFT_MSG_NEWSET, NFT_MSG_NEWSETELEM, NFT_MSG_NEWTABLE, - AsyncNFTSocket, NFTSocket, nft_chain_msg, nft_rule_msg, @@ -220,22 +217,22 @@ def __repr__(self): return str(self.as_dict()) -class AsyncNFTables(AsyncNFTSocket): +class NFTables(NFTSocket): # TODO: documentation # TODO: tests # TODO: dump()/load() with support for json and xml - async def get_tables(self): - return await self.request_get(nfgen_msg(), NFT_MSG_GETTABLE) + def get_tables(self): + return self.request_get(nfgen_msg(), NFT_MSG_GETTABLE) - async def get_chains(self): - return await self.request_get(nfgen_msg(), NFT_MSG_GETCHAIN) + def get_chains(self): + return self.request_get(nfgen_msg(), NFT_MSG_GETCHAIN) - async def get_rules(self): - return await self.request_get(nfgen_msg(), NFT_MSG_GETRULE) + def get_rules(self): + return self.request_get(nfgen_msg(), NFT_MSG_GETRULE) - async def get_sets(self): - return await self.request_get(nfgen_msg(), NFT_MSG_GETSET) + def get_sets(self): + return self.request_get(nfgen_msg(), NFT_MSG_GETSET) # # The nft API is in the prototype stage and may be @@ -243,7 +240,7 @@ async def get_sets(self): # the API is 0.5.2 # - async def table(self, cmd, **kwarg): + def table(self, cmd, **kwarg): ''' Example:: @@ -255,9 +252,9 @@ async def table(self, cmd, **kwarg): 'del': NFT_MSG_DELTABLE, 'get': NFT_MSG_GETTABLE, } - return await self._command(nft_table_msg, commands, cmd, kwarg) + return self._command(nft_table_msg, commands, cmd, kwarg) - async def chain(self, cmd, **kwarg): + def chain(self, cmd, **kwarg): ''' Example:: @@ -294,9 +291,9 @@ async def chain(self, cmd, **kwarg): } if 'type' not in kwarg: kwarg['type'] = 'filter' - return await self._command(nft_chain_msg, commands, cmd, kwarg) + return self._command(nft_chain_msg, commands, cmd, kwarg) - async def rule(self, cmd, **kwarg): + def rule(self, cmd, **kwarg): ''' Example:: @@ -324,9 +321,9 @@ async def rule(self, cmd, **kwarg): for exp in kwarg['expressions']: expressions.extend(exp) kwarg['expressions'] = expressions - return await self._command(nft_rule_msg, commands, cmd, kwarg) + return self._command(nft_rule_msg, commands, cmd, kwarg) - async def sets(self, cmd, **kwarg): + def sets(self, cmd, **kwarg): ''' Example:: nft.sets("add", table="filter", name="test0", key_type="ipv4_addr", @@ -349,17 +346,12 @@ async def sets(self, cmd, **kwarg): else: nft_set = NFTSet(**kwarg) kwarg = nft_set.as_netlink() - msg = [ - x - async for x in await self._command( - nft_set_msg, commands, cmd, kwarg - ) - ] + msg = self._command(nft_set_msg, commands, cmd, kwarg) if cmd == "get": return NFTSet.from_netlink(msg) return nft_set - async def set_elems(self, cmd, **kwarg): + def set_elems(self, cmd, **kwarg): ''' Example:: nft.set_elems("add", table="filter", set="test0", @@ -384,9 +376,7 @@ async def set_elems(self, cmd, **kwarg): kwarg["table"] = nft_set.table kwarg["set"] = nft_set.name else: - nft_set = await self.sets( - "get", table=kwarg["table"], name=kwarg["set"] - ) + nft_set = self.sets("get", table=kwarg["table"], name=kwarg["set"]) found = DATA_TYPE_NAME_TO_INFO.get(nft_set.key_type) if found: @@ -402,10 +392,7 @@ async def set_elems(self, cmd, **kwarg): ["NFTA_SET_ELEM_LIST_TABLE", kwarg["table"]], ["NFTA_SET_ELEM_LIST_SET", kwarg["set"]], ] - msg = [ - x - async for x in await self.request_get(msg, NFT_MSG_GETSETELEM) - ][0] + msg = self.request_get(msg, NFT_MSG_GETSETELEM)[0] elements = set() for elem in msg.get_attr('NFTA_SET_ELEM_LIST_ELEMENTS'): elements.add(NFTSetElem.from_netlink(elem, modifier)) @@ -419,34 +406,4 @@ async def set_elems(self, cmd, **kwarg): elem = NFTSetElem(value=elem) elements.append(elem.as_netlink(modifier)) kwarg["elements"] = elements - return await self._command(nft_set_elem_list_msg, commands, cmd, kwarg) - - -class NFTables(NFTSocket): - def __init__(self, version=1, attr_version=0, nfgen_family=2): - self.asyncore = AsyncNFTables(version, attr_version, nfgen_family) - self.asyncore.local.keep_event_loop = True - self.asyncore.event_loop.run_until_complete( - self.asyncore.setup_endpoint() - ) - - @staticmethod - async def _collect_dump(func): - return [x async for x in await func()] - - def __getattr__(self, name): - async_dump_methods = [ - 'get_tables', - 'get_chains', - 'get_rules', - 'get_sets', - ] - async_cmd_methods = ['table', 'chain', 'rule', 'sets', 'set_elems'] - symbol = getattr(self.asyncore, name) - if name in async_dump_methods: - return partial( - self._run_with_cleanup, partial(self._collect_dump, symbol) - ) - elif name in async_cmd_methods: - return partial(self._run_with_cleanup, symbol) - return symbol + return self._command(nft_set_elem_list_msg, commands, cmd, kwarg) diff --git a/pyroute2/nslink/nslink.py b/pyroute2/nslink/nslink.py new file mode 100644 index 000000000..1c7caf10f --- /dev/null +++ b/pyroute2/nslink/nslink.py @@ -0,0 +1,231 @@ +''' +NetNS objects +============= + +A NetNS object is IPRoute-like. It runs in the main network +namespace, but also creates a proxy process running in +the required netns. All the netlink requests are done via +that proxy process. + +NetNS supports standard IPRoute API, so can be used instead +of IPRoute, e.g., in IPDB:: + + # start the main network settings database: + ipdb_main = IPDB() + # start the same for a netns: + ipdb_test = IPDB(nl=NetNS('test')) + + # create VETH + ipdb_main.create(ifname='v0p0', kind='veth', peer='v0p1').commit() + + # move peer VETH into the netns + with ipdb_main.interfaces.v0p1 as veth: + veth.net_ns_fd = 'test' + + # please keep in mind, that netns move clears all the settings + # on a VETH interface pair, so one should run netns assignment + # as a separate operation only + + # assign addresses + # please notice, that `v0p1` is already in the `test` netns, + # so should be accessed via `ipdb_test` + with ipdb_main.interfaces.v0p0 as veth: + veth.add_ip('172.16.200.1/24') + veth.up() + with ipdb_test.interfaces.v0p1 as veth: + veth.add_ip('172.16.200.2/24') + veth.up() + +Please review also the test code, under `tests/test_netns.py` for +more examples. + +By default, NetNS creates requested netns, if it doesn't exist, +or uses existing one. To control this behaviour, one can use flags +as for `open(2)` system call:: + + # create a new netns or fail, if it already exists + netns = NetNS('test', flags=os.O_CREAT | os.O_EXCL) + + # create a new netns or use existing one + netns = NetNS('test', flags=os.O_CREAT) + + # the same as above, the default behaviour + netns = NetNS('test') + +To remove a network namespace:: + + from pyroute2 import NetNS + netns = NetNS('test') + netns.close() + netns.remove() + +One should stop it first with `close()`, and only after that +run `remove()`. + +''' + +import atexit +import errno +import logging +import os +from functools import partial + +from pyroute2.iproute import RTNL_API +from pyroute2.netlink.rtnl import RTMGRP_DEFAULTS +from pyroute2.netlink.rtnl.iprsocket import MarshalRtnl +from pyroute2.netns import remove, setns + +from ..remote.transport import RemoteSocket, Server, Transport + +log = logging.getLogger(__name__) + + +class FD(object): + def __init__(self, fd): + self.fd = fd + for name in ('read', 'write', 'close'): + setattr(self, name, partial(getattr(os, name), self.fd)) + + def fileno(self): + return self.fd + + def flush(self): + return None + + +class NetNS(RTNL_API, RemoteSocket): + ''' + NetNS is the IPRoute API with network namespace support. + + **Why not IPRoute?** + + The task to run netlink commands in some network namespace, being in + another network namespace, requires the architecture, that differs + too much from a simple Netlink socket. + + NetNS starts a proxy process in a network namespace and uses + `multiprocessing` communication channels between the main and the proxy + processes to route all `recv()` and `sendto()` requests/responses. + + **Any specific API calls?** + + Nope. `NetNS` supports all the same, that `IPRoute` does, in the same + way. It provides full `socket`-compatible API and can be used in + poll/select as well. + + The only difference is the `close()` call. In the case of `NetNS` it + is **mandatory** to close the socket before exit. + + ''' + + def __init__( + self, + netns, + flags=os.O_CREAT, + target=None, + libc=None, + groups=RTMGRP_DEFAULTS, + ): + self.netns = netns + self.flags = flags + target = target or netns + trnsp_in, self.remote_trnsp_out = [Transport(FD(x)) for x in os.pipe()] + self.remote_trnsp_in, trnsp_out = [Transport(FD(x)) for x in os.pipe()] + + self.child = os.fork() + if self.child == 0: + # child process + trnsp_in.close() + trnsp_out.close() + trnsp_in.file_obj.close() + trnsp_out.file_obj.close() + try: + setns(self.netns, self.flags, libc=libc) + except OSError as e: + (self.remote_trnsp_out.send({'stage': 'init', 'error': e})) + os._exit(e.errno) + except Exception as e: + ( + self.remote_trnsp_out.send( + { + 'stage': 'init', + 'error': OSError(errno.ECOMM, str(e), self.netns), + } + ) + ) + os._exit(255) + + try: + Server( + self.remote_trnsp_in, + self.remote_trnsp_out, + target=target, + groups=groups, + ) + finally: + os._exit(0) + + try: + self.remote_trnsp_in.close() + self.remote_trnsp_out.close() + super(NetNS, self).__init__(trnsp_in, trnsp_out, groups=groups) + self.target = target + except Exception: + self.close() + raise + atexit.register(self.close) + self.marshal = MarshalRtnl() + + def clone(self): + return type(self)(self.netns, self.flags) + + def _cleanup_atexit(self): + if hasattr(atexit, 'unregister'): + atexit.unregister(self.close) + else: + try: + atexit._exithandlers.remove((self.close, (), {})) + except ValueError: + pass + + def close(self, code=errno.ECONNRESET): + self._cleanup_atexit() + try: + super(NetNS, self).close(code=code) + except: + # something went wrong, force server shutdown + try: + self.trnsp_out.send({'stage': 'shutdown'}) + except Exception: + pass + log.error('forced shutdown procedure, clean up netns manually') + + def open_file(self, path): + '''Proxy the open_file method if we are the parent.''' + if self.child != 0: + return self.proxy('open_file', path) + + return super(NetNS, self).open_file(path) + + def close_file(self, fd): + '''Proxy the close_file method if we are the parent.''' + if self.child != 0: + return self.proxy('close_file', fd) + + return super(NetNS, self).close_file(fd) + + def get_pid(self): + '''Proxy the get_pid method if we are the parent.''' + if self.child != 0: + return self.proxy('get_pid') + + return super(NetNS, self).get_pid() + + def post_init(self): + pass + + def remove(self): + ''' + Try to remove this network namespace from the system. + ''' + remove(self.netns) diff --git a/pyroute2/plan9/__init__.py b/pyroute2/plan9/__init__.py deleted file mode 100644 index aa16f88b2..000000000 --- a/pyroute2/plan9/__init__.py +++ /dev/null @@ -1,430 +0,0 @@ -import builtins -import json -import struct - -from pyroute2.netlink import nlmsg -from pyroute2.netlink.nlsocket import Marshal - -Tversion = 100 -Rversion = 101 -Tauth = 102 -Rauth = 103 -Tattach = 104 -Rattach = 105 -Terror = 106 # illegal -Rerror = 107 -Tflush = 108 -Rflush = 109 -Twalk = 110 -Rwalk = 111 -Topen = 112 -Ropen = 113 -Tcreate = 114 -Rcreate = 115 -Tread = 116 -Rread = 117 -Twrite = 118 -Rwrite = 119 -Tclunk = 120 -Rclunk = 121 -Tremove = 122 -Rremove = 123 -Tstat = 124 -Rstat = 125 -Twstat = 126 -Rwstat = 127 -Topenfd = 98 -Ropenfd = 99 - -# 9P2000.pr2 extensions -Tcall = 80 -Rcall = 81 - - -class Plan9Exit(Exception): - pass - - -def array(kind, header='H'): - class CustomArray: - - @staticmethod - def decode_count(data, offset): - (count,) = struct.unpack_from(header, data, offset) - return count, offset + struct.calcsize(header) - - @staticmethod - def decode_from(data, offset): - count, offset = CustomArray.decode_count(data, offset) - ret = [] - for _ in range(count): - value, offset = kind.decode_from(data, offset) - ret.append(value) - return ret, offset - - @staticmethod - def encode_into(data, offset, value): - if not isinstance(value, (tuple, list)): - value = [] - data.extend([0] * struct.calcsize(header)) - struct.pack_into(header, data, offset, len(value)) - offset += struct.calcsize(header) - for item in value: - offset = kind.encode_into(data, offset, item) - return offset - - return CustomArray - - -class Qid(dict): - length = 13 - - def __init__(self, qtype, vers, path): - self['type'] = qtype - self['vers'] = vers - self['path'] = path - - @staticmethod - def decode_from(data, offset): - return ( - dict( - zip( - ('type', 'vers', 'path'), - struct.unpack_from('=BIQ', data, offset), - ) - ), - offset + struct.calcsize('=BIQ'), - ) - - @staticmethod - def encode_into(data, offset, value): - data.extend([0] * Qid.length) - struct.pack_into( - '=BIQ', data, offset, value['type'], value['vers'], value['path'] - ) - return offset + Qid.length - - -class Stat(dict): - header_fmt = 'H' - - def __init__(self): - self['size'] = 58 - self['type'] = 0 - self['dev'] = 0 - self['qid.type'] = 0 - self['qid.vers'] = 0 - self['qid.path'] = 0 - self['mode'] = 0 - self['atime'] = 0 - self['mtime'] = 0 - self['length'] = 0 - self['name'] = '' - self['uid'] = '' - self['gid'] = '' - self['muid'] = '' - - @staticmethod - def decode_from(data, offset=0): - ret = dict( - zip( - ( - 'size', - 'type', - 'dev', - 'qid.type', - 'qid.vers', - 'qid.path', - 'mode', - 'atime', - 'mtime', - 'length', - ), - struct.unpack_from('=HHIBIQIIIQ', data, offset), - ) - ) - offset += 41 - for key in ('name', 'uid', 'gid', 'muid'): - ret[key], offset = String.decode_from(data, offset) - return ret, offset - - @staticmethod - def encode_into(data, offset, value): - data.extend([0] * 41) - # size of all the data except uint16 `size` header - # 41 + 2 + name + 2 + uid + 2 + gid + 2 + muid - value['size'] = ( - 47 - + len(value['name']) - + len(value['uid']) - + len(value['gid']) - + len(value['muid']) - ) - struct.pack_into( - '=HHIBIQIIIQ', - data, - offset, - value['size'], - value['type'], - value['dev'], - value['qid.type'], - value['qid.vers'], - value['qid.path'], - value['mode'], - value['atime'], - value['mtime'], - value['length'], - ) - offset += 41 - for key in ('name', 'uid', 'gid', 'muid'): - offset = String.encode_into(data, offset, value[key]) - return offset - - -class WStat(Stat): - @staticmethod - def decode_from(data, offset=0): - # just ignore plength for now - return Stat.decode_from(data, offset + 2) - - @staticmethod - def encode_into(data, offset, value): - data.extend([0] * 2) - new_offset = Stat.encode_into(data, offset + 2, value) - # size of all the data except uint16 `plength` header - struct.pack_into('H', data, offset, new_offset - offset - 2) - return new_offset - - -class CData: - header_fmt = 'I' - - @staticmethod - def decode_from(data, offset=0): - (length,) = struct.unpack_from(CData.header_fmt, data, offset) - offset += struct.calcsize(CData.header_fmt) - return bytearray(data[offset : offset + length]), offset + length - - @staticmethod - def encode_into(data, offset, value): - length = len(value) - if isinstance(value, str): - value = value.encode('utf-8') - data.extend([0] * (length + struct.calcsize(CData.header_fmt))) - struct.pack_into( - f'{CData.header_fmt}{length}s', data, offset, length, value - ) - return offset + length + struct.calcsize(CData.header_fmt) - - -class String: - header_fmt = 'H' - - @staticmethod - def decode_from(data, offset=0): - (length,) = struct.unpack_from(String.header_fmt, data, offset) - offset += struct.calcsize(String.header_fmt) - (value,) = struct.unpack_from(f'{length}s', data, offset) - value = value.decode('utf-8') - return value, offset + length - - @staticmethod - def encode_into(data, offset, value): - length = len(value) - data.extend([0] * (length + struct.calcsize(String.header_fmt))) - struct.pack_into( - f'{String.header_fmt}{length}s', - data, - offset, - length, - value.encode('utf-8'), - ) - return offset + length + struct.calcsize(String.header_fmt) - - -class msg_base(nlmsg): - align = 0 - header = (('length', '=I'), ('type', 'B'), ('tag', 'H')) - - -class msg_terror(msg_base): - defaults = {'header': {'type': Terror}} - - -class msg_rerror(msg_base): - defaults = {'header': {'type': Rerror}} - fields = (('ename', String),) - - -class msg_tversion(msg_base): - defaults = {'header': {'type': Tversion}} - fields = (('msize', 'I'), ('version', String)) - - -class msg_rversion(msg_base): - defaults = {'header': {'type': Rversion}} - fields = (('msize', 'I'), ('version', String)) - - -class msg_tauth(msg_base): - defaults = {'header': {'type': Tauth}} - fields = (('afid', 'I'), ('uname', String), ('aname', String)) - - -class msg_rauth(msg_base): - defaults = {'header': {'type': Rauth}} - fields = (('aqid', '13B'),) - - -class msg_tattach(msg_base): - defaults = {'header': {'type': Tattach}} - fields = ( - ('fid', 'I'), - ('afid', 'I'), - ('uname', String), - ('aname', String), - ) - - -class msg_rattach(msg_base): - defaults = {'header': {'type': Rattach}} - fields = (('qid', Qid),) - - -class msg_twalk(msg_base): - defaults = {'header': {'type': Twalk}} - fields = (('fid', 'I'), ('newfid', 'I'), ('wname', array(String))) - - -class msg_rwalk(msg_base): - defaults = {'header': {'type': Rwalk}} - fields = (('wqid', array(Qid)),) - - -class msg_tstat(msg_base): - defaults = {'header': {'type': Tstat}} - fields = (('fid', 'I'),) - - -class msg_rstat(msg_base): - defaults = {'header': {'type': Rstat}} - fields = (('stat', WStat),) - - -class msg_twstat(msg_base): - defaults = {'header': {'type': Twstat}} - fields = (('fid', 'I'), ('stat', WStat)) - - -class msg_rwstat(msg_base): - defaults = {'header': {'type': Rwstat}} - - -class msg_tclunk(msg_base): - defaults = {'header': {'type': Tclunk}} - fields = (('fid', 'I'),) - - -class msg_rclunk(msg_base): - defaults = {'header': {'type': Rclunk}} - pass - - -class msg_topen(msg_base): - defaults = {'header': {'type': Topen}} - fields = (('fid', 'I'), ('mode', 'B')) - - -class msg_ropen(msg_base): - defaults = {'header': {'type': Ropen}} - fields = (('qid', Qid), ('iounit', 'I')) - - -class msg_tread(msg_base): - defaults = {'header': {'type': Tread}} - fields = (('fid', 'I'), ('offset', 'Q'), ('count', 'I')) - - -class msg_rread(msg_base): - defaults = {'header': {'type': Rread}, 'data': b''} - fields = (('data', CData),) - - -class msg_twrite(msg_base): - defaults = {'header': {'type': Twrite}} - fields = (('fid', 'I'), ('offset', 'Q'), ('data', CData)) - - -class msg_rwrite(msg_base): - defaults = {'header': {'type': Rwrite}} - fields = (('count', 'I'),) - - -class msg_tcall(msg_base): - defaults = {'header': {'type': Tcall}} - fields = (('fid', 'I'), ('text', String), ('data', CData)) - - -class msg_rcall(msg_base): - defaults = {'header': {'type': Rcall}} - fields = (('err', 'H'), ('text', String), ('data', CData)) - - -class Marshal9P(Marshal): - default_message_class = msg_rerror - - msg_map = { - Tversion: msg_tversion, - Rversion: msg_rversion, - Tauth: msg_tauth, - Rauth: msg_rauth, - Tattach: msg_tattach, - Rattach: msg_rattach, - Rerror: msg_rerror, - Twalk: msg_twalk, - Rwalk: msg_rwalk, - Topen: msg_topen, - Ropen: msg_ropen, - Tread: msg_tread, - Rread: msg_rread, - Tclunk: msg_tclunk, - Rclunk: msg_rclunk, - Tstat: msg_tstat, - Rstat: msg_rstat, - Twstat: msg_twstat, - Rwstat: msg_rwstat, - Twrite: msg_twrite, - Rwrite: msg_rwrite, - Tcall: msg_tcall, - Rcall: msg_rcall, - Tcreate: msg_base, - Rcreate: msg_base, - Tremove: msg_base, - Rremove: msg_base, - } - - def parse(self, data, seq=None, callback=None, skip_alien_seq=False): - offset = 0 - while offset <= len(data) - 5: - (length, key, tag) = struct.unpack_from('=IBH', data, offset) - if skip_alien_seq and tag != seq: - continue - if not 0 < length <= len(data): - break - parser = self.get_parser(key, 0, tag) - msg = parser(data, offset, length) - if key == Rerror: - spec = json.loads(msg['ename']) - if spec['class'] in dir(builtins): - cls = getattr(builtins, spec['class']) - elif spec['class'] == 'Plan9Exit': - cls = Plan9Exit - else: - cls = Exception - if not spec.get('argv'): - spec['argv'] = [spec['str']] - raise cls(*spec['argv']) - offset += length - if msg is None: - continue - yield msg diff --git a/pyroute2/plan9/client.py b/pyroute2/plan9/client.py deleted file mode 100644 index 7035b91f2..000000000 --- a/pyroute2/plan9/client.py +++ /dev/null @@ -1,204 +0,0 @@ -import json -import os -import pwd -import struct - -from pyroute2.common import AddrPool -from pyroute2.netlink.core import ( - AsyncCoreSocket, - CoreMessageQueue, - CoreStreamProtocol, -) -from pyroute2.netlink.coredata import CoreConfig, CoreSocketSpec -from pyroute2.plan9 import ( - Marshal9P, - msg_tattach, - msg_tcall, - msg_tread, - msg_tversion, - msg_twalk, - msg_twrite, -) - - -class Plan9ClientSocket(AsyncCoreSocket): - '''9p2000 client. - - * address -- `('address', port)` to listen on - * use_socket -- alternatively, provide a connected SOCK_STRAM socket - ''' - - def __init__(self, address=None, use_socket=None, use_event_loop=None): - self.spec = CoreSocketSpec( - CoreConfig( - tag_field='tag', - target='localhost', - netns=None, - address=address, - use_socket=use_socket is not None, - use_event_loop=use_event_loop is not None, - ) - ) - self.marshal = Marshal9P() - self.wnames = {'': 0} - self.cwd = 0 - self.fid_pool = AddrPool(minaddr=0x00000001, maxaddr=0x0000FFFF) - super().__init__(use_socket=use_socket, use_event_loop=use_event_loop) - - def enqueue(self, data, addr): - tag = struct.unpack_from('H', data, 5)[0] - return self.msg_queue.put_nowait(tag, data) - - async def setup_endpoint(self, loop=None): - if getattr(self.local, 'transport', None) is not None: - return - self.local.msg_queue = CoreMessageQueue(event_loop=self.event_loop) - if self.status['use_socket']: - address = {'sock': self.use_socket} - else: - address = { - 'host': self.status['address'][0], - 'port': self.status['address'][1], - } - self.local.transport, self.local.protocol = ( - await self.event_loop.create_connection( - lambda: CoreStreamProtocol( - self.connection_lost, - self.enqueue, - self._error_event, - self.status, - ), - **address, - ) - ) - - async def start_session(self): - '''Initiate 9p2000 session. - - One must await this routine before running any other requests. - ''' - await self.setup_endpoint() - await self.version() - await self.auth() - await self.attach() - - async def request(self, msg, tag=0): - await self.setup_endpoint() - if tag == 0: - tag = self.addr_pool.alloc() - try: - msg['header']['tag'] = tag - msg.reset() - msg.encode() - self.msg_queue.ensure_tag(tag) - self.transport.write(msg.data) - return [x async for x in self.get(msg_seq=tag)][0] - finally: - self.addr_pool.free(tag, ban=0xFF) - - async def version(self): - '''`Tverion` request. No arguments required.''' - m = msg_tversion() - m['header']['tag'] = 0xFFFF - m['msize'] = 8192 - m['version'] = '9P2000' - return await self.request(m, tag=0xFFFF) - - async def auth(self): - pass - - async def attach(self, aname=''): - '''`Tattach` request. - - * `aname` (optional) -- aname to attach to - ''' - m = msg_tattach() - m['fid'] = 0 - m['afid'] = 0xFFFFFFFF - m['uname'] = pwd.getpwuid(os.getuid()).pw_name - m['aname'] = aname - return await self.request(m) - - async def walk(self, path, newfid=None, fid=None): - '''`Twalk` request. - - * `path` -- string path to the file - * `newfid` (optional) -- use this fid to store the info - * `fid` (optional) -- use this fid to walk from, otherwise walk - from the current directory for this client session - ''' - m = msg_twalk() - m['fid'] = self.cwd if fid is None else fid - m['newfid'] = newfid if newfid is not None else self.fid_pool.alloc() - m['wname'] = path.split(os.path.sep) - self.wnames[path] = m['newfid'] - return await self.request(m) - - async def fid(self, path): - '''Walk the path and return `fid` to the required file. - - * `path` -- string path to the file - ''' - if path not in self.wnames: - newfid = self.fid_pool.alloc() - await self.walk(path, newfid) - self.wnames[path] = newfid - return self.wnames[path] - - async def read(self, fid, offset=0, count=8192): - '''`Tread` request. - - * `fid` -- fid of the file to read from - * `offset` (optional, default 0) -- read offset - * `count` (optional, default 8192) -- read count - ''' - m = msg_tread() - m['fid'] = fid - m['offset'] = offset - m['count'] = count - return await self.request(m) - - async def write(self, fid, data, offset=0): - '''`Twrite` request. - - * `fid` -- fid of the file to write to - * `data` -- bytes to write - * `offset` (optional, default 0) -- write offset - ''' - m = msg_twrite() - m['fid'] = fid - m['offset'] = 0 - m['data'] = data - return await self.request(m) - - async def call( - self, - fid, - argv=None, - kwarg=None, - data=b'', - data_arg='data', - loader=json.loads, - ): - '''`Tcall` request. - - * `fid` -- fid of the file that represents a registered function - * `argv` (optional) -- positional arguments as an iterable - * `kwarg` (optional) -- keyword arguments as a dictionary - * `data` (opional) -- optional binary data - * `data_arg` (optional) -- name of the argument to use with - the binary data - * `loader` (optional, default `json.loads`) -- loader for the - response data - ''' - spec = { - 'argv': argv if argv is not None else [], - 'kwarg': kwarg if kwarg is not None else {}, - 'data_arg': data_arg, - } - m = msg_tcall() - m['fid'] = fid - m['text'] = json.dumps(spec) - m['data'] = data - response = await self.request(m) - return loader(response['data']) diff --git a/pyroute2/plan9/filesystem.py b/pyroute2/plan9/filesystem.py deleted file mode 100644 index bae6c7175..000000000 --- a/pyroute2/plan9/filesystem.py +++ /dev/null @@ -1,236 +0,0 @@ -import grp -import io -import json -import os -import pwd -import time -from dataclasses import dataclass -from functools import partial - -from pyroute2.plan9 import Plan9Exit, Qid, Stat, Tcall, Tread, Twrite - - -def _publish_function_w(session, inode, request, response): - inode.metadata.dirty = True - inode.data.seek(0) - inode.data.truncate() - inode.data.write(request['data']) - response['count'] = len(request['data']) - return response - - -def _publish_function_r( - func, loader, dumper, session, inode, request, response -): - if request['offset'] == 0 and inode.metadata.has_new_data(): - try: - kwarg = loader(inode.data.getvalue()) - ret = func(**kwarg) - inode.metadata.dirty = False - except Plan9Exit: - raise - except Exception as e: - ret = e - inode.data.seek(0) - inode.data.truncate() - inode.data.write(dumper(ret)) - - inode.data.seek(request['offset']) - response['data'] = inode.data.read(request['count']) - return response - - -def _publish_function_c( - func, loader, dumper, session, inode, request, response -): - spec = loader(request['text']) - - if 'argv' not in spec: - spec['argv'] = [] - if 'kwarg' not in spec: - spec['kwarg'] = {} - if 'data_arg' not in spec: - spec['data_arg'] = 'data' - data = request['data'] - if data: - spec['kwarg'][spec['data_arg']] = data - ret = func(*spec['argv'], **spec['kwarg']) - - response['err'] = 0 - response['text'] = '' - response['data'] = dumper(ret) - - return response - - -@dataclass -class InodeMetadata: - call_on_read: bool = False - dirty: bool = False - - def has_new_data(self) -> bool: - return self.dirty or self.call_on_read - - -class Inode: - children = None - parents = None - qid = None - stat = None - data = None - callbacks = None - - def __init__( - self, - name, - path, - data='', - qtype=0, - mode=0o600, - uid=None, - gid=None, - parents=None, - children=None, - ): - self.data = io.BytesIO(data.encode('utf-8')) - self.parents = parents if parents is not None else set() - self.children = children if children is not None else set() - self.callbacks = {} - self.metadata = InodeMetadata() - self.stat = Stat() - self.qid = Qid(qtype, 0, path) - self.stat['uid'] = ( - uid if uid is not None else pwd.getpwuid(os.getuid()).pw_name - ) - self.stat['gid'] = ( - gid if gid is not None else grp.getgrgid(os.getgid()).gr_name - ) - self.stat['muid'] = self.stat['uid'] - self.stat['qid.type'] = self.qid['type'] - self.stat['qid.vers'] = self.qid['vers'] - self.stat['qid.path'] = self.qid['path'] - # shift qid.type 3 bytes left and OR with mode - self.stat['mode'] = (self.qid['type'] << (8 * 3)) | mode - self.stat['type'] = self.qid['path'] - self.stat['dev'] = 0 - self.stat['mtime'] = int(time.time()) - self.stat['atime'] = int(time.time()) - self.stat['name'] = name - self.sync() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - pass - - def sync(self): - self.stat['length'] = len(self.data.getvalue()) - - def get_parent(self, name=None): - return tuple(self.parents)[0] - - def get_child(self, name): - for child in self.children: - if child.stat['name'] == name: - return child - raise KeyError('file not found') - - def register_function( - self, - func, - loader=json.loads, - dumper=lambda x: x if isinstance(x, bytes) else str(x).encode('utf-8'), - ): - self.add_callback(Twrite, _publish_function_w) - self.add_callback( - Tread, partial(_publish_function_r, func, loader, dumper) - ) - self.add_callback( - Tcall, partial(_publish_function_c, func, loader, dumper) - ) - - def add_callback(self, call, f): - self.callbacks[call] = f - return self - - def add_parent(self, inode): - return self.parents.add(inode) - - def add_child(self, inode): - inode.add_parent(self) - return self.children.add(inode) - - def del_parent(self, inode, force=False): - if force or len(self.parents) > 1: - return self.parents.remove(inode) - raise KeyError('can not remove the only parent') - - def del_child(self, inode): - inode.del_parent(self, force=True) - return self.children.remove(inode) - - def __id__(self): - return self.qid['path'] - - -class Filesystem: - - inodes = {} - - def __init__(self): - self.__path = 255 - # create the root inode - path = 0 - self.inodes[path] = root = Inode('/', path, qtype=0x80, mode=0o755) - root.add_parent(root) - - @property - def new_path(self): - self.__path += 1 - return self.__path - - def walk(self, wname): - inode = self.inodes[0] - for name in wname.split(os.path.sep): - if name == '': - continue - inode = inode.get_child(name) - return inode - - def create(self, wname, qtype=0, mode=None, data=''): - if mode is None: - mode = 0o750 if qtype & 0x80 else 0o640 - pname, name = os.path.split(wname) - parent = self.walk(pname) - path = self.new_path - inode = Inode(name, path, qtype=qtype, mode=mode, data=data) - parent.add_child(inode) - return inode - - def get_inode(self, path): - return self.inodes[path] - - def set_inode(self, path, inode): - self.inodes[path] = inode - - -class Session: - - fid_table = {} - - def __init__(self, filesystem): - self.filesystem = filesystem - - @property - def root(self): - return self.filesystem.inodes[0] - - def set_fid(self, fid, inode): - self.fid_table[fid] = inode - - def get_fid(self, fid): - return self.fid_table[fid] - - def create(self, name, mode): - pass diff --git a/pyroute2/plan9/ipc.py b/pyroute2/plan9/ipc.py deleted file mode 100644 index 46a5baa68..000000000 --- a/pyroute2/plan9/ipc.py +++ /dev/null @@ -1,51 +0,0 @@ -import collections -import multiprocessing -from socket import socketpair - -from pyroute2.netlink.nlsocket import NetlinkSocket -from pyroute2.plan9.client import Plan9ClientSocket - -IPCSocketPair = collections.namedtuple('IPCSocketPair', ('server', 'client')) - - -class IPCSocket(NetlinkSocket): - - def setup_socket(self): - # create socket pair - sp = IPCSocketPair(*socketpair()) - # start the server - self.socket = sp - self.p9server = multiprocessing.Process(target=self.ipc_server) - self.p9server.daemon = True - self.p9server.start() - # create and init the client - self.p9client = Plan9ClientSocket(use_socket=sp.client) - self.p9client.init() - return sp - - def ipc_server(self): - raise NotImplementedError() - - def recv(self, buffersize, flags=0): - ret = self.p9client.call( - fid=self.p9client.fid('call'), - fname='recv', - kwarg={'buffersize': buffersize, 'flags': flags}, - ) - return ret['data'] - - def send(self, data, flags=0): - return self.p9client.call( - fid=self.p9client.fid('call'), - fname='send', - kwarg={'flags': flags}, - data=data, - ) - - def bind(self): - return self.p9client.call(fid=self.p9client.fid('call'), fname='bind') - - def close(self): - self.socket.client.close() - self.socket.server.close() - self.p9server.wait() diff --git a/pyroute2/plan9/plan9socket.py b/pyroute2/plan9/plan9socket.py deleted file mode 100644 index 710e6ecca..000000000 --- a/pyroute2/plan9/plan9socket.py +++ /dev/null @@ -1,9 +0,0 @@ -from pyroute2.netlink.core import CoreSocket -from pyroute2.plan9 import Marshal9P - - -class Plan9Socket(CoreSocket): - def __init__(self, use_socket=None): - super().__init__(use_socket=use_socket) - self.marshal = Marshal9P() - self.spec['tag_field'] = 'tag' diff --git a/pyroute2/plan9/server.py b/pyroute2/plan9/server.py deleted file mode 100644 index 30154d668..000000000 --- a/pyroute2/plan9/server.py +++ /dev/null @@ -1,471 +0,0 @@ -import asyncio -import json - -from pyroute2.netlink.core import AsyncCoreSocket, CoreMessageQueue -from pyroute2.netlink.coredata import CoreConfig, CoreSocketSpec -from pyroute2.plan9 import ( - Marshal9P, - Plan9Exit, - Stat, - Tattach, - Tauth, - Tcall, - Tclunk, - Tcreate, - Topen, - Tread, - Tremove, - Tstat, - Tversion, - Twalk, - Twrite, - Twstat, - msg_rattach, - msg_rcall, - msg_rclunk, - msg_rerror, - msg_ropen, - msg_rread, - msg_rstat, - msg_rversion, - msg_rwalk, - msg_rwrite, - msg_rwstat, -) -from pyroute2.plan9.filesystem import Filesystem, Session - -data = str(dir()) - - -def get_exception_args(exc): - args = [] - if hasattr(exc, 'errno'): - args.append(exc.errno) - args.append(exc.strerror) - return args - - -def route(rtable, request, state): - def decorator(f): - rtable[request] = f - return f - - return decorator - - -class Plan9ServerProtocol(asyncio.Protocol): - rtable = {} - - def __init__(self, on_con_lost, marshal, filesystem): - self.transport = None - self.session = None - self.filesystem = filesystem - self.marshal = marshal - self.on_con_lost = on_con_lost - - @route(rtable, request=Tversion, state=(None,)) - def t_version(self, req): - m = msg_rversion() - m['header']['tag'] = 0xFFFF - m['msize'] = req['msize'] - m['version'] = '9P2000' - return m - - @route(rtable, request=Tauth, state=(Tversion,)) - def t_auth(self, req): - m = msg_rerror() - m['ename'] = 'no authentication required' - return m - - @route(rtable, request=Tattach, state=(Tauth,)) - def t_attach(self, req): - m = msg_rattach() - root = self.session.filesystem.inodes[0] - self.session.set_fid(req['fid'], root) - m['qid'] = root.qid - return m - - @route(rtable, request=Twalk, state=(Tattach,)) - def t_walk(self, req): - m = msg_rwalk() - inode = self.session.get_fid(req['fid']) - wqid = [] - if len(req['wname']) == 0: - self.session.set_fid(req['newfid'], inode) - else: - for name in req['wname']: - if name == '.': - continue - elif name == '..': - inode = inode.get_parent() - else: - inode = inode.get_child(name) - wqid.append(inode.qid) - m['wqid'] = wqid - self.session.set_fid(req['newfid'], inode) - return m - - @route(rtable, request=Tstat, state=(Twalk,)) - def t_stat(self, req): - m = msg_rstat() - inode = self.session.get_fid(req['fid']) - inode.sync() - m['stat'] = inode.stat - return m - - @route(rtable, request=Twstat, state=(Twalk,)) - def t_wstat(self, req): - m = msg_rwstat() - return m - - @route(rtable, request=Topen, state=(Twalk, Tstat)) - def t_open(self, req): - m = msg_ropen() - m['qid'] = self.session.get_fid(req['fid']).qid - m['iounit'] = 8192 - return m - - @route(rtable, request=Tcall, state=(Twalk, Topen, Tstat)) - def t_call(self, req): - m = msg_rcall() - inode = self.session.get_fid(req['fid']) - m['err'] = 255 - if Tcall in inode.callbacks: - m = inode.callbacks[Tcall](self.session, inode, req, m) - return m - - @route(rtable, request=Twrite, state=(Topen,)) - def t_write(self, req): - m = msg_rwrite() - inode = self.session.get_fid(req['fid']) - if Twrite in inode.callbacks: - return inode.callbacks[Twrite](self.session, inode, req, m) - if inode.qid['type'] & 0x80: - raise TypeError('can not call write() on dir') - inode.data.seek(req['offset']) - m['count'] = inode.data.write(req['data']) - return m - - @route(rtable, request=Tread, state=(Topen,)) - def t_read(self, req): - m = msg_rread() - inode = self.session.get_fid(req['fid']) - if Tread in inode.callbacks: - return inode.callbacks[Tread](self.session, inode, req, m) - if inode.qid['type'] & 0x80: - data = bytearray() - offset = 0 - for child in inode.children: - offset = Stat.encode_into(data, offset, child.stat) - data = data[req['offset'] : req['offset'] + req['count']] - else: - inode.data.seek(req['offset']) - data = inode.data.read(req['count']) - m['data'] = data - return m - - @route(rtable, request=Tclunk, state=(Topen, Tstat, Twalk, Tread)) - def t_clunk(self, req): - return msg_rclunk() - - @route(rtable, request=Tcreate, state=(Twalk,)) - def t_create(self, req): - return self.permission_denied(req) - - @route(rtable, request=Tremove, state=(Twalk,)) - def t_remove(self, req): - return self.permission_denied(req) - - def permission_denied(self, req): - r_message = msg_rerror() - r_message['ename'] = 'permission denied' - r_message['header']['tag'] = req['header']['tag'] - return r_message - - def error(self, e, tag=0): - r_message = msg_rerror() - spec = { - 'class': e.__class__.__name__, - 'argv': get_exception_args(e), - 'str': str(e), - } - r_message['ename'] = json.dumps(spec) - r_message['header']['tag'] = tag - r_message.encode() - self.transport.write(r_message.data) - - def data_received(self, data): - for t_message in self.marshal.parse(data): - tag = t_message['header']['tag'] - try: - r_message = self.rtable[t_message['header']['type']]( - self, t_message - ) - r_message['header']['tag'] = tag - r_message.encode() - except Plan9Exit as e: - self.error(e, tag) - self.transport.abort() - self.transport.close() - return - except Exception as e: - self.error(e, tag) - return - self.transport.write(r_message.data) - - def connection_made(self, transport): - self.transport = transport - self.session = Session(self.filesystem) - - -class Plan9ServerSocket(AsyncCoreSocket): - '''9p2000 server. - - Requires either an IP address to listen on, or an open - `SOCK_STREAM` socket to operate. An IP example, suitable - to establish IPC between processes in one network: - - .. testcode:: - - from pyroute2 import Plan9ClientSocket, Plan9ServerSocket - - address = ('localhost', 8149) - p9server = Plan9ServerSocket(address=address) - p9client = Plan9ClientSocket(address=address) - - Server/client running on a `socketpair()` suitable - for internal API within one process, or between - parent/child processes: - - .. testcode:: - - from socket import socketpair - - from pyroute2 import Plan9ClientSocket, Plan9ServerSocket - - server, client = socketpair() - p9server = Plan9ServerSocket(use_socket=server) - p9client = Plan9ClientSocket(use_socket=client) - - - ''' - - def __init__(self, address=None, use_socket=None, use_event_loop=None): - self.spec = CoreSocketSpec( - CoreConfig( - tag_field='tag', - target='localhost', - netns=None, - address=address, - use_socket=use_socket is not None, - use_event_loop=use_event_loop is not None, - ) - ) - self.filesystem = Filesystem() - self.marshal = Marshal9P() - super().__init__(use_socket=use_socket, use_event_loop=use_event_loop) - - def register_function( - self, - func, - inode, - loader=json.loads, - dumper=lambda x: json.dumps(x).encode('utf-8'), - ): - '''Register a function to an file. - - The file usage: - - * `write()`: write arguments for the call as a json dictionary of - keyword arguments to the file data. - * `read()`: - 1. if the arguments were written to the data, call the function - and write the result to the file data - 2. read the file data and return to the client - * `call()`: protocol extension, `Tcall` = 80, `Rcall` = 81, make - this in one turn. - - .. testcode:: - :hide: - - from pyroute2.plan9 import Tcall, Rcall - - assert Tcall == 80 - assert Rcall == 81 - - Registering a function: - - .. testcode:: - - def communicate(a, b): - return a + b - - - def example_register(): - fd = p9server.filesystem.create('test_func') - p9server.register_function(communicate, fd) - - Communication using Twrite/Tread: - - .. testcode:: - - import json - - - async def example_write(): - fid = await p9client.fid('test_func') - await p9client.write( - fid, - json.dumps({"a": 17, "b": 25}) - ) - msg = await p9client.read(fid) - response = json.loads(msg['data']) - assert response == 42 - - Same, using a command line 9p client from plan9port:: - - $ echo '{"a": 17, "b": 25}' | 9p -a localhost:8149 write test_func - $ 9p -a localhost:8149 read test_func - 42 - - And using a mounted file system via FUSE client from plan9port:: - - $ 9pfuse localhost:8149 mnt - $ echo '{"a": 17, "b": 25}' >mnt/test_func - $ cat mnt/test_func - 42 - - And the same, but using Tcall: - - .. testcode:: - - async def example_call(): - fid = await p9client.fid('test_func') - response = await p9client.call(fid, argv=(17, 25)) - assert response == 42 - - And finnaly run this code: - - .. testcode:: - - async def main(): - server_task = await p9server.async_run() - example_register() - await p9client.start_session() - await example_write() - await example_call() - server_task.cancel() - - asyncio.run(main()) - ''' - return inode.register_function(func, loader, dumper) - - async def setup_endpoint(self): - if getattr(self.local, 'server', None) is not None: - return - self.local.msg_queue = CoreMessageQueue(event_loop=self.event_loop) - if self.status['use_socket']: - self.local.server = None - self.local.transport, self.local.protocol = ( - await self.event_loop.create_connection( - lambda: Plan9ServerProtocol( - self.connection_lost, self.marshal, self.filesystem - ), - sock=self.use_socket, - ) - ) - else: - self.local.transport = None - self.local.protocol = None - self.local.server = await self.event_loop.create_server( - lambda: Plan9ServerProtocol( - self.connection_lost, self.marshal, self.filesystem - ), - *self.status['address'], - ) - - async def async_run(self): - '''Return the server asyncio task. - - Using this task one can stop the server: - - .. testcode:: - - async def main(): - server = Plan9ServerSocket(address=('localhost', 8149)) - server_task = await server.async_run() - # ... server is running here - server_task.cancel() - # ... server is stopped - - asyncio.run(main()) - - To forcefully close all client connections and stop the server - immediately from a registered function, one can pass this task - to the function, cancel it, and raise `Plan9Exit()` exception: - - .. testcode:: - - import functools - - from pyroute2.plan9 import Plan9Exit - - server_sock, client_sock = socketpair() - - - def test_exit_func(context): - if 'server_task' in context: - context['server_task'].cancel() - raise Plan9Exit('server stopped upon client request') - return 'server starting, please wait' - - - async def server(): - p9server = Plan9ServerSocket(use_socket=server_sock) - context = {} - - inode = p9server.filesystem.create('stop') - p9server.register_function( - functools.partial(test_exit_func, context), - inode - ) - context['server_task'] = await p9server.async_run() - - try: - await context['server_task'] - except asyncio.exceptions.CancelledError: - pass - - assert context['server_task'].cancelled() - - .. testcode:: - :hide: - - async def client(): - p9client = Plan9ClientSocket(use_socket=client_sock) - await p9client.start_session() - fid = await p9client.fid('stop') - try: - await p9client.call(fid) - except Plan9Exit: - pass - - - async def main(): - await asyncio.gather(server(), client()) - - asyncio.run(main()) - ''' - await self.setup_endpoint() - if self.status['use_socket']: - return self.protocol.on_con_lost - else: - return asyncio.create_task(self.local.server.serve_forever()) - - def run(self): - '''A simple synchronous runner. - - Uses `event_loop.run_forever()`. - ''' - self.event_loop.create_task(self.async_run()) - self.event_loop.run_forever() diff --git a/pyroute2/process.py b/pyroute2/process.py deleted file mode 100644 index a3b219fd3..000000000 --- a/pyroute2/process.py +++ /dev/null @@ -1,245 +0,0 @@ -import builtins -import gc -import json -import logging -import multiprocessing -import os -import select -import signal -import socket -import struct -from collections import namedtuple -from typing import Any, Callable, Optional, Union - -from pyroute2 import config -from pyroute2.common import USE_DEFAULT_TIMEOUT -from pyroute2.netlink import exceptions as pyroute2_exceptions - -log = logging.getLogger(__name__) - -ChildProcessReturnValue = namedtuple( - 'ChildProcessReturnValue', ('payload', 'fds') -) -ChildFuncReturnType = Union[None, ChildProcessReturnValue, bytearray, bytes] - - -def wrapper( - ctrl: socket.socket, - func: Callable[..., ChildFuncReturnType], - argv: list[Any], -) -> None: - '''Child function wrapper. - - This code will be executed in the child process after running fork(). - The internal data structures might be damaged, so the function `func` - must be as simple as possible, and use only local variables of simple - types. - - The garbage collector should be disabled as well to minimize possible - deadlocks. - - If process doesn't response in time, it will get killed. - ''' - gc.disable() - payload: bytes = b'' - ret_data: bytes = b'' - fds: list[int] = [] - sockets: list[socket.socket] = [] - try: - ret = func(*argv) - if isinstance(ret, bytes): - ret_data = ret - if isinstance(ret, ChildProcessReturnValue): - ret_data, sockets = ret - if isinstance(ret_data, bytearray): - ret_data = bytes(ret_data) - if not isinstance(ret_data, bytes): - raise TypeError('return values not supported') - payload = struct.pack('B', 2) + ret_data - if sockets: - fds = [x.fileno() for x in sockets] - except Exception as e: - payload = struct.pack('B', 1) + json.dumps( - {'exception': e.__class__.__name__, 'options': e.args} - ).encode('utf-8') - fds = [] - finally: - socket.send_fds(ctrl, [payload], fds, len(fds)) - - -class ChildProcess: - ctrl_r: socket.socket - ctrl_w: socket.socket - - def __init__( - self, target: Callable[..., ChildFuncReturnType], args: list[Any] - ): - self._mode: str = config.child_process_mode - self._target: Callable[..., ChildFuncReturnType] = target - self._args: list[Any] = args - self._proc: Optional[multiprocessing.Process] = None - self._running: bool = False - self._exitcode: Optional[int] = None - self._pid: Optional[int] = None - - def close(self): - self.stop() - - def __enter__(self): - self.run() - return self - - def __exit__(self, *_): - self.close() - - @property - def mode(self): - return self._mode - - @property - def pid(self): - return self._pid - - def communicate( - self, timeout: int = USE_DEFAULT_TIMEOUT - ) -> tuple[bytes, list[int]]: - '''Communicate with the child process. - - Raises: - - * struct.error -- error unpacking response from the child process - * OSError -- OS level error communicating with the child process - * TimeoutError -- the child process is alive, but doesn't response - * RuntimeError -- the child process is dead or not started - * TypeError -- error loading propagated exception - ''' - if not self._running: - raise RuntimeError('child process not started yet') - if timeout == USE_DEFAULT_TIMEOUT: - timeout = config.default_communicate_timeout - rl, _, _ = select.select([self.ctrl_r], [], [], timeout) - if not len(rl): - # no data received within timeout - # 1. the child process is killed - # 2. the child process is stuck - # - # So first check the process, if it is killed, raise - # RuntimeError, otherwise raise TimeoutError. - # - c_pid, w_status = os.waitpid(self.pid, os.WNOHANG) - if c_pid != 0: - # process is dead - self._exitcode = os.waitstatus_to_exitcode(w_status) - raise RuntimeError( - f'child process is dead, status: {self.exitcode}' - ) - # process is alive, but stuck, kill it - self.stop(kill=True, reason='no response from the child') - raise TimeoutError(f'no response from the child pid {self.pid}') - - ret_data = b'' - # raises OSError - (raw_data, fds, _, _) = socket.recv_fds(self.ctrl_r, 1024, 1) - # get the return type - # raises struct.error - (ret_type,) = struct.unpack('B', raw_data[:1]) - raw_data = raw_data[1:] - if ret_type == 1: - # exception - payload = json.loads(raw_data.decode('utf-8')) - if not set(payload.keys()) == set(('exception', 'options')): - raise TypeError('error loading child exception') - if payload.get('exception') is not None: - error_class = getattr(builtins, payload['exception'], None) - if error_class is None: - error_class = getattr( - pyroute2_exceptions, payload['exception'], None - ) - if error_class is None: - error_class = Exception - if not issubclass(error_class, Exception): - raise TypeError('error loading child error') - raise error_class(*payload['options']) - elif ret_type == 2: - # raw_data - ret_data = raw_data - return ret_data, fds - - def get_data(self, timeout: int = USE_DEFAULT_TIMEOUT) -> bytes: - return self.communicate(timeout)[0] - - def get_fds(self, timeout: int = USE_DEFAULT_TIMEOUT) -> list[int]: - return self.communicate(timeout)[1] - - @property - def proc(self) -> multiprocessing.Process: - if self._proc is None: - raise RuntimeError('not started') - return self._proc - - @proc.setter - def proc(self, value: Optional[multiprocessing.Process]) -> None: - self._proc = value - - def _unsupported(self) -> Exception: - return TypeError('unsupported mode') - - def run(self) -> None: - if self._running: - return - self._running = True - self.ctrl_r, self.ctrl_w = socket.socketpair( - socket.AF_UNIX, socket.SOCK_DGRAM - ) - if self.mode == 'fork': - self._pid = os.fork() - if self._pid == 0: - wrapper(self.ctrl_w, self._target, self._args) - os._exit(0) - elif self.mode == 'mp': - self.proc = multiprocessing.Process( - target=wrapper, args=[self.ctrl_w, self._target, self._args] - ) - self.proc.start() - self._pid = self.proc.pid - else: - raise self._unsupported() - - def stop(self, kill: bool = False, reason: Optional[str] = None) -> None: - if not self._running: - return - self._running = False - self.ctrl_r.close() - self.ctrl_w.close() - if config.force_gc: - gc.collect() - if self.mode == 'fork': - try: - if kill: - os.kill(self.pid, signal.SIGKILL) - else: - os.kill(self.pid, signal.SIGTERM) - _, status = os.waitpid(self.pid, 0) - self._exitcode = os.waitstatus_to_exitcode(status) - except ProcessLookupError: - # the child process has already exited - pass - elif self.mode == 'mp': - if kill: - self.proc.kill() - else: - self.proc.terminate() - self.proc.join() - else: - raise self._unsupported() - if reason is not None: - log.warning(reason) - - @property - def exitcode(self) -> Optional[int]: - if self.mode == 'fork': - return self._exitcode - elif self.mode == 'mp': - return self.proc.exitcode - else: - raise self._unsupported() diff --git a/pyroute2/protocols/__init__.py b/pyroute2/protocols/__init__.py index e39901252..7951757ee 100644 --- a/pyroute2/protocols/__init__.py +++ b/pyroute2/protocols/__init__.py @@ -1,14 +1,5 @@ import struct from socket import AF_INET, inet_ntop, inet_pton -from typing import ( - Any, - Callable, - ClassVar, - Literal, - NamedTuple, - Optional, - Union, -) from pyroute2.common import basestring, hexdump @@ -115,107 +106,73 @@ ETH_P_CAIF = 0x00F7 # ST-Ericsson CAIF protocol -def _decode_mac(value: Union[bytes, tuple[int, ...]]) -> str: - if isinstance(value, tuple): - value = bytes(value) - if not len(value) == 6: - raise ValueError('Invalid mac address, must be 6 bytes long') - return value.hex(sep=':') - - -def _encode_mac(value: str) -> list[int]: - return [int(i, 16) for i in value.split(':')] - - -class Policy(NamedTuple): - '''Used in option/msg subclasses. - - The format is either a struct format or 'string'. - encode is called with the python value to serialize it. - decode is called with the result(s) of the unpacking to decode values. - ''' - - format: str - # FIXME: there are so many Any here it's nearly useless, - # encode/decode callbacks should have a clean definition instead - encode: Callable[[Any], Any] = lambda x: x - decode: Union[Callable[[Any], Any], Callable[[tuple[Any, ...]], Any]] = ( - lambda x: x - ) - - class msg(dict): - fields: ClassVar[ - tuple[Union[tuple[str, str, Union[int, bytes]], tuple[str, str]], ...] - ] = () - types: ClassVar[dict[str, Union[str, Policy]]] = { + buf = None + data_len = None + fields = () + _fields_names = () + types = { 'uint8': 'B', 'uint16': 'H', 'uint32': 'I', 'be16': '>H', - 'be32': '>I', - 'ip4addr': Policy( - format='4s', - decode=lambda x: inet_ntop(AF_INET, x), - encode=lambda x: [inet_pton(AF_INET, x)], - ), - 'l2addr': Policy(format='6B', decode=_decode_mac, encode=_encode_mac), - 'l2paddr': Policy( - format='6B10s', - decode=lambda x: _decode_mac(x[:6]), - encode=lambda x: _encode_mac(x) + [10 * b'\x00'], - ), + 'ip4addr': { + 'format': '4s', + 'decode': lambda x: inet_ntop(AF_INET, x), + 'encode': lambda x: [inet_pton(AF_INET, x)], + }, + 'l2addr': { + 'format': '6B', + 'decode': lambda x: ':'.join(['%x' % i for i in x]), + 'encode': lambda x: [int(i, 16) for i in x.split(':')], + }, + 'l2paddr': { + 'format': '6B10s', + 'decode': lambda x: ':'.join(['%x' % i for i in x[:6]]), + 'encode': lambda x: [int(i, 16) for i in x.split(':')] + + [10 * b'\x00'], + }, } - def __init__( - self, - content: Optional[dict] = None, - buf: bytes = b'', - offset: int = 0, - value: Any = None, - ) -> None: + def __init__(self, content=None, buf=b'', offset=0, value=None): content = content or {} dict.__init__(self, content) self.buf = buf self.offset = offset self.value = value - self._fields_names: tuple[str, ...] = () self._register_fields() - def _register_fields(self) -> None: + def _register_fields(self): self._fields_names = tuple([x[0] for x in self.fields]) - def _get_routine( - self, mode: Literal['encode', 'decode'], fmt: str - ) -> tuple[str, Callable[[Any], Any]]: - real_fmt = self.types.get(fmt, fmt) - if isinstance(real_fmt, Policy): - return (real_fmt.format, getattr(real_fmt, mode, lambda x: x)) + def _get_routine(self, mode, fmt): + fmt = self.types.get(fmt, fmt) + if isinstance(fmt, dict): + return (fmt['format'], fmt.get(mode, lambda x: x)) else: - return (real_fmt, lambda x: x) + return (fmt, lambda x: x) - def reset(self) -> None: + def reset(self): self.buf = b'' - def decode(self) -> 'msg': + def decode(self): + self._register_fields() for field in self.fields: name, sfmt = field[:2] fmt, routine = self._get_routine('decode', sfmt) size = struct.calcsize(fmt) - try: - value = struct.unpack_from(fmt, self.buf, self.offset) - except struct.error as err: - raise ValueError( - f'Cannot decode {type(self).__name__} {name}: {err}' - ) + value = struct.unpack( + fmt, self.buf[self.offset : self.offset + size] + ) if len(value) == 1: value = value[0] - if isinstance(value, bytes) and sfmt[-1] == 's': - value = value.lstrip(b'\x00') - try: - value = value.decode('utf-8') - except UnicodeDecodeError: - value = hexdump(value) + if isinstance(value, basestring) and sfmt[-1] == 's': + value = value[: value.find(b'\x00')] + if isinstance(sfmt, str) and sfmt[-1] == 's': + try: + value = value.decode('utf-8') + except UnicodeDecodeError: + value = hexdump(value) self[name] = routine(value) self.offset += size return self diff --git a/pyroute2/remote/__init__.py b/pyroute2/remote/__init__.py new file mode 100644 index 000000000..bd8085cf4 --- /dev/null +++ b/pyroute2/remote/__init__.py @@ -0,0 +1,8 @@ +try: + from .iproute import RemoteIPRoute +except ImportError: + from pyroute2.common import failed_class + + RemoteIPRoute = failed_class('mitogen library is not installed') + +classes = [RemoteIPRoute] diff --git a/pyroute2/remote/__main__.py b/pyroute2/remote/__main__.py new file mode 100644 index 000000000..bd41712cb --- /dev/null +++ b/pyroute2/remote/__main__.py @@ -0,0 +1,5 @@ +import sys + +from pyroute2.remote import Server, Transport + +Server(Transport(sys.stdin), Transport(sys.stdout)) diff --git a/pyroute2/remote/iproute.py b/pyroute2/remote/iproute.py new file mode 100644 index 000000000..d9441fa0a --- /dev/null +++ b/pyroute2/remote/iproute.py @@ -0,0 +1,149 @@ +import errno +import os +import threading + +import mitogen.core +import mitogen.master + +from pyroute2.iproute.linux import RTNL_API +from pyroute2.netlink.rtnl.iprsocket import MarshalRtnl + +from .transport import RemoteSocket, Server, Transport + + +class Channel(object): + def __init__(self, ch): + self.ch = ch + self._pfdr, self._pfdw = os.pipe() + self.th = None + self.closed = False + self.lock = threading.RLock() + self.shutdown_lock = threading.RLock() + self.read = self._read_sync + self.buf = '' + + def flush(self): + pass + + def _read_sync(self, size): + with self.lock: + if self.buf: + ret = self.buf[:size] + self.buf = self.buf[size:] + return ret + ret = self.ch.get().unpickle() + if len(ret) > size: + self.buf = ret[size:] + return ret[:size] + + def _read_async(self, size): + with self.lock: + return os.read(self._pfdr, size) + + def write(self, data): + with self.lock: + self.ch.send(data) + return len(data) + + def start(self): + with self.lock: + if self.th is None: + self.read = self._read_async + self.th = threading.Thread( + target=self._monitor_thread, + name='Channel <%s> I/O' % self.ch, + ) + self.th.start() + + def fileno(self): + return self._pfdr + + def close(self): + with self.shutdown_lock: + if not self.closed: + os.close(self._pfdw) + os.close(self._pfdr) + if self.th is not None: + self.th.join() + self.closed = True + if hasattr(self.ch, 'send'): + self.ch.send(None) + + def _monitor_thread(self): + while True: + msg = self.ch.get().unpickle() + if msg is None: + raise EOFError() + os.write(self._pfdw, msg) + + +@mitogen.core.takes_router +def MitogenServer(ch_out, netns, target, router): + ch_in = mitogen.core.Receiver(router) + ch_out.send(ch_in.to_sender()) + + trnsp_in = Transport(Channel(ch_in)) + trnsp_in.file_obj.start() + trnsp_out = Transport(Channel(ch_out)) + + return Server(trnsp_in, trnsp_out, netns, target) + + +class RemoteIPRoute(RTNL_API, RemoteSocket): + def __init__(self, *argv, **kwarg): + self._argv = tuple(argv) + self._kwarg = dict(kwarg) + if 'router' in kwarg: + self._mitogen_broker = None + self._mitogen_router = kwarg.pop('router') + else: + self._mitogen_broker = mitogen.master.Broker() + self._mitogen_router = mitogen.master.Router(self._mitogen_broker) + + netns = kwarg.pop('netns', None) + target = kwarg.pop('target', 'remote') + try: + if 'context' in kwarg: + context = kwarg['context'] + else: + protocol = kwarg.pop('protocol', 'local') + context = getattr(self._mitogen_router, protocol)( + *argv, **kwarg + ) + ch_in = mitogen.core.Receiver( + self._mitogen_router, respondent=context + ) + self._mitogen_call = context.call_async( + MitogenServer, + ch_out=ch_in.to_sender(), + netns=netns, + target=target, + ) + ch_out = ch_in.get().unpickle() + super(RemoteIPRoute, self).__init__( + Transport(Channel(ch_in)), Transport(Channel(ch_out)) + ) + except Exception: + if self._mitogen_broker is not None: + self._mitogen_broker.shutdown() + self._mitogen_broker.join() + raise + self.marshal = MarshalRtnl() + self.target = target + self.groups = 67372509 + + def clone(self): + return type(self)(*self._argv, **self._kwarg) + + def close(self, code=errno.ECONNRESET): + with self.shutdown_lock: + if not self.closed: + super(RemoteIPRoute, self).close(code=code) + self.closed = True + try: + self._mitogen_call.get() + except mitogen.core.ChannelError: + pass + if self._mitogen_broker is not None: + self._mitogen_broker.shutdown() + self._mitogen_broker.join() diff --git a/pyroute2/remote/shell.py b/pyroute2/remote/shell.py new file mode 100644 index 000000000..557e30bec --- /dev/null +++ b/pyroute2/remote/shell.py @@ -0,0 +1,72 @@ +import atexit +import errno +import logging +import struct +import subprocess + +from pyroute2.iproute import RTNL_API +from pyroute2.netlink.rtnl.iprsocket import MarshalRtnl +from pyroute2.remote.transport import RemoteSocket, Transport + +log = logging.getLogger(__name__) + + +class ShellIPR(RTNL_API, RemoteSocket): + def __init__(self, target): + self.target = target + cmd = '%s python -m pyroute2.remote' % target + self.shell = subprocess.Popen( + cmd.split(), + bufsize=0, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + trnsp_in = Transport(self.shell.stdout) + trnsp_out = Transport(self.shell.stdin) + + try: + super(ShellIPR, self).__init__(trnsp_in, trnsp_out) + except Exception: + self.close() + raise + atexit.register(self.close) + self.marshal = MarshalRtnl() + + def clone(self): + return type(self)(self.target) + + def _cleanup_atexit(self): + if hasattr(atexit, 'unregister'): + atexit.unregister(self.close) + else: + try: + atexit._exithandlers.remove((self.close, (), {})) + except ValueError: + pass + + def close(self, code=errno.ECONNRESET): + self._cleanup_atexit() + # something went wrong, force server shutdown + try: + self.trnsp_out.send({'stage': 'shutdown'}) + if code > 0: + data = { + 'stage': 'broadcast', + 'data': struct.pack('IHHQIQQ', 28, 2, 0, 0, code, 0, 0), + 'error': None, + } + self.trnsp_in.brd_queue.put(data) + except Exception: + pass + # force cleanup command channels + for close in (self.trnsp_in.close, self.trnsp_out.close): + try: + close() + except Exception: + pass # Maybe already closed in remote.Client.close + + self.shell.kill() + self.shell.wait() + + def post_init(self): + pass diff --git a/pyroute2/remote/transport.py b/pyroute2/remote/transport.py new file mode 100644 index 000000000..11b3a3d11 --- /dev/null +++ b/pyroute2/remote/transport.py @@ -0,0 +1,369 @@ +import atexit +import errno +import logging +import os +import pickle +import select +import signal +import struct +import threading +import traceback +from io import BytesIO +from socket import SO_RCVBUF, SOL_SOCKET + +from pyroute2 import config +from pyroute2 import netns as netnsmod +from pyroute2.netlink.nlsocket import NetlinkSocketBase + +if config.uname[0][-3:] == 'BSD': + from pyroute2.iproute.bsd import IPRoute +else: + from pyroute2.iproute.linux import IPRoute +try: + import queue +except ImportError: + import Queue as queue + +log = logging.getLogger(__name__) + + +class Transport(object): + ''' + A simple transport protocols to send objects between two + end-points. Requires an open file-like object at init. + ''' + + def __init__(self, file_obj): + self.file_obj = file_obj + self.lock = threading.Lock() + self.cmd_queue = queue.Queue() + self.brd_queue = queue.Queue() + self.run = True + + def fileno(self): + return self.file_obj.fileno() + + def send(self, obj): + dump = BytesIO() + pickle.dump(obj, dump) + packet = struct.pack("II", len(dump.getvalue()) + 8, 0) + packet += dump.getvalue() + self.file_obj.write(packet) + self.file_obj.flush() + + def __recv(self): + length, offset = struct.unpack("II", self.file_obj.read(8)) + dump = BytesIO() + dump.write(self.file_obj.read(length - 8)) + dump.seek(0) + ret = pickle.load(dump) + return ret + + def _m_recv(self, own_queue, other_queue, check): + while self.run: + if self.lock.acquire(False): + try: + try: + ret = own_queue.get(False) + if ret is None: + continue + else: + return ret + except queue.Empty: + pass + ret = self.__recv() + if not check(ret['stage']): + other_queue.put(ret) + else: + other_queue.put(None) + return ret + finally: + self.lock.release() + else: + ret = None + try: + ret = own_queue.get(timeout=1) + except queue.Empty: + pass + if ret is not None: + return ret + + def recv(self): + return self._m_recv( + self.brd_queue, self.cmd_queue, lambda x: x == 'broadcast' + ) + + def recv_cmd(self): + return self._m_recv( + self.cmd_queue, self.brd_queue, lambda x: x != 'broadcast' + ) + + def close(self): + self.run = False + + +class ProxyChannel(object): + def __init__(self, channel, stage): + self.target = channel + self.stage = stage + + def send(self, data): + return self.target.send( + {'stage': self.stage, 'data': data, 'error': None} + ) + + +def Server(trnsp_in, trnsp_out, netns=None, target='localhost', groups=0): + def stop_server(signum, frame): + Server.run = False + + Server.run = True + signal.signal(config.signal_stop_remote, stop_server) + + try: + if netns is not None: + netnsmod.setns(netns) + ipr = IPRoute(target=target, groups=groups) + lock = ipr._sproxy.lock + ipr._s_channel = ProxyChannel(trnsp_out, 'broadcast') + except Exception as e: + trnsp_out.send({'stage': 'init', 'error': e}) + return 255 + + inputs = [ipr.fileno(), trnsp_in.fileno()] + broadcasts = {ipr.fileno(): ipr} + outputs = [] + + # all is OK so far + trnsp_out.send({'stage': 'init', 'uname': config.uname, 'error': None}) + + # 8<------------------------------------------------------------- + while Server.run: + try: + events, _, _ = select.select(inputs, outputs, inputs) + except: + continue + for fd in events: + if fd in broadcasts: + sock = broadcasts[fd] + bufsize = sock.getsockopt(SOL_SOCKET, SO_RCVBUF) // 2 + with lock: + error = None + data = None + try: + data = sock.recv(bufsize) + except Exception as e: + error = e + error.tb = traceback.format_exc() + trnsp_out.send( + {'stage': 'broadcast', 'data': data, 'error': error} + ) + elif fd == trnsp_in.fileno(): + cmd = trnsp_in.recv_cmd() + if cmd['stage'] == 'shutdown': + ipr.close() + data = struct.pack('IHHQIQQ', 28, 2, 0, 0, 104, 0, 0) + trnsp_out.send( + {'stage': 'broadcast', 'data': data, 'error': None} + ) + return + elif cmd['stage'] == 'reconstruct': + error = None + try: + msg = cmd['argv'][0]() + msg.load(pickle.loads(cmd['argv'][1])) + ipr.sendto_gate(msg, cmd['argv'][2]) + except Exception as e: + error = e + error.tb = traceback.format_exc() + trnsp_out.send( + { + 'stage': 'reconstruct', + 'error': error, + 'return': None, + 'cookie': cmd['cookie'], + } + ) + + elif cmd['stage'] == 'command': + error = None + try: + ret = getattr(ipr, cmd['name'])( + *cmd['argv'], **cmd['kwarg'] + ) + if ( + cmd['name'] == 'bind' + and ipr._brd_socket is not None + ): + inputs.append(ipr._brd_socket.fileno()) + broadcasts[ipr._brd_socket.fileno()] = ( + ipr._brd_socket + ) + except Exception as e: + ret = None + error = e + error.tb = traceback.format_exc() + trnsp_out.send( + { + 'stage': 'command', + 'error': error, + 'return': ret, + 'cookie': cmd['cookie'], + } + ) + + +class RemoteSocket(NetlinkSocketBase): + trnsp_in = None + trnsp_out = None + remote_trnsp_in = None + remote_trnsp_out = None + + def __init__(self, trnsp_in, trnsp_out, groups=0): + super(RemoteSocket, self).__init__(groups=groups) + self.trnsp_in = trnsp_in + self.trnsp_out = trnsp_out + self.cmdlock = threading.Lock() + self.shutdown_lock = threading.RLock() + self.closed = False + init = self.trnsp_in.recv_cmd() + if init['stage'] != 'init': + raise TypeError('incorrect protocol init') + if init['error'] is not None: + raise init['error'] + else: + self.uname = init['uname'] + atexit.register(self.close) + + def sendto_gate(self, msg, addr): + with self.cmdlock: + self.trnsp_out.send( + { + 'stage': 'reconstruct', + 'cookie': None, + 'name': None, + 'argv': [type(msg), pickle.dumps(msg.dump()), addr], + 'kwarg': None, + } + ) + ret = self.trnsp_in.recv_cmd() + if ret['error'] is not None: + raise ret['error'] + return ret['return'] + + def recv(self, bufsize, flags=0): + msg = None + while True: + msg = self.trnsp_in.recv() + if msg is None: + raise EOFError() + if msg['stage'] == 'signal': + os.kill(os.getpid(), msg['data']) + else: + break + if msg['error'] is not None: + raise msg['error'] + return msg['data'] + + def _cleanup_atexit(self): + if hasattr(atexit, 'unregister'): + atexit.unregister(self.close) + else: + try: + atexit._exithandlers.remove((self.close, (), {})) + except ValueError: + pass + + def close(self, code=errno.ECONNRESET): + with self.shutdown_lock: + if not self.closed: + super(RemoteSocket, self).close() + self.closed = True + self._cleanup_atexit() + self.trnsp_out.send({'stage': 'shutdown'}) + # send loopback nlmsg to terminate possible .get() + if code > 0 and self.remote_trnsp_out is not None: + data = struct.pack('IHHQIQQ', 28, 2, 0, 0, code, 0, 0) + self.remote_trnsp_out.send( + {'stage': 'broadcast', 'data': data, 'error': None} + ) + with self.trnsp_in.lock: + pass + + transport_objs = ( + self.trnsp_out, + self.trnsp_in, + self.remote_trnsp_in, + self.remote_trnsp_out, + ) + + # Stop the transport objects. + for trnsp in transport_objs: + try: + if hasattr(trnsp, 'close'): + trnsp.close() + except Exception: + pass + + # Close the file descriptors. + for trnsp in transport_objs: + try: + trnsp.file_obj.close() + except Exception: + pass + try: + os.kill(self.child, config.signal_stop_remote) + os.waitpid(self.child, 0) + except OSError: + pass + + def proxy(self, cmd, *argv, **kwarg): + with self.cmdlock: + self.trnsp_out.send( + { + 'stage': 'command', + 'cookie': None, + 'name': cmd, + 'argv': argv, + 'kwarg': kwarg, + } + ) + ret = self.trnsp_in.recv_cmd() + if ret['error'] is not None: + raise ret['error'] + return ret['return'] + + def fileno(self): + return self.trnsp_in.fileno() + + def bind(self, *argv, **kwarg): + if 'async' in kwarg: + # FIXME + # raise deprecation error after 0.5.3 + # + log.warning( + 'use "async_cache" instead of "async", ' + '"async" is a keyword from Python 3.7' + ) + del kwarg['async'] + # do not work with async servers + kwarg['async_cache'] = False + return self.proxy('bind', *argv, **kwarg) + + def send(self, *argv, **kwarg): + return self.proxy('send', *argv, **kwarg) + + def sendto(self, *argv, **kwarg): + return self.proxy('sendto', *argv, **kwarg) + + def getsockopt(self, *argv, **kwarg): + return self.proxy('getsockopt', *argv, **kwarg) + + def setsockopt(self, *argv, **kwarg): + return self.proxy('setsockopt', *argv, **kwarg) + + def _sendto(self, *argv, **kwarg): + return self.sendto(*argv, **kwarg) + + def _recv(self, *argv, **kwarg): + return self.recv(*argv, **kwarg) diff --git a/pyroute2/requests/address.py b/pyroute2/requests/address.py index e70e962f8..0c7603e31 100644 --- a/pyroute2/requests/address.py +++ b/pyroute2/requests/address.py @@ -2,7 +2,6 @@ from socket import AF_INET, AF_INET6 from pyroute2.common import dqn2int, get_address_family, getbroadcast -from pyroute2.netlink.rt_files import RtAddrProtosFile from .common import Index, IPRouteFilter, NLAKeyTransform @@ -10,11 +9,6 @@ class AddressFieldFilter(Index, NLAKeyTransform): _nla_prefix = 'IFA_' - def get_flags(self, context, mode): - if mode == 'field': - return 0 - return context['flags'] - def set_prefixlen(self, context, value): if isinstance(value, str): if '.' in value: @@ -66,15 +60,10 @@ def set_cacheinfo(self, context, value): cacheinfo = value.copy() if self.command != 'dump': for i in ('preferred', 'valid'): - cacheinfo[f'ifa_{i}'] = context.get(i, pow(2, 32) - 1) + cacheinfo[f'ifa_{i}'] = cacheinfo.get(i, pow(2, 32) - 1) return {'cacheinfo': cacheinfo} return {} - def set_proto(self, context, value): - if isinstance(value, str): - value = RtAddrProtosFile().get_rt_id(value) - return {'proto': value} - def set_broadcast(self, context, value): ret = {} if self.command != 'dump' and isinstance(value, bool): @@ -95,14 +84,6 @@ def set_broadcast(self, context, value): def finalize(self, context): if self.command != 'dump': - if 'cacheinfo' in context: - context.pop('preferred', None) - context.pop('valid', None) - if ( - context['cacheinfo']['ifa_valid'] - < context['cacheinfo']['ifa_preferred'] - ): - raise ValueError('preferred_lft is greater than valid_lft') if 'family' not in context and 'address' in context: context['family'] = get_address_family(context['address']) if 'prefixlen' not in context: diff --git a/pyroute2/requests/bridge.py b/pyroute2/requests/bridge.py index 6f33322cd..c5bb63048 100644 --- a/pyroute2/requests/bridge.py +++ b/pyroute2/requests/bridge.py @@ -1,12 +1,7 @@ -from functools import reduce - -from pyroute2.config import AF_BRIDGE from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg, protinfo_bridge from .common import Index, IPRouteFilter, NLAKeyTransform -PREFIXLEN = len(protinfo_bridge.prefix) - class BridgeFieldFilter(Index, NLAKeyTransform): _nla_prefix = ifinfmsg.prefix @@ -127,33 +122,13 @@ def finalize(self, context): class BridgePortFieldFilter(IPRouteFilter): _nla_prefix = ifinfmsg.prefix - _nla_names = [x[0] for x in protinfo_bridge.nla_map] - # allow lowercase names w/o prefixes - _nla_names = reduce( - lambda x, y: x + y, [[x, x[PREFIXLEN:].lower()] for x in _nla_names] - ) - # allow tuple ids for the dump_filter lookup - _nla_names = reduce( - lambda x, y: x + y, [[x, ('protinfo', x)] for x in _nla_names] - ) - allowed = list(_nla_names) - allowed.append('attrs') - allowed.append('family') - allowed.append('index') + _allowed = [x[0] for x in protinfo_bridge.nla_map] + _allowed.append('attrs') def finalize(self, context): - context['family'] = AF_BRIDGE keys = tuple(context.keys()) - if self.command in ('show', 'dump'): - for key in keys: - if isinstance(key, str) and key in self._nla_names: - value = context[key] - if isinstance(value, bool): - value = int(value) - context[('protinfo', key)] = value - del context[key] - return - attrs = [] + context['attrs'] = [] for key in keys: - attrs.append((protinfo_bridge.name2nla(key), context[key])) - context['attrs'] = [('IFLA_PROTINFO', {'attrs': attrs}, 0x8000)] + context['attrs'].append( + (protinfo_bridge.name2nla(key), context[key]) + ) diff --git a/pyroute2/requests/common.py b/pyroute2/requests/common.py index c357ab81d..edcd08924 100644 --- a/pyroute2/requests/common.py +++ b/pyroute2/requests/common.py @@ -4,7 +4,6 @@ from socket import AF_INET, AF_INET6 from pyroute2.common import AF_MPLS, dqn2int, get_address_family -from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg class MPLSTarget(OrderedDict): @@ -108,11 +107,11 @@ def parse_target(self, key, context, value): def set_dst(self, context, value): if value in ('', 'default'): - return {} - elif value in ('0', '0.0.0.0', '0.0.0.0/0'): - return {'family': AF_INET} + return {'dst': ''} + elif value in ('0', '0.0.0.0'): + return {'dst': '', 'family': AF_INET} elif value in ('::', '::/0'): - return {'family': AF_INET6} + return {'dst': '', 'family': AF_INET6} return self.parse_target('dst', context, value) def set_src(self, context, value): @@ -132,8 +131,6 @@ class Index: def set_index(self, context, value): if isinstance(value, (list, tuple)): value = value[0] - if isinstance(value, ifinfmsg): - value = value['index'] return {'index': value} @@ -150,7 +147,7 @@ def policy(self, key): class NLAKeyTransform: _nla_prefix = '' - def key_transform(self, key): + def _key_transform(self, key): if isinstance(key, str) and key.startswith(self._nla_prefix): key = key[len(self._nla_prefix) :].lower() return key diff --git a/pyroute2/requests/link.py b/pyroute2/requests/link.py index a5991f7ee..1a3b8c5b2 100644 --- a/pyroute2/requests/link.py +++ b/pyroute2/requests/link.py @@ -1,5 +1,4 @@ -from pyroute2.netlink.rt_files import RtGroupFile -from pyroute2.netlink.rtnl.ifinfmsg import IFF_NOARP, IFF_UP, ifinfmsg +from pyroute2.netlink.rtnl.ifinfmsg import IFF_NOARP, ifinfmsg from pyroute2.netlink.rtnl.ifinfmsg.plugins.vlan import flags as vlan_flags from .common import Index, IPRouteFilter, NLAKeyTransform @@ -76,29 +75,24 @@ def set_state(self, context, value): if self.command == 'dump': return {'state': value} if value == 'up': - ret['flags'] = (context.get('flags', 0) or 0) | IFF_UP - ret['change'] = (context.get('change', 0) or 0) | IFF_UP + ret['flags'] = context.get('flags', 0) or 0 | 1 + ret['change'] = context.get('change', 0) or 0 | 1 return ret def set_arp(self, context, value): ret = {} if not value: - ret['flags'] = (context.get('flags', 0) or 0) | IFF_NOARP - ret['change'] = (context.get('change', 0) or 0) | IFF_NOARP + ret['flags'] = context.get('flags', 0) or 0 | IFF_NOARP + ret['change'] = context.get('change', 0) or 0 | IFF_NOARP return ret def set_noarp(self, context, value): ret = {} if value: - ret['flags'] = (context.get('flags', 0) or 0) | IFF_NOARP - ret['change'] = (context.get('change', 0) or 0) | IFF_NOARP + ret['flags'] = context.get('flags', 0) or 0 | IFF_NOARP + ret['change'] = context.get('change', 0) or 0 | IFF_NOARP return ret - def set_group(self, context, value): - if isinstance(value, str): - value = RtGroupFile().get_rt_id(value) - return {'group': value} - def finalize(self, context): # set interface type specific attributes self.kind = context.pop('kind', None) @@ -114,7 +108,6 @@ def finalize(self, context): self.specific[nla[len(prefix) :].lower()] = nla if self.command == 'dump': - context.pop('peer', None) context[('linkinfo', 'kind')] = self.kind for key, value in tuple(context.items()): if key in self.specific: diff --git a/pyroute2/requests/main.py b/pyroute2/requests/main.py index 334fffff1..f1ea1512b 100644 --- a/pyroute2/requests/main.py +++ b/pyroute2/requests/main.py @@ -2,27 +2,13 @@ General request and RTNL object data filters. ''' -import abc import weakref from collections import ChainMap -class RequestFilter(abc.ABC): - pass - - class RequestProcessor(dict): - field_filters = tuple() - mark = tuple() - context = None - combined = None - parameters = None - - def __init__(self, context=None, prime=None, parameters=None): - self.reset_filters() - self.reset_mark() - self.parameters = dict(parameters) if parameters else {} - prime = {} if prime is None else prime + def __init__(self, field_filter=None, context=None, prime=None): + self.field_filter = field_filter self.context = ( context if isinstance(context, (dict, weakref.ProxyType)) else {} ) @@ -33,93 +19,45 @@ def __init__(self, context=None, prime=None, parameters=None): def __setitem__(self, key, value): if value is None: return - new_data = self.filter(key, value) - op = 'set' - # - if isinstance(new_data, tuple): - op, new_data = new_data - elif not isinstance(new_data, dict): - raise ValueError('invalid new_data type') - # - if op == 'set': - if key in self: - super().__delitem__(key) - for nkey, nvalue in new_data.items(): - if nkey in self: - super().__delitem__(nkey) - super().__setitem__(nkey, nvalue) - return - # - if op == 'patch': - for nkey, nvalue in new_data.items(): - if nkey not in self: - self[nkey] = [] - self[nkey].extend(nvalue) - return - # - raise RuntimeError('error applying new values') - - def reset_filters(self): - self.field_filters = [] - - def set_parameter(self, name, value): - self.parameters[name] = value - - def reset_mark(self): - self.mark = [] - - def items(self): - for key, value in super().items(): - if key not in self.mark: - yield key, value - - def get_value(self, key, default=None, mode=None): - for field_filter in self.field_filters: - getter = getattr(field_filter, f'get_{key}', None) - if getter is not None: - return getter(self, mode) - self.mark.append(key) - return self.get(key, default) + if key in self: + del self[key] + for nkey, nvalue in self.filter(key, value).items(): + super(RequestProcessor, self).__setitem__(nkey, nvalue) def filter(self, key, value): - job = {key: value} - ret = None - - for field_filter in self.field_filters: - for k, v in tuple(job.items()): - if hasattr(field_filter, 'key_transform'): - k = field_filter.key_transform(k) - if ( - hasattr(field_filter, 'allowed') - and k not in field_filter.allowed - ): - return {} - if hasattr(field_filter, 'policy') and not field_filter.policy( - k - ): - return {} - setter = getattr(field_filter, f'set_{k}', None) - if setter is not None: - if ret is None: - ret = {} - ret.update(setter(ChainMap(self.combined, ret), v)) - - if ret is not None: - job = ret - - return ret if ret is not None else {key: value} + if hasattr(self.field_filter, '_key_transform'): + key = self.field_filter._key_transform(key) + if ( + hasattr(self.field_filter, '_allowed') + and key not in self.field_filter._allowed + ): + return {} + if hasattr( + self.field_filter, 'policy' + ) and not self.field_filter.policy(key): + return {} + return getattr( + self.field_filter, f'set_{key}', lambda *argv: {key: value} + )(self.combined, value) def update(self, prime): for key, value in tuple(prime.items()): self[key] = value - def add_filter(self, field_filter): - self.field_filters.append(field_filter) + def set_filter(self, field_filter): + self.field_filter = field_filter return self - def finalize(self): + def apply_filter(self, field_filter): + self.field_filter = field_filter self.update(self) - for field_filter in self.field_filters: - if hasattr(field_filter, 'finalize'): - field_filter.finalize(self.combined) + return self + + def finalize(self, cmd_context=None): + if hasattr(self.field_filter, 'finalize_for_iproute'): + # old interface + self.field_filter.finalize_for_iproute(self.combined, cmd_context) + if hasattr(self.field_filter, 'finalize'): + # new interface + self.field_filter.finalize(self.combined) return self diff --git a/pyroute2/requests/neighbour.py b/pyroute2/requests/neighbour.py index 9f0817e81..e0326a627 100644 --- a/pyroute2/requests/neighbour.py +++ b/pyroute2/requests/neighbour.py @@ -1,3 +1,5 @@ +from socket import AF_INET + from pyroute2.common import get_address_family from pyroute2.netlink.rtnl import ndmsg @@ -34,10 +36,15 @@ def set_dst(self, context, value): class NeighbourIPRouteFilter(IPRouteFilter): + def set_dst(self, context, value): + ret = {'dst': value} + if 'family' not in context: + ret['family'] = get_address_family(value) + return ret def finalize(self, context): if self.command not in ('dump', 'get'): if 'state' not in context: context['state'] = ndmsg.NUD_PERMANENT - if 'dst' in context and 'family' not in context: - context['family'] = get_address_family(context['dst']) + if 'family' not in context: + context['family'] = AF_INET diff --git a/pyroute2/requests/probe.py b/pyroute2/requests/probe.py deleted file mode 100644 index 60a009f96..000000000 --- a/pyroute2/requests/probe.py +++ /dev/null @@ -1,23 +0,0 @@ -from socket import AF_INET - -from .common import NLAKeyTransform - - -class ProbeFieldFilter(NLAKeyTransform): - _nla_prefix = 'PROBE_' - - def finalize(self, context): - if 'family' not in context: - context['family'] = AF_INET - if 'proto' not in context: - context['proto'] = 1 - if 'port' not in context: - context['port'] = 0 - if 'dst_len' not in context: - context['dst_len'] = 32 - if 'kind' not in context: - context['kind'] = 'ping' - if 'num' not in context: - context['num'] = 1 - if 'timeout' not in context: - context['timeout'] = 1 diff --git a/pyroute2/requests/route.py b/pyroute2/requests/route.py index f2f6b2dac..72704ac36 100644 --- a/pyroute2/requests/route.py +++ b/pyroute2/requests/route.py @@ -1,13 +1,6 @@ -from socket import AF_INET, AF_INET6 +from socket import AF_INET6 from pyroute2.common import AF_MPLS -from pyroute2.netlink.rt_files import ( - RtDsfieldFile, - RtProtosFile, - RtRealmsFile, - RtScopesFile, - RtTablesFile, -) from pyroute2.netlink.rtnl import encap_type, rt_proto, rt_scope, rt_type from pyroute2.netlink.rtnl.rtmsg import IP6_RT_PRIO_USER, LWTUNNEL_ENCAP_MPLS from pyroute2.netlink.rtnl.rtmsg import nh as nh_header @@ -85,20 +78,12 @@ def set_encap(self, context, value): def set_scope(self, context, value): if isinstance(value, str): - try: - value = rt_scope[value] - except KeyError: - # lookup with rt_scopes* files - value = RtScopesFile().get_rt_id(value) + return {'scope': rt_scope[value]} return {'scope': value} def set_proto(self, context, value): if isinstance(value, str): - try: - value = rt_proto[value] - except KeyError: - # lookup with rt_proto* files - value = RtProtosFile().get_rt_id(value) + return {'proto': rt_proto[value]} return {'proto': value} def set_encap_type(self, context, value): @@ -111,34 +96,8 @@ def set_type(self, context, value): return {'type': rt_type[value]} return {'type': value} - def set_table(self, context, value): - if isinstance(value, str): - value = RtTablesFile().get_rt_id(value, 0) - return {"table": value} - - def set_tos(self, context, value): - if isinstance(value, str): - value = RtDsfieldFile().get_rt_id(value, 0) - return {"tos": value} - class RouteIPRouteFilter(IPRouteFilter): - - def get_table(self, context, mode): - table = context.get('table', 0) - if mode == 'field': - if context.parameters['strict_check']: - return 0 - return table if 0 < table < 255 else 254 - return table - - def set_flow(self, context, value): - if isinstance(value, str): - value = RtRealmsFile().get_rt_id(value) - return {"flow": value} - - set_realms = set_flow - def set_metrics(self, context, value): if value and 'attrs' not in value: metrics = {'attrs': []} @@ -151,12 +110,11 @@ def set_metrics(self, context, value): return {} def set_multipath(self, context, value): - ret = {} if value: - hops = [] + ret = [] for v in value: if 'attrs' in v: - hops.append(v) + ret.append(v) continue nh = {'attrs': []} nh_fields = [x[0] for x in nh_header.fields] @@ -189,20 +147,10 @@ def set_multipath(self, context, value): else: rta = rtmsg.name2nla(name) nh['attrs'].append([rta, v[name]]) - hops.append(nh) - if hops: - ret = {'multipath': hops} - if context.get('family') is None: - # autodetect and propagate family - hop = ret[0] - attrs = hop.get('attrs', []) - for attr in attrs: - if attr[0] == 'RTA_GATEWAY': - ret['family'] = ( - AF_INET6 if attr[1].find(':') >= 0 else AF_INET - ) - break - return ret + ret.append(nh) + if ret: + return {'multipath': ret} + return {} def set_encap(self, context, value): if ( @@ -298,21 +246,6 @@ def set_encap(self, context, value): return {} def finalize(self, context): - # cleanup extra NLA for AF_MPLS - if context.get('family') == AF_MPLS: - for key in tuple(context.keys()): - if key not in ( - 'family', - 'proto', - 'type', - 'dst', - 'newdst', - 'via', - 'multipath', - 'oif', - ): - context.pop(key, None) - # cleanup empty strings for key in context: if context[key] in ('', None): try: diff --git a/pyroute2/requests/rule.py b/pyroute2/requests/rule.py index d18559499..3a667c0fd 100644 --- a/pyroute2/requests/rule.py +++ b/pyroute2/requests/rule.py @@ -12,13 +12,6 @@ class RuleFieldFilter(IPTargets, Index, NLAKeyTransform): class RuleIPRouteFilter(IPRouteFilter): - - def get_table(self, context, mode): - table = context.get('table', 0) - if mode == 'field': - return table if 0 <= table <= 255 else 252 - return table - def set_action(self, context, value): if isinstance(value, str): return { @@ -37,10 +30,7 @@ def finalize(self, context): if 'table' in context and 'action' not in context: context['action'] = 'to_tbl' for key in ('src_len', 'dst_len'): - if ( - context.get(key, None) is None - and context.get(key[:3], None) is not None - ): + if context.get(key, None) is None and key[:3] in context: context[key] = {socket.AF_INET6: 128, socket.AF_INET: 32}[ context['family'] ] diff --git a/pyroute2/requests/tc.py b/pyroute2/requests/tc.py deleted file mode 100644 index 0769abd04..000000000 --- a/pyroute2/requests/tc.py +++ /dev/null @@ -1,76 +0,0 @@ -from pyroute2.netlink.rt_files import TcClsFile -from pyroute2.netlink.rtnl import TC_H_ROOT -from pyroute2.netlink.rtnl.tcmsg import plugins as tc_plugins - -from .common import IPRouteFilter - - -class TcRequestFilter: - def transform_handle(self, key, context, handle): - if isinstance(handle, str): - if ':' not in handle: - handle = TcClsFile().get_rt_id(handle) - else: - (major, minor) = [ - int(x if x else '0', 16) for x in handle.split(':') - ] - handle = (major << 8 * 2) | minor - return {key: handle} - - def set_handle(self, context, value): - return self.transform_handle('handle', context, value) - - def set_root(self, context, value): - ret = {} - if value: - ret['parent'] = 0xFFFFFFFF - if 'handle' not in context: - ret['handle'] = 0x10000 - return ret - - def set_target(self, context, value): - return self.transform_handle('target', context, value) - - def set_parent(self, context, value): - return self.transform_handle('parent', context, value) - - def set_default(self, context, value): - return self.transform_handle('default', context, value) - - -class TcIPRouteFilter(IPRouteFilter): - - def finalize(self, context): - - if self.command.startswith('dump'): - context.pop('kind', None) - return - - if 'index' not in context: - context['index'] = 0 - if 'handle' not in context: - context['handle'] = 0 - - # get & run the plugin - if 'kind' in context: - if context['kind'] in tc_plugins: - plugin = tc_plugins[context['kind']] - context['parent'] = context.get( - 'parent', getattr(plugin, 'parent', 0) - ) - if set(context.keys()) > set(('kind', 'index', 'handle')): - get_parameters = None - if self.command[-5:] == 'class': - get_parameters = getattr( - plugin, 'get_class_parameters', None - ) - else: - get_parameters = getattr( - plugin, 'get_parameters', None - ) - if get_parameters is not None: - context['options'] = get_parameters(dict(context)) - if hasattr(plugin, 'fix_request'): - plugin.fix_request(context) - else: - context['parent'] = TC_H_ROOT diff --git a/pyroute2/statsd.py b/pyroute2/statsd.py deleted file mode 100644 index 9ed29a25b..000000000 --- a/pyroute2/statsd.py +++ /dev/null @@ -1,63 +0,0 @@ -import ctypes -import os -import socket -from typing import Optional, Union - -from pyroute2 import netns -from pyroute2.config import metric_type -from pyroute2.netlink.coredata import CoreConfig, CoreSocketSpec - - -class StatsDClientSocket(socket.socket): - '''StatsD client.''' - - def __init__( - self, - address: Optional[tuple[str, int]] = None, - use_socket: Optional[socket.socket] = None, - flags: int = os.O_CREAT, - libc: Optional[ctypes.CDLL] = None, - ): - self.spec = CoreSocketSpec( - CoreConfig( - netns=None, address=address, use_socket=use_socket is not None - ) - ) - self.status = self.spec.status - self.buffer: str = '' - if use_socket is not None: - fd = use_socket.fileno() - else: - prime = netns.create_socket( - self.spec['netns'], socket.AF_INET, socket.SOCK_DGRAM - ) - fd = os.dup(prime.fileno()) - prime.close() - super().__init__(fileno=fd) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def commit(self): - self.sendto(self.buffer.encode(), self.spec['address']) - self.buffer = '' - - def put( - self, name: str, value: Union[int, str], kind: metric_type - ) -> None: - self.buffer += f'{name}:{value}|{kind}\n' - - def incr(self, name: str, value: int = 1) -> None: - self.put(name, value, 'c') - self.commit() - - def gauge(self, name: str, value: int) -> None: - self.put(name, value, 'g') - self.commit() - - def timing(self, name: str, value: int) -> None: - self.put(name, value, 'ms') - self.commit() diff --git a/pyroute2/wiset.py b/pyroute2/wiset.py index 5e70385c4..73706c91b 100644 --- a/pyroute2/wiset.py +++ b/pyroute2/wiset.py @@ -343,7 +343,7 @@ def update_dict_content(self, ndmsg): comment=entry.get_attr("IPSET_ATTR_COMMENT"), skbmark=skbmark, timeout=timeout, - **entry_flag_parsed, + **entry_flag_parsed ) self._content[key] = value @@ -364,7 +364,7 @@ def create(self, **kwargs): counters=self.counters, hashsize=self.hashsize, skbinfo=self.skbinfo, - **kwargs, + **kwargs ) def destroy(self): diff --git a/requirements.dev.txt b/requirements.dev.txt new file mode 100644 index 000000000..d4bb27179 --- /dev/null +++ b/requirements.dev.txt @@ -0,0 +1,8 @@ +build +twine +flake8 +netaddr +pytest +pytest-cov +pre-commit +findimports diff --git a/requirements.docs.txt b/requirements.docs.txt new file mode 100644 index 000000000..904a564e8 --- /dev/null +++ b/requirements.docs.txt @@ -0,0 +1,7 @@ +build +twine +aafigure +sphinx==5.1.1 +sphinx-code-include +pre-commit +docutils diff --git a/requirements.repo.txt b/requirements.repo.txt new file mode 100644 index 000000000..8fa399720 --- /dev/null +++ b/requirements.repo.txt @@ -0,0 +1,4 @@ +nox +pytest +pytest-cov +setuptools diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..c046bb4df --- /dev/null +++ b/setup.cfg @@ -0,0 +1,43 @@ +[metadata] +name = pyroute2 +version = file: VERSION +description = Python Netlink library +long_description = file: README.rst +author = Peter Saveliev +author_email = peter@svinota.eu +long_description_content_type = text/x-rst +url = https://github.com/svinota/pyroute2 +license = GPL-2.0-or-later OR Apache-2.0 +classifiers = + License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Topic :: Software Development :: Libraries :: Python Modules + Topic :: System :: Networking + Topic :: System :: Systems Administration + Operating System :: POSIX :: Linux + Intended Audience :: Developers + Intended Audience :: System Administrators + Intended Audience :: Telecommunications Industry + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Development Status :: 4 - Beta + +[options] +install_requires = + win_inet_pton ; platform_system == "Windows" + importlib-metadata ; python_version < "3.8" +packages_dir = + =pyroute2 +packages = find: + +[options.entry_points] +console_scripts = + ss2 = pyroute2.netlink.diag.ss2:run [psutil] + pyroute2-cli = pyroute2.ndb.cli:run + pyroute2-dhcp-client = pyroute2.dhcp.client:run + pyroute2-test-platform = pyroute2.config.test_platform:run diff --git a/setup.minimal.cfg b/setup.minimal.cfg new file mode 100644 index 000000000..4544544ac --- /dev/null +++ b/setup.minimal.cfg @@ -0,0 +1,63 @@ +[metadata] +name = pyroute2.minimal +version = file: VERSION +description = Python Netlink library: minimal distribution +long_description = file: README.minimal.rst +author = Peter Saveliev +author_email = peter@svinota.eu +long_description_content_type = text/x-rst +url = https://github.com/svinota/pyroute2 +license = GPL-2.0-or-later OR Apache-2.0 +classifiers = + License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Topic :: Software Development :: Libraries :: Python Modules + Topic :: System :: Networking + Topic :: System :: Systems Administration + Operating System :: POSIX :: Linux + Intended Audience :: Developers + Intended Audience :: System Administrators + Intended Audience :: Telecommunications Industry + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Development Status :: 4 - Beta + +[options] +install_requires = + win_inet_pton ; platform_system == "Windows" + importlib-metadata ; python_version < "3.8" +packages_dir = + =pyroute2 +packages = + pr2modules + pyroute2 + pyroute2.bsd + pyroute2.bsd.pf_route + pyroute2.bsd.rtmsocket + pyroute2.config + pyroute2.inotify + pyroute2.iproute + pyroute2.netlink + pyroute2.netlink.devlink + pyroute2.netlink.diag + pyroute2.netlink.event + pyroute2.netlink.generic + pyroute2.netlink.ipq + pyroute2.netlink.nfnetlink + pyroute2.netlink.nl80211 + pyroute2.netlink.rtnl + pyroute2.netlink.rtnl.ifinfmsg + pyroute2.netlink.rtnl.ifinfmsg.plugins + pyroute2.netlink.rtnl.tcmsg + pyroute2.netlink.taskstats + pyroute2.netlink.uevent + pyroute2.netns + pyroute2.protocols + pyroute2.requests diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..606849326 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/stubs/.gitkeep b/stubs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..d5b65ca12 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,44 @@ +Test modules +============ + +* `test_limits` -- resource limits, fd leaks, etc +* `test_linux` -- functional tests for Linux, may require root +* `test_minimal` -- test pyroute2.minimal package +* `test_neutron` -- integration with OpenStack Neutron +* `test_repo` -- repository checks +* `test_openbsd` -- functional tests for OpenBSD +* `test_unit` -- unittests + +Functional tests under `test_linux` directory require root +access to create, destroy and set up network objects -- +routes, addresses, interfaces, etc. They use mainly dummy +interfaces, but the main OS setup may be affected. + +Requirements +============ + +* nox +* python >= 3.6 +* `-r requirements.dev.txt` + +Run tests +========= + +All the tests should be started via corresponding nox session, +see `noxfile.py`. Alternatively there is a `make` target left +for those who prefer:: + + # using nox + $ nox --list + $ nox -e unit # run only unit tests + $ nox -e unit -- '{"pdb": true}' # provide a session config + $ nox # run all the tests + + # using make + $ sudo make test # run the default sessions + $ make nox session=unit # run only unit tests + $ make nox session=openbsd # OpenBSD tests + +Get code coverage and run PDB on failures:: + + $ nox -e linux -- '{"pdb": true, "coverage": true}' diff --git a/tests/decoder/README.md b/tests/decoder/README.md new file mode 100644 index 000000000..1225a5ae2 --- /dev/null +++ b/tests/decoder/README.md @@ -0,0 +1,64 @@ +Description +----------- + +This is the data directory for the parser testing. + +The test module: `general/test_parser.py` + +Data format +----------- + +The data files can be in two formats. The strace hex dump: +``` + \x00\x00\x00... +``` + +And the pyroute2 hex dump: +``` + 00:00:00... +``` + +When the data file gets loaded, all the spaces, comments and +new lines are ignored. There can be several packets in the +same file, the parser deals with it. Comments should start +with `#` or `;`: +``` + # field one + 00:00:00 + # field two + 00:00:00 + ... +``` + +All the data after `.` is also ignored. It can be used to +provide detailed descriptions of the file after the dump +data: +``` + \x00\x00\x00... + . + Here goes the data description +``` + +How to collect +-------------- + +To collect the data, one can use either of two approaches. +First, use strace: +``` + $ strace -e trace=network -f -x -s 4096 netlink_utility + ... + sendto(3, "\x28\x00\x00\x00\x12\x00\x01\x03\x67\x9a..."... ) +``` + +Then just copy and paste to the data file strings from `sendto()` +and `recvmsg()` calls. + +Or one can use packets parsed with pyroute2: +``` + >>> from pyroute2 import IPRoute + >>> from pyroute2.common import hexdump + >>> ipr = IPRoute() + >>> pkts = ipr.get_addr() + >>> hexdump(pkts[0].raw) + '4c:00:00:00:14:00:02:00:ff:00:00:00:...' +``` diff --git a/tests/decoder/decoder.py b/tests/decoder/decoder.py new file mode 100644 index 000000000..cb2685a62 --- /dev/null +++ b/tests/decoder/decoder.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +''' +Usage:: + + ./decoder.py [module] [data_file] + +Sample:: + + ./decoder.py pyroute2.netlink.rtnl.tcmsg.tcmsg ./sample_packet_01.data + ./decoder.py pyroute2.netlink.nl80211.nl80211cmd ./nl80211.data + +Module is a name within rtnl hierarchy. File should be a +binary data in the escaped string format (see samples). +''' +import sys +from importlib import import_module +from pprint import pprint + +from pyroute2.common import hexdump, load_dump + +mod = sys.argv[1] +mod = mod.replace('/', '.') +f = open(sys.argv[2], 'r') +s = mod.split('.') +package = '.'.join(s[:-1]) +module = s[-1] +m = import_module(package) +met = getattr(m, module) + + +data = load_dump(f) + +offset = 0 +inbox = [] +while offset < len(data): + msg = met(data[offset:]) + msg.decode() + print(hexdump(msg.data)) + pprint(msg) + print('.' * 40) + offset += msg['header']['length'] diff --git a/tests/net_tools.py b/tests/net_tools.py deleted file mode 100644 index 83c9d68ca..000000000 --- a/tests/net_tools.py +++ /dev/null @@ -1,194 +0,0 @@ -''' -This module should be used to assert network objects like interfaces, -addresses and routes. - -It MUST NOT use any code from pyroute2 project, directly or indirectly. -Thus, only stdlib imports are allowed here. - -It MAY use external utilities. Right now it uses iproute2 tools with -`-json` parameter. -''' - -import json -import subprocess -import time -from collections import namedtuple -from functools import reduce -from socket import AF_INET, AF_INET6 - -ip_object_filter = namedtuple('ip_object_filter', ('query', 'value')) - - -def fix_list(value): - if not isinstance(value, list): - value = [value] - return value - - -def wait_for_ip_object(cmd, filters, timeout, retry): - timeout_ns = timeout * 1_000_000_000 - ts = time.time_ns() - found = False - - while not found: - check = subprocess.run( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if check.returncode != 0: - break - - dump = json.loads(check.stdout) - found = False - for node in dump: - if all( - ( - flt.value - in reduce( - lambda x, y: ( - fix_list(x.get(y, {})) - if isinstance(x, dict) - else [z.get(y, {}) for z in x] - ), - [node] + flt.query.lstrip('.').split('.'), - ) - for flt in filters - ) - ): - found = True - break - - if found or (time.time_ns() > ts + timeout_ns): - break - - time.sleep(retry) - return found - - -def address_exists( - address, - ifname=None, - preferred=None, - valid=None, - netns=None, - timeout=1, - retry=0.2, -): - ns = [] if netns is None else ['ip', 'netns', 'exec', netns] - filters = [ip_object_filter(query='.addr_info.local', value=address)] - if preferred is not None: - filters.append( - ip_object_filter( - query='.addr_info.preferred_life_time', value=preferred - ) - ) - ifspec = ['dev', ifname] if ifname is not None else [] - return wait_for_ip_object( - ns + ['ip', '-json', 'addr', 'show'] + ifspec, filters, timeout, retry - ) - - -def interface_exists(ifname, netns=None, timeout=1, retry=0.2): - ns = [] if netns is None else ['ip', 'netns', 'exec', netns] - filters = [ip_object_filter(query='.ifname', value=ifname)] - return wait_for_ip_object( - ns + ['ip', '-json', 'link', 'show'], filters, timeout, retry - ) - - -def route_exists(dst, table='main', netns=None, timeout=1, retry=0.2): - ns = [] if netns is None else ['ip', 'netns', 'exec', netns] - filters = [ip_object_filter(query='.dst', value=dst)] - return wait_for_ip_object( - ns + ['ip', '-json', 'route', 'show', 'table', str(table)], - filters, - timeout, - retry, - ) - - -def rule_exists(priority, proto=AF_INET, netns=None, timeout=1, retry=0.2): - ns = [] if netns is None else ['ip', 'netns', 'exec', netns] - proto = {AF_INET: '-4', AF_INET6: '-6'}.get(proto, '-4') - filters = [ip_object_filter(query='.priority', value=priority)] - return wait_for_ip_object( - ns + ['ip', '-json', proto, 'rule', 'show'], filters, timeout, retry - ) - - -def class_exists( - ifname, - handle, - kind, - parent=None, - root=None, - netns=None, - timeout=1, - retry=0.2, -): - ns = [] if netns is None else ['ip', 'netns', 'exec', netns] - filters = [ - ip_object_filter(query='.class', value=kind), - ip_object_filter(query='.handle', value=handle), - ] - if root is not None: - filters.append(ip_object_filter(query='.root', value=root)) - if parent is not None: - filters.append(ip_object_filter(query='.parent', value=parent)) - return wait_for_ip_object( - ns + ['tc', '-json', 'class', 'show', 'dev', ifname], - filters, - timeout, - retry, - ) - - -def filter_exists( - ifname, - kind, - parent=None, - protocol=None, - match_value=None, - match_mask=None, - netns=None, - timeout=1, - retry=0.2, -): - ns = [] if netns is None else ['ip', 'netns', 'exec', netns] - filters = [ip_object_filter(query='.kind', value=kind)] - if parent is not None: - filters.append(ip_object_filter(query='.parent', value=parent)) - if protocol is not None: - filters.append(ip_object_filter(query='.protocol', value=protocol)) - if match_value is not None: - filters.append( - ip_object_filter(query='.options.match.value', value=match_value) - ) - if match_mask is not None: - filters.append( - ip_object_filter(query='.options.match.mask', value=match_mask) - ) - return wait_for_ip_object( - ns + ['tc', '-json', 'filter', 'show', 'dev', ifname], - filters, - timeout, - retry, - ) - - -def qdisc_exists( - ifname, handle, default=None, rate=None, netns=None, timeout=1, retry=0.2 -): - ns = [] if netns is None else ['ip', 'netns', 'exec', netns] - filters = [ip_object_filter(query='.handle', value=handle)] - if default is not None: - filters.append( - ip_object_filter(query='.options.default', value=default) - ) - if rate is not None: - filters.append(ip_object_filter(query='.options.rate', value=rate)) - return wait_for_ip_object( - ns + ['tc', '-json', 'qdisc', 'show', 'dev', ifname], - filters, - timeout, - retry, - ) diff --git a/tests/test_ci/conftest.py b/tests/test_ci/conftest.py deleted file mode 100644 index e3412ee8d..000000000 --- a/tests/test_ci/conftest.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - -from pyroute2 import config, netns - -config.mock_netlink = True -config.mock_netns = True -pytest_plugins = [ - 'pyroute2.fixtures.iproute', - 'pyroute2.fixtures.ndb', - 'pyroute2.fixtures.plan9', -] -cleanup_netns_set = set() - - -@pytest.fixture -def netns_create_list(): - return set([x[0][0] for x in netns.create().call_args_list if x[0]]) - - -@pytest.fixture -def netns_remove_list(): - return set([x[0][0] for x in netns.remove().call_args_list if x[0]]) - - -def check_fixture_spec_func(fixture, scope, name): - return fixture.name == name - - -@pytest.fixture -def check_fixture_spec(): - yield check_fixture_spec_func - - -@pytest.fixture -def cleanup_netns(): - global cleanup_netns_set - yield cleanup_netns_set diff --git a/tests/test_ci/test_fixtures.py b/tests/test_ci/test_fixtures.py deleted file mode 100644 index a21cc3904..000000000 --- a/tests/test_ci/test_fixtures.py +++ /dev/null @@ -1,60 +0,0 @@ -import pytest - -from pyroute2.fixtures import iproute, ndb - - -def test_nsname_setup(nsname, cleanup_netns, netns_create_list): - cleanup_netns.add(nsname) - assert nsname in netns_create_list - - -def test_nsname_cleanup(cleanup_netns, netns_remove_list): - assert cleanup_netns <= netns_remove_list - - -@pytest.mark.parametrize( - 'fixture,scope,name', - ( - (iproute.async_context, 'function', 'async_context'), - (iproute.async_ipr, 'function', 'async_ipr'), - (iproute.nsname, 'function', 'nsname'), - (iproute.setns_context, 'function', 'setns_context'), - (iproute.sync_context, 'function', 'sync_context'), - (iproute.sync_ipr, 'function', 'sync_ipr'), - (iproute.test_link, 'function', 'test_link'), - (iproute.test_link_address, 'function', 'test_link_address'), - (iproute.test_link_ifinfmsg, 'function', 'test_link_ifinfmsg'), - (iproute.test_link_ifname, 'function', 'test_link_ifname'), - (iproute.test_link_index, 'function', 'test_link_index'), - (ndb.ndb, 'function', 'ndb'), - ), - ids=( - 'async_context', - 'async_ipr', - 'ndb', - 'nsname', - 'setns_context', - 'sync_context', - 'sync_ipr', - 'test_link', - 'test_link_address', - 'test_link_ifinfmsg', - 'test_link_ifname', - 'test_link_index', - ), -) -def test_fixture_spec(check_fixture_spec, fixture, scope, name): - assert check_fixture_spec(fixture, scope=scope, name=name) - - -def test_chain(nsname, sync_ipr, async_ipr, test_link, ndb, netns_create_list): - assert nsname == sync_ipr.status['netns'] - assert nsname == async_ipr.status['netns'] - assert nsname == test_link.netns - assert ( - ndb.sources['localhost'] - .nl.status['netns'] - .decode('utf-8') - .endswith(nsname) - ) - assert nsname in netns_create_list diff --git a/tests/test_core/conftest.py b/tests/test_core/conftest.py deleted file mode 100644 index 9bf2d5cae..000000000 --- a/tests/test_core/conftest.py +++ /dev/null @@ -1,5 +0,0 @@ -pytest_plugins = [ - 'pyroute2.fixtures.iproute', - 'pyroute2.fixtures.ndb', - 'pyroute2.fixtures.plan9', -] diff --git a/tests/test_core/test_check_tid.py b/tests/test_core/test_check_tid.py deleted file mode 100644 index bb4ad0283..000000000 --- a/tests/test_core/test_check_tid.py +++ /dev/null @@ -1,33 +0,0 @@ -import logging -import threading -import warnings - -import pytest - -from pyroute2 import IPRoute - - -def catch_tid_error(ipr): - try: - ipr._check_tid(tag='0x8241', level=logging.ERROR) - except RuntimeError as e: - assert '#0x8241' in e.args[0] - warnings.warn('#0x8242') - - -@pytest.mark.parametrize( - 'func,tag', - ( - (lambda x: x.bind(), '#bind'), - (lambda x: x._check_tid(tag='0x8240', level=logging.WARN), '#0x8240'), - (catch_tid_error, '#0x8242'), - ), -) -def test_calls(func, tag): - with warnings.catch_warnings(record=True) as wrec: - with IPRoute() as ipr: - t = threading.Thread(target=func, args=[ipr]) - t.start() - t.join() - assert len(wrec) == 1 - assert tag in wrec[0].message.args[0] diff --git a/tests/test_core/test_event_loop.py b/tests/test_core/test_event_loop.py deleted file mode 100644 index 8e5024108..000000000 --- a/tests/test_core/test_event_loop.py +++ /dev/null @@ -1,196 +0,0 @@ -import asyncio -import errno -import gc -import threading -import traceback -import weakref - -import pytest - -from pyroute2 import AsyncIPRoute, IPRoute - - -def get_event_loops(): - for obj in gc.get_objects(): - try: - module = getattr(obj, '__module__', None) - cls = getattr(obj, '__class__', None) - except ReferenceError: - continue - if ( - isinstance(obj, weakref.ProxyType) - or not isinstance(module, str) - or not module.startswith('asyncio') - or not cls.__name__.endswith('EventLoop') - or not hasattr(obj, 'is_closed') - or obj.is_closed() - ): - continue - yield weakref.proxy(obj) - - -def diff_event_loops(state=0): - return len(list(get_event_loops())) - state - - -def threading_target_run(func_list, exc, ret): - for func in func_list: - try: - ret.append(func()) - except Exception as e: - e.tb = traceback.format_exc() - ret.append(e) - exc.append(e) - - -def test_event_loop_new(): - event_loops = list(get_event_loops()) - - with IPRoute() as ipr: - assert ipr.status['event_loop'] == 'new' - assert len(list(ipr.get_links())) > 0 - - gc.collect() - assert len(event_loops) >= len(list(get_event_loops())) - - -def test_event_loop_count(): - state = diff_event_loops() - ipr = IPRoute() - assert diff_event_loops(state) == 1 - assert len(list(ipr.get_links())) > 0 - assert diff_event_loops(state) == 1 - ipr.close() - assert diff_event_loops(state) == 0 - - -def test_threading_close_noop(): - state = diff_event_loops() - ipr = IPRoute() - exc = [] - ret = [] - tt = threading.Thread( - target=threading_target_run, - args=[ - [ - lambda: diff_event_loops(state), - lambda: ipr.close(), - lambda: diff_event_loops(state), - ], - exc, - ret, - ], - ) - tt.start() - tt.join() - ipr.close() - assert len(exc) == 0 - assert len(ret) == 3 - assert ret == [1, None, 1] - assert diff_event_loops(state) == 0 - - -def test_threading_close_op(): - state = diff_event_loops() - ipr = IPRoute() - exc = [] - ret = [] - tt = threading.Thread( - target=threading_target_run, - args=[ - [ - lambda: diff_event_loops(state), - lambda: len(list(ipr.get_links())), - lambda: diff_event_loops(state), - lambda: ipr.close(), - lambda: diff_event_loops(state), - ], - exc, - ret, - ], - ) - tt.start() - tt.join() - ipr.close() - assert len(exc) == 0 - assert len(ret) == 5 - assert ret[0] == 1 - assert ret[1] > 1 - assert ret[2] == 1 - assert ret[3] is None - assert ret[4] == 1 - assert diff_event_loops(state) == 0 - - -def test_threading_sync(): - event_loop = asyncio.new_event_loop() - ipr = IPRoute(use_event_loop=event_loop) - exc = [] - ret = [] - assert len(list(ipr.get_links())) > 1 - tt = threading.Thread( - target=threading_target_run, - args=[[lambda: list(ipr.get_links())], exc, ret], - ) - tt.start() - tt.join() - assert len(exc) == 1 - assert isinstance(exc[0], RuntimeError) - assert ( - exc[0].args[0] - == 'Predefined event loop can not be used in another thread' - ) - - -def test_threading_bind(): - state = diff_event_loops() - ipr = IPRoute() - assert diff_event_loops(state) == 1 - ready_event = threading.Event() - close_event = threading.Event() - exc = [] - ret = [] - tt = threading.Thread( - target=threading_target_run, - args=[ - [ - lambda: ipr.bind(), - lambda: diff_event_loops(state), - lambda: ready_event.set(), - lambda: close_event.wait(), - lambda: [x for x in ipr.get()], - lambda: diff_event_loops(state), - ], - exc, - ret, - ], - ) - tt.start() - ready_event.wait() - ipr.close() - close_event.set() - tt.join() - assert ret[1] == 2 - assert isinstance(ret[4], OSError) - assert ret[4].errno == errno.EBADF - assert ret[5] == 0 - - -@pytest.mark.asyncio -async def test_event_loop_auto(): - event_loops = list(get_event_loops()) - - async with AsyncIPRoute() as ipr: - assert not ipr.event_loop.is_closed() - assert ipr.status['event_loop'] == 'auto' - assert len([x async for x in await ipr.get_links()]) > 0 - - gc.collect() - assert len(event_loops) >= len(list(get_event_loops())) - - -@pytest.mark.asyncio -async def test_sync_fail_in_async_context(): - # sync IPRoute must fail in the async context - with pytest.raises(RuntimeError): - IPRoute() diff --git a/tests/test_core/test_ndb/test_repl.py b/tests/test_core/test_ndb/test_repl.py deleted file mode 100644 index d65b9e25c..000000000 --- a/tests/test_core/test_ndb/test_repl.py +++ /dev/null @@ -1,193 +0,0 @@ -import ast -import code -import contextlib -import io -import sys - -import pytest - - -def check_output(source, combined, *func): - parsed = ast.literal_eval(source) - if len(parsed) != len(func): - return False - return combined(*[f(v) for v, f in zip(parsed, func)]) - - -@pytest.mark.parametrize( - ('script', 'check', 'combined_fields', 'combined_records'), - ( - ( - ( - 'from pyroute2 import NDB', - 'ndb = NDB()', - '# output repr() of a summary', - 'ndb.interfaces.summary()', - 'ndb.close()', - ), - ( - lambda v: v == 'localhost', - lambda v: v == 0, - lambda v: isinstance(v, int) and v > 0, - lambda v: isinstance(v, str) and 0 < len(v) < 16, - lambda v: isinstance(v, str) or v is None, - lambda v: isinstance(v, int) and v > 0, - lambda v: isinstance(v, str) or v is None, - ), - lambda *r: all(r), - lambda *s: all(s), - ), - ( - ( - 'from pyroute2 import NDB', - 'ndb = NDB()', - '# no repr() expected', - 's = ndb.interfaces.summary()', - '# select_fields() returns report -> repr()', - 's.select_fields("target", "index")' - '# one more repr(), must NOT be empty', - 's', - '# one more repr(), must NOT be empty', - 's', - 'ndb.close()', - ), - ( - lambda v: v == 'localhost', - lambda v: isinstance(v, int) and v == 1, - ), - lambda *r: all(r), - lambda *s: sum(s) == 3, - ), - ( - ( - 'from pyroute2 import NDB', - 'ndb = NDB()', - 's = ndb.interfaces.summary()', - '_ = s.select_fields("index", "ifname")', - '_ = s.select_records(**{"ifname": lambda v: v == "lo"})', - 's', - 's', - 'ndb.close()', - ), - (lambda v: v == 1, lambda v: v == 'lo'), - lambda *r: all(r), - lambda *s: sum(s) == 2, - ), - ( - ( - 'from pyroute2 import NDB', - 'ndb = NDB()', - 's = ndb.addresses.summary()', - '(s', - ' .transform_fields(', - ' address=lambda r: f"{r.address}/{r.prefixlen}"', - ' )' ' .select_fields(', - ' "ifname", "address",' ' )', - ')', - 'ndb.close()', - ), - (lambda v: v == 'lo', lambda v: v == '127.0.0.1/8'), - lambda *r: all(r), - lambda *s: sum(s) == 1, - ), - ( - ( - 'from pyroute2 import NDB, netns', - 'from pyroute2.common import uifname', - 'nsname = uifname()', - 'ndb = NDB(', - ' sources=[', - ' {', - ' "target": "localhost",', - ' "netns": nsname,', - ' }', - ' ],', - ')', - 'ndb.interfaces["lo"].set("state", "up").commit()', - '(ndb ', - ' .addresses', - ' .summary()', - ' .select_records(address="127.0.0.1")', - ' .count(),', - ')', - 'ndb.close()', - 'netns.remove(nsname)', - ), - (lambda v: v == 1,), - lambda *r: sum(r) == 1, - lambda *s: sum(s) == 1, - ), - ( - ( - 'import json', - 'from pyroute2 import NDB', - 'ndb = NDB()', - 'report = "".join(', - ' ndb', - ' .routes', - ' .summary()', - ' .select_fields("dst", "ifname")', - ' .format("json")', - ')', - '(', - ' len(', - ' tuple(', - ' filter(', - ' lambda x: x["dst"] == "127.0.0.1",', - ' json.loads(report)', - ' )', - ' )', - ' ),', - ')', - 'ndb.close()', - ), - (lambda v: v == 1,), - lambda *r: sum(r) == 1, - lambda *s: sum(s) == 1, - ), - ( - ( - 'from pyroute2 import NDB', - 'ndb = NDB()', - 's = ndb.routes.summary().format("csv")', - 's', - 's', - 'ndb.close()', - ), - ( - lambda v: v == 'target', - lambda v: v == 'tflags', - lambda v: v == 'table', - lambda v: v == 'ifname', - lambda v: v == 'dst', - lambda v: v == 'dst_len', - lambda v: v == 'gateway', - ), - lambda *r: all(r), - lambda *s: sum(s) == 2, - ), - ), - ids=( - 'summary', - 'select_fields+repr x2', - 'select_records+repr x2', - 'transform_fields+repr', - 'count', - 'format(json)', - 'format(csv)', - ), -) -def test_report(script, check, combined_fields, combined_records): - output = io.StringIO() - console = code.InteractiveConsole() - sys.ps1 = '' - with contextlib.redirect_stdout(output): - for line in script: - console.push(line) - assert combined_records( - *[ - check_output(source, combined_fields, *check) - for source in output.getvalue().split('\n') - if len(source) > 0 - ] - ) diff --git a/tests/test_core/test_ndb/test_sync_api.py b/tests/test_core/test_ndb/test_sync_api.py deleted file mode 100644 index 64b9d4377..000000000 --- a/tests/test_core/test_ndb/test_sync_api.py +++ /dev/null @@ -1,77 +0,0 @@ -import pytest - -from pyroute2 import NDB - - -@pytest.mark.parametrize( - 'sequence,spec', - ( - ( - ( - ('add_ip', (lambda n, x, s: s in x.ipaddr,)), - ('ensure_ip', (lambda n, x, s: s in x.ipaddr,)), - ('del_ip', (lambda n, x, s: s not in x.ipaddr,)), - ('ensure_ip', (lambda n, x, s: s in x.ipaddr,)), - ), - {'address': '10.1.2.3', 'prefixlen': 24}, - ), - ( - ( - ('add_neighbour', (lambda n, x, s: s in x.neighbours,)), - ('ensure_neighbour', (lambda n, x, s: s in x.neighbours,)), - ('del_neighbour', (lambda n, x, s: s not in x.neighbours,)), - ('ensure_neighbour', (lambda n, x, s: s in x.neighbours,)), - ), - {'dst': '10.1.2.4', 'lladdr': '00:11:22:00:11:22'}, - ), - ( - ( - ('add_port', (lambda n, x, s: s in x.ports,)), - ( - 'del_port', - ( - lambda n, x, s: s not in x.ports, - lambda n, x, s: s in n.interfaces, - ), - ), - ), - {'ifname': '{test_link_ifname}'}, - ), - ( - ( - ( - 'add_altname', - (lambda n, x, s: s['ifname'] in x['alt_ifname_list'],), - ), - ( - 'del_altname', - (lambda n, x, s: s['ifname'] not in x['alt_ifname_list'],), - ), - ), - {'ifname': 'pr-altname42'}, - ), - ), - ids=('ipaddr', 'neighbour', 'vlan', 'altname'), -) -def test_interface_sequence( - nsname, ndb, test_link_ifname, tmp_link_ifname, sequence, spec -): - for key, value in tuple(spec.items()): - if isinstance(value, str) and value[0] == '{' and value[-1] == '}': - spec[key] = value.format( - **{ - 'nsname': nsname, - 'test_link_ifname': test_link_ifname, - 'tmp_link_ifname': tmp_link_ifname, - } - ) - with NDB(sources=[{'target': 'localhost', 'netns': nsname}]) as test_ndb: - ifname = tmp_link_ifname - test_ndb.interfaces.create( - ifname=ifname, kind='bridge', state='up' - ).commit() - for func_name, check_list in sequence: - func = getattr(test_ndb.interfaces[ifname], func_name) - func(**spec).commit() - for check in check_list: - assert check(ndb, ndb.interfaces[ifname], spec) diff --git a/tests/test_core/test_plan9/test_basic.py b/tests/test_core/test_plan9/test_basic.py deleted file mode 100644 index 15e2a47e0..000000000 --- a/tests/test_core/test_plan9/test_basic.py +++ /dev/null @@ -1,31 +0,0 @@ -import time - -import pytest - - -@pytest.mark.asyncio -async def test_server_data_read(async_p9_context): - fid = await async_p9_context.client.fid('test_file') - response = await async_p9_context.client.read(fid) - assert response['data'] == async_p9_context.sample_data - - -@pytest.mark.asyncio -async def test_server_time_read(async_p9_context): - ts = time.time_ns() - fid = await async_p9_context.client.fid('test_time') - responses = [await async_p9_context.client.read(fid) for _ in range(5)] - times = {int(x['data'].decode('utf-8')) for x in responses} - assert len(times) == 5 - assert min(times) > ts - assert max(times) < time.time_ns() - - -@pytest.mark.asyncio -async def test_server_data_write(async_p9_context): - new_sample = b'aevei3PhaeGeiseh' - fid = await async_p9_context.client.fid('test_file') - await async_p9_context.client.write(fid, new_sample) - response = await async_p9_context.client.read(fid) - assert response['data'] == new_sample - assert new_sample != async_p9_context.sample_data diff --git a/tests/test_core/test_socket_create.py b/tests/test_core/test_socket_create.py deleted file mode 100644 index 4ca2da21c..000000000 --- a/tests/test_core/test_socket_create.py +++ /dev/null @@ -1,23 +0,0 @@ -import time - -import pytest - -from pyroute2 import AsyncIPRoute, config, netns -from pyroute2.common import uifname - - -def fake_create(*argv, **kwarg): - time.sleep(600) - - -@pytest.mark.asyncio -async def test_netns_timeout(monkeypatch): - monkeypatch.setattr(config, 'default_create_socket_timeout', 1) - monkeypatch.setattr(config, 'default_communicate_timeout', 0.3) - monkeypatch.setattr(netns, '_create_socket_child', fake_create) - - ipr = AsyncIPRoute(netns=uifname()) - ts_start = time.time() - with pytest.raises(TimeoutError): - await ipr.setup_endpoint() - assert time.time() - ts_start < 2 diff --git a/tests/test_core/test_statsd.py b/tests/test_core/test_statsd.py deleted file mode 100644 index 465af670a..000000000 --- a/tests/test_core/test_statsd.py +++ /dev/null @@ -1,109 +0,0 @@ -import shutil -import socket -import subprocess -import time -from collections import deque - -import pytest - -from pyroute2 import IPRoute, config -from pyroute2.common import uifname -from pyroute2.statsd import StatsDClientSocket - -PORT_UDP = 8234 -PORT_TCP = 8235 - - -class StatsDServer: - def __init__(self, tmp_path): - config = tmp_path / 'statsd.json' - config.write_text(f'{{"port": {PORT_UDP}, "mgmt_port": {PORT_TCP}}}') - statsd = shutil.which('statsd') - if statsd is None: - return pytest.skip('statsd not found') - self.server = subprocess.Popen( - [statsd, config.as_posix()], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - self.query('counters') - - def try_query(self, kind: str) -> str: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect(('localhost', PORT_TCP)) - s.send(f'{kind}\n'.encode()) - - response = [] - while True: - data = s.recv(4096) - if not data: - break - data = data.decode('utf-8') - response.append(data) - if 'END' in data: - break - - return ''.join(response) - - def query(self, kind: str) -> str: - start_time = time.time() - timeout = 5 - interval = 0.2 - - while True: - try: - return self.try_query(kind) - except (socket.error, socket.timeout): - if time.time() - start_time > timeout: - raise RuntimeError('statsd server unreachable') - time.sleep(interval) - - def get(self, kind: str, name: str) -> int: - d = deque(self.query(kind).split()) - while True: - token = d.popleft() - if token == f'{name}:': - return int(d.popleft()) - raise KeyError('metric not found') - - def close(self): - self.server.terminate() - self.server.kill() - - -@pytest.fixture -def statsd(tmp_path): - sd = StatsDServer(tmp_path) - yield sd - sd.close() - - -@pytest.mark.parametrize( - 'kind,func,ret', - ( - ('counters', lambda x, n: x.incr(n), 1), - ('gauges', lambda x, n: x.gauge(n, 10), 10), - ), -) -def test_call(statsd, kind, func, ret): - with StatsDClientSocket(address=('localhost', PORT_UDP)) as sc: - metric = uifname() - func(sc, metric) - assert statsd.get(kind, metric) == ret - - -def test_telemetry(statsd, monkeypatch): - monkeypatch.setattr(config, 'telemetry', ('localhost', PORT_UDP)) - with IPRoute() as ipr: - assert len(list(ipr.link_lookup(ifname='lo'))) == 1 - assert ( - len( - list( - filter( - lambda x: x == 'link_lookup:', - statsd.query('counters').split(), - ) - ) - ) - > 0 - ) diff --git a/tests/test_decoder/nl0.json b/tests/test_decoder/nl0.json deleted file mode 100644 index 53940c7ec..000000000 --- a/tests/test_decoder/nl0.json +++ /dev/null @@ -1,1591 +0,0 @@ -[ - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=724847, incl_len=36, orig_len=36, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "", - "exception": "error('unpack_from requires a buffer of at least 24 bytes for unpacking 4 bytes at offset 20 (actual buffer size is 20)')", - "data": "14:00:00:00:12:00:01:03:61:20:f2:65:00:00:00:00:00:00:00:00" - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=724888, incl_len=2916, orig_len=2916, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "", - "exception": null, - "data": { - "family": 0, - "__align": [], - "ifi_type": 772, - "index": 1, - "flags": 65609, - "change": 0, - "attrs": [ - [ - "IFLA_IFNAME", - "lo" - ], - [ - "IFLA_TXQLEN", - 1000 - ], - [ - "IFLA_OPERSTATE", - "UNKNOWN" - ], - [ - "IFLA_LINKMODE", - 0 - ], - [ - "IFLA_MTU", - 65536 - ], - [ - "IFLA_MIN_MTU", - 0 - ], - [ - "IFLA_MAX_MTU", - 0 - ], - [ - "IFLA_GROUP", - 0 - ], - [ - "IFLA_PROMISCUITY", - 0 - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 61 - } - } - ], - [ - "IFLA_NUM_TX_QUEUES", - 1 - ], - [ - "IFLA_GSO_MAX_SEGS", - 65535 - ], - [ - "IFLA_GSO_MAX_SIZE", - 65536 - ], - [ - "IFLA_GRO_MAX_SIZE", - 65536 - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 63 - } - } - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 64 - } - } - ], - [ - "IFLA_TSO_MAX_SIZE", - 524280 - ], - [ - "IFLA_TSO_MAX_SEGS", - 65535 - ], - [ - "IFLA_NUM_RX_QUEUES", - 1 - ], - [ - "IFLA_CARRIER", - 1 - ], - [ - "IFLA_QDISC", - "noqueue" - ], - [ - "IFLA_CARRIER_CHANGES", - 0 - ], - [ - "IFLA_CARRIER_UP_COUNT", - 0 - ], - [ - "IFLA_CARRIER_DOWN_COUNT", - 0 - ], - [ - "IFLA_PROTO_DOWN", - 0 - ], - [ - "IFLA_MAP", - { - "mem_start": 0, - "mem_end": 0, - "base_addr": 0, - "irq": 0, - "dma": 0, - "port": 0 - } - ], - [ - "IFLA_ADDRESS", - "00:00:00:00:00:00" - ], - [ - "IFLA_BROADCAST", - "00:00:00:00:00:00" - ], - [ - "IFLA_STATS64", - { - "rx_packets": 58259, - "tx_packets": 58259, - "rx_bytes": 4299127, - "tx_bytes": 4299127, - "rx_errors": 0, - "tx_errors": 0, - "rx_dropped": 0, - "tx_dropped": 0, - "multicast": 0, - "collisions": 0, - "rx_length_errors": 0, - "rx_over_errors": 0, - "rx_crc_errors": 0, - "rx_frame_errors": 0, - "rx_fifo_errors": 0, - "rx_missed_errors": 0, - "tx_aborted_errors": 0, - "tx_carrier_errors": 0, - "tx_fifo_errors": 0, - "tx_heartbeat_errors": 0, - "tx_window_errors": 0, - "rx_compressed": 0, - "tx_compressed": 0 - } - ], - [ - "IFLA_STATS", - { - "rx_packets": 58259, - "tx_packets": 58259, - "rx_bytes": 4299127, - "tx_bytes": 4299127, - "rx_errors": 0, - "tx_errors": 0, - "rx_dropped": 0, - "tx_dropped": 0, - "multicast": 0, - "collisions": 0, - "rx_length_errors": 0, - "rx_over_errors": 0, - "rx_crc_errors": 0, - "rx_frame_errors": 0, - "rx_fifo_errors": 0, - "rx_missed_errors": 0, - "tx_aborted_errors": 0, - "tx_carrier_errors": 0, - "tx_fifo_errors": 0, - "tx_heartbeat_errors": 0, - "tx_window_errors": 0, - "rx_compressed": 0, - "tx_compressed": 0 - } - ], - [ - "IFLA_XDP", - { - "attrs": [ - [ - "IFLA_XDP_ATTACHED", - null - ] - ] - } - ], - [ - "IFLA_AF_SPEC", - { - "attrs": [ - [ - "UNKNOWN", - { - "header": { - "length": 12, - "type": 45 - } - } - ], - [ - "AF_INET", - { - "dummy": 65672, - "forwarding": 1, - "mc_forwarding": 0, - "proxy_arp": 0, - "accept_redirects": 1, - "secure_redirects": 1, - "send_redirects": 1, - "shared_media": 1, - "rp_filter": 2, - "accept_source_route": 0, - "bootp_relay": 0, - "log_martians": 0, - "tag": 0, - "arpfilter": 0, - "medium_id": 0, - "noxfrm": 1, - "nopolicy": 1, - "force_igmp_version": 0, - "arp_announce": 0, - "arp_ignore": 0, - "promote_secondaries": 1, - "arp_accept": 0, - "arp_notify": 0, - "accept_local": 0, - "src_vmark": 0, - "proxy_arp_pvlan": 0, - "route_localnet": 0, - "igmpv2_unsolicited_report_interval": 10000, - "igmpv3_unsolicited_report_interval": 1000 - } - ], - [ - "AF_INET6", - { - "attrs": [ - [ - "IFLA_INET6_FLAGS", - 2147483648 - ], - [ - "IFLA_INET6_CACHEINFO", - { - "max_reasm_len": 65535, - "tstamp": 128, - "reachable_time": 28649, - "retrans_time": 1000 - } - ], - [ - "IFLA_INET6_CONF", - { - "forwarding": 0, - "hop_limit": 64, - "mtu": 65536, - "accept_ra": 1, - "accept_redirects": 1, - "autoconf": 1, - "dad_transmits": 1, - "router_solicitations": 4294967295, - "router_solicitation_interval": 4000, - "router_solicitation_delay": 1000, - "use_tempaddr": 4294967295, - "temp_valid_lft": 604800, - "temp_preferred_lft": 86400, - "regen_max_retry": 3, - "max_desync_factor": 600, - "max_addresses": 16, - "force_mld_version": 0, - "accept_ra_defrtr": 1, - "accept_ra_pinfo": 1, - "accept_ra_rtr_pref": 1, - "router_probe_interval": 60000, - "accept_ra_rt_info_max_plen": 0, - "proxy_ndp": 0, - "optimistic_dad": 0, - "accept_source_route": 0, - "mc_forwarding": 0, - "disable_ipv6": 0, - "accept_dad": 4294967295, - "force_tllao": 0, - "ndisc_notify": 0 - } - ], - [ - "IFLA_INET6_STATS", - { - "num": 37, - "inpkts": 27938, - "inoctets": 2067992, - "indelivers": 27938, - "outforwdatagrams": 0, - "outpkts": 27938, - "outoctets": 2067992, - "inhdrerrors": 0, - "intoobigerrors": 0, - "innoroutes": 0, - "inaddrerrors": 0, - "inunknownprotos": 0, - "intruncatedpkts": 0, - "indiscards": 0, - "outdiscards": 0, - "outnoroutes": 0, - "reasmtimeout": 0, - "reasmreqds": 0, - "reasmoks": 0, - "reasmfails": 0, - "fragoks": 0, - "fragfails": 0, - "fragcreates": 0, - "inmcastpkts": 0, - "outmcastpkts": 2, - "inbcastpkts": 0, - "outbcastpkts": 0, - "inmcastoctets": 0, - "outmcastoctets": 152, - "inbcastoctets": 0, - "outbcastoctets": 0, - "csumerrors": 0, - "noectpkts": 27938, - "ect1pkts": 0, - "ect0pkts": 0, - "cepkts": 0 - } - ], - [ - "IFLA_INET6_ICMP6STATS", - { - "num": 7, - "inmsgs": 1874, - "inerrors": 0, - "outmsgs": 1874, - "outerrors": 0, - "csumerrors": 0 - } - ], - [ - "IFLA_INET6_TOKEN", - "::" - ], - [ - "IFLA_INET6_ADDR_GEN_MODE", - 0 - ] - ] - } - ] - ] - } - ], - [ - "UNKNOWN", - { - "header": { - "length": 4, - "type": 32830 - } - } - ] - ], - "header": { - "length": 1432, - "type": 16, - "flags": 2, - "sequence_number": 1710366817, - "pid": 778142 - }, - "state": "up" - } - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=724951, incl_len=7476, orig_len=7476, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "", - "exception": null, - "data": { - "family": 0, - "__align": [], - "ifi_type": 1, - "index": 3, - "flags": 4099, - "change": 0, - "attrs": [ - [ - "IFLA_IFNAME", - "virbr1" - ], - [ - "IFLA_TXQLEN", - 1000 - ], - [ - "IFLA_OPERSTATE", - "DOWN" - ], - [ - "IFLA_LINKMODE", - 0 - ], - [ - "IFLA_MTU", - 1500 - ], - [ - "IFLA_MIN_MTU", - 68 - ], - [ - "IFLA_MAX_MTU", - 65535 - ], - [ - "IFLA_GROUP", - 0 - ], - [ - "IFLA_PROMISCUITY", - 0 - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 61 - } - } - ], - [ - "IFLA_NUM_TX_QUEUES", - 1 - ], - [ - "IFLA_GSO_MAX_SEGS", - 65535 - ], - [ - "IFLA_GSO_MAX_SIZE", - 65536 - ], - [ - "IFLA_GRO_MAX_SIZE", - 65536 - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 63 - } - } - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 64 - } - } - ], - [ - "IFLA_TSO_MAX_SIZE", - 65536 - ], - [ - "IFLA_TSO_MAX_SEGS", - 65535 - ], - [ - "IFLA_NUM_RX_QUEUES", - 1 - ], - [ - "IFLA_CARRIER", - 0 - ], - [ - "IFLA_QDISC", - "noqueue" - ], - [ - "IFLA_CARRIER_CHANGES", - 1 - ], - [ - "IFLA_CARRIER_UP_COUNT", - 0 - ], - [ - "IFLA_CARRIER_DOWN_COUNT", - 1 - ], - [ - "IFLA_PROTO_DOWN", - 0 - ], - [ - "IFLA_MAP", - { - "mem_start": 0, - "mem_end": 0, - "base_addr": 0, - "irq": 0, - "dma": 0, - "port": 0 - } - ], - [ - "IFLA_ADDRESS", - "52:54:00:9e:4d:b4" - ], - [ - "IFLA_BROADCAST", - "ff:ff:ff:ff:ff:ff" - ], - [ - "IFLA_STATS64", - { - "rx_packets": 0, - "tx_packets": 0, - "rx_bytes": 0, - "tx_bytes": 0, - "rx_errors": 0, - "tx_errors": 0, - "rx_dropped": 0, - "tx_dropped": 0, - "multicast": 0, - "collisions": 0, - "rx_length_errors": 0, - "rx_over_errors": 0, - "rx_crc_errors": 0, - "rx_frame_errors": 0, - "rx_fifo_errors": 0, - "rx_missed_errors": 0, - "tx_aborted_errors": 0, - "tx_carrier_errors": 0, - "tx_fifo_errors": 0, - "tx_heartbeat_errors": 0, - "tx_window_errors": 0, - "rx_compressed": 0, - "tx_compressed": 0 - } - ], - [ - "IFLA_STATS", - { - "rx_packets": 0, - "tx_packets": 0, - "rx_bytes": 0, - "tx_bytes": 0, - "rx_errors": 0, - "tx_errors": 0, - "rx_dropped": 0, - "tx_dropped": 0, - "multicast": 0, - "collisions": 0, - "rx_length_errors": 0, - "rx_over_errors": 0, - "rx_crc_errors": 0, - "rx_frame_errors": 0, - "rx_fifo_errors": 0, - "rx_missed_errors": 0, - "tx_aborted_errors": 0, - "tx_carrier_errors": 0, - "tx_fifo_errors": 0, - "tx_heartbeat_errors": 0, - "tx_window_errors": 0, - "rx_compressed": 0, - "tx_compressed": 0 - } - ], - [ - "IFLA_XDP", - { - "attrs": [ - [ - "IFLA_XDP_ATTACHED", - null - ] - ] - } - ], - [ - "IFLA_LINKINFO", - { - "attrs": [ - [ - "IFLA_INFO_KIND", - "bridge" - ], - [ - "IFLA_INFO_DATA", - { - "attrs": [ - [ - "IFLA_BR_HELLO_TIMER", - 173 - ], - [ - "IFLA_BR_TCN_TIMER", - 0 - ], - [ - "IFLA_BR_TOPOLOGY_CHANGE_TIMER", - 0 - ], - [ - "IFLA_BR_GC_TIMER", - 9722 - ], - [ - "IFLA_BR_FORWARD_DELAY", - 200 - ], - [ - "IFLA_BR_HELLO_TIME", - 200 - ], - [ - "IFLA_BR_MAX_AGE", - 2000 - ], - [ - "IFLA_BR_AGEING_TIME", - 30000 - ], - [ - "IFLA_BR_STP_STATE", - 1 - ], - [ - "IFLA_BR_PRIORITY", - 32768 - ], - [ - "IFLA_BR_VLAN_FILTERING", - 0 - ], - [ - "IFLA_BR_GROUP_FWD_MASK", - 0 - ], - [ - "IFLA_BR_BRIDGE_ID", - { - "prio": 128, - "addr": "52:54:00:9e:4d:b4" - } - ], - [ - "IFLA_BR_ROOT_ID", - { - "prio": 128, - "addr": "52:54:00:9e:4d:b4" - } - ], - [ - "IFLA_BR_ROOT_PORT", - 0 - ], - [ - "IFLA_BR_ROOT_PATH_COST", - 0 - ], - [ - "IFLA_BR_TOPOLOGY_CHANGE", - 0 - ], - [ - "IFLA_BR_TOPOLOGY_CHANGE_DETECTED", - 0 - ], - [ - "IFLA_BR_GROUP_ADDR", - "01:80:c2:00:00:00" - ], - [ - "UNKNOWN", - { - "header": { - "length": 12, - "type": 46 - } - } - ], - [ - "IFLA_BR_VLAN_PROTOCOL", - 33024 - ], - [ - "IFLA_BR_VLAN_DEFAULT_PVID", - 1 - ], - [ - "IFLA_BR_VLAN_STATS_ENABLED", - 0 - ], - [ - "UNKNOWN", - { - "header": { - "length": 5, - "type": 45 - } - } - ], - [ - "IFLA_BR_MCAST_ROUTER", - 1 - ], - [ - "IFLA_BR_MCAST_SNOOPING", - 1 - ], - [ - "IFLA_BR_MCAST_QUERY_USE_IFADDR", - 0 - ], - [ - "IFLA_BR_MCAST_QUERIER", - 0 - ], - [ - "IFLA_BR_MCAST_STATS_ENABLED", - 0 - ], - [ - "IFLA_BR_MCAST_HASH_ELASTICITY", - 16 - ], - [ - "IFLA_BR_MCAST_HASH_MAX", - 4096 - ], - [ - "IFLA_BR_MCAST_LAST_MEMBER_CNT", - 2 - ], - [ - "IFLA_BR_MCAST_STARTUP_QUERY_CNT", - 2 - ], - [ - "IFLA_BR_MCAST_IGMP_VERSION", - 2 - ], - [ - "IFLA_BR_MCAST_MLD_VERSION", - 1 - ], - [ - "IFLA_BR_MCAST_LAST_MEMBER_INTVL", - 100 - ], - [ - "IFLA_BR_MCAST_MEMBERSHIP_INTVL", - 26000 - ], - [ - "IFLA_BR_MCAST_QUERIER_INTVL", - 25500 - ], - [ - "IFLA_BR_MCAST_QUERY_INTVL", - 12500 - ], - [ - "IFLA_BR_MCAST_QUERY_RESPONSE_INTVL", - 1000 - ], - [ - "IFLA_BR_MCAST_STARTUP_QUERY_INTVL", - 3125 - ], - [ - "IFLA_BR_NF_CALL_IPTABLES", - 0 - ], - [ - "IFLA_BR_NF_CALL_IP6TABLES", - 0 - ], - [ - "IFLA_BR_NF_CALL_ARPTABLES", - 0 - ] - ] - } - ] - ] - } - ], - [ - "IFLA_AF_SPEC", - { - "attrs": [ - [ - "AF_INET", - { - "dummy": 65672, - "forwarding": 1, - "mc_forwarding": 0, - "proxy_arp": 0, - "accept_redirects": 1, - "secure_redirects": 1, - "send_redirects": 1, - "shared_media": 1, - "rp_filter": 2, - "accept_source_route": 0, - "bootp_relay": 0, - "log_martians": 0, - "tag": 0, - "arpfilter": 0, - "medium_id": 0, - "noxfrm": 0, - "nopolicy": 0, - "force_igmp_version": 0, - "arp_announce": 0, - "arp_ignore": 0, - "promote_secondaries": 1, - "arp_accept": 0, - "arp_notify": 0, - "accept_local": 0, - "src_vmark": 0, - "proxy_arp_pvlan": 0, - "route_localnet": 0, - "igmpv2_unsolicited_report_interval": 10000, - "igmpv3_unsolicited_report_interval": 1000 - } - ], - [ - "AF_INET6", - { - "attrs": [ - [ - "IFLA_INET6_FLAGS", - 0 - ], - [ - "IFLA_INET6_CACHEINFO", - { - "max_reasm_len": 65535, - "tstamp": 3570, - "reachable_time": 28040, - "retrans_time": 1000 - } - ], - [ - "IFLA_INET6_CONF", - { - "forwarding": 0, - "hop_limit": 64, - "mtu": 1500, - "accept_ra": 0, - "accept_redirects": 1, - "autoconf": 0, - "dad_transmits": 1, - "router_solicitations": 4294967295, - "router_solicitation_interval": 4000, - "router_solicitation_delay": 1000, - "use_tempaddr": 0, - "temp_valid_lft": 604800, - "temp_preferred_lft": 86400, - "regen_max_retry": 3, - "max_desync_factor": 600, - "max_addresses": 16, - "force_mld_version": 0, - "accept_ra_defrtr": 1, - "accept_ra_pinfo": 1, - "accept_ra_rtr_pref": 1, - "router_probe_interval": 60000, - "accept_ra_rt_info_max_plen": 0, - "proxy_ndp": 0, - "optimistic_dad": 0, - "accept_source_route": 0, - "mc_forwarding": 0, - "disable_ipv6": 1, - "accept_dad": 1, - "force_tllao": 0, - "ndisc_notify": 0 - } - ], - [ - "IFLA_INET6_STATS", - { - "num": 37, - "inpkts": 0, - "inoctets": 0, - "indelivers": 0, - "outforwdatagrams": 0, - "outpkts": 0, - "outoctets": 0, - "inhdrerrors": 0, - "intoobigerrors": 0, - "innoroutes": 0, - "inaddrerrors": 0, - "inunknownprotos": 0, - "intruncatedpkts": 0, - "indiscards": 0, - "outdiscards": 0, - "outnoroutes": 0, - "reasmtimeout": 0, - "reasmreqds": 0, - "reasmoks": 0, - "reasmfails": 0, - "fragoks": 0, - "fragfails": 0, - "fragcreates": 0, - "inmcastpkts": 0, - "outmcastpkts": 0, - "inbcastpkts": 0, - "outbcastpkts": 0, - "inmcastoctets": 0, - "outmcastoctets": 0, - "inbcastoctets": 0, - "outbcastoctets": 0, - "csumerrors": 0, - "noectpkts": 0, - "ect1pkts": 0, - "ect0pkts": 0, - "cepkts": 0 - } - ], - [ - "IFLA_INET6_ICMP6STATS", - { - "num": 7, - "inmsgs": 0, - "inerrors": 0, - "outmsgs": 0, - "outerrors": 0, - "csumerrors": 0 - } - ], - [ - "IFLA_INET6_TOKEN", - "::" - ], - [ - "IFLA_INET6_ADDR_GEN_MODE", - 0 - ] - ] - } - ] - ] - } - ], - [ - "UNKNOWN", - { - "header": { - "length": 4, - "type": 32830 - } - } - ] - ], - "header": { - "length": 1852, - "type": 16, - "flags": 2, - "sequence_number": 1710366817, - "pid": 778142 - }, - "state": "up" - } - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=724992, incl_len=5660, orig_len=5660, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "", - "exception": null, - "data": { - "family": 0, - "__align": [], - "ifi_type": 1, - "index": 239, - "flags": 4098, - "change": 0, - "attrs": [ - [ - "IFLA_IFNAME", - "gretap0" - ], - [ - "IFLA_TXQLEN", - 1000 - ], - [ - "IFLA_OPERSTATE", - "DOWN" - ], - [ - "IFLA_LINKMODE", - 0 - ], - [ - "IFLA_MTU", - 1462 - ], - [ - "IFLA_MIN_MTU", - 68 - ], - [ - "IFLA_MAX_MTU", - 0 - ], - [ - "IFLA_GROUP", - 0 - ], - [ - "IFLA_PROMISCUITY", - 0 - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 61 - } - } - ], - [ - "IFLA_NUM_TX_QUEUES", - 1 - ], - [ - "IFLA_GSO_MAX_SEGS", - 65535 - ], - [ - "IFLA_GSO_MAX_SIZE", - 65536 - ], - [ - "IFLA_GRO_MAX_SIZE", - 65536 - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 63 - } - } - ], - [ - "UNKNOWN", - { - "header": { - "length": 8, - "type": 64 - } - } - ], - [ - "IFLA_TSO_MAX_SIZE", - 65536 - ], - [ - "IFLA_TSO_MAX_SEGS", - 65535 - ], - [ - "IFLA_NUM_RX_QUEUES", - 1 - ], - [ - "IFLA_CARRIER", - 1 - ], - [ - "IFLA_QDISC", - "noop" - ], - [ - "IFLA_CARRIER_CHANGES", - 0 - ], - [ - "IFLA_CARRIER_UP_COUNT", - 0 - ], - [ - "IFLA_CARRIER_DOWN_COUNT", - 0 - ], - [ - "IFLA_PROTO_DOWN", - 0 - ], - [ - "IFLA_MAP", - { - "mem_start": 0, - "mem_end": 0, - "base_addr": 0, - "irq": 0, - "dma": 0, - "port": 0 - } - ], - [ - "IFLA_ADDRESS", - "00:00:00:00:00:00" - ], - [ - "IFLA_BROADCAST", - "ff:ff:ff:ff:ff:ff" - ], - [ - "IFLA_STATS64", - { - "rx_packets": 0, - "tx_packets": 0, - "rx_bytes": 0, - "tx_bytes": 0, - "rx_errors": 0, - "tx_errors": 0, - "rx_dropped": 0, - "tx_dropped": 0, - "multicast": 0, - "collisions": 0, - "rx_length_errors": 0, - "rx_over_errors": 0, - "rx_crc_errors": 0, - "rx_frame_errors": 0, - "rx_fifo_errors": 0, - "rx_missed_errors": 0, - "tx_aborted_errors": 0, - "tx_carrier_errors": 0, - "tx_fifo_errors": 0, - "tx_heartbeat_errors": 0, - "tx_window_errors": 0, - "rx_compressed": 0, - "tx_compressed": 0 - } - ], - [ - "IFLA_STATS", - { - "rx_packets": 0, - "tx_packets": 0, - "rx_bytes": 0, - "tx_bytes": 0, - "rx_errors": 0, - "tx_errors": 0, - "rx_dropped": 0, - "tx_dropped": 0, - "multicast": 0, - "collisions": 0, - "rx_length_errors": 0, - "rx_over_errors": 0, - "rx_crc_errors": 0, - "rx_frame_errors": 0, - "rx_fifo_errors": 0, - "rx_missed_errors": 0, - "tx_aborted_errors": 0, - "tx_carrier_errors": 0, - "tx_fifo_errors": 0, - "tx_heartbeat_errors": 0, - "tx_window_errors": 0, - "rx_compressed": 0, - "tx_compressed": 0 - } - ], - [ - "IFLA_XDP", - { - "attrs": [ - [ - "IFLA_XDP_ATTACHED", - null - ] - ] - } - ], - [ - "IFLA_LINKINFO", - { - "attrs": [ - [ - "IFLA_INFO_KIND", - "gretap" - ], - [ - "IFLA_INFO_DATA", - { - "attrs": [ - [ - "IFLA_GRE_LINK", - 0 - ], - [ - "IFLA_GRE_IFLAGS", - 0 - ], - [ - "IFLA_GRE_OFLAGS", - 0 - ], - [ - "IFLA_GRE_IKEY", - 0 - ], - [ - "IFLA_GRE_OKEY", - 0 - ], - [ - "IFLA_GRE_LOCAL", - "0.0.0.0" - ], - [ - "IFLA_GRE_REMOTE", - "0.0.0.0" - ], - [ - "IFLA_GRE_TTL", - 0 - ], - [ - "IFLA_GRE_TOS", - 0 - ], - [ - "IFLA_GRE_PMTUDISC", - 0 - ], - [ - "IFLA_GRE_FWMARK", - 0 - ], - [ - "IFLA_GRE_ENCAP_TYPE", - 0 - ], - [ - "IFLA_GRE_ENCAP_SPORT", - 0 - ], - [ - "IFLA_GRE_ENCAP_DPORT", - 0 - ], - [ - "IFLA_GRE_ENCAP_FLAGS", - 0 - ], - [ - "IFLA_GRE_IGNORE_DF", - 0 - ] - ] - } - ] - ] - } - ], - [ - "IFLA_LINK", - 0 - ], - [ - "IFLA_AF_SPEC", - { - "attrs": [ - [ - "AF_INET", - { - "dummy": 65672, - "forwarding": 1, - "mc_forwarding": 0, - "proxy_arp": 0, - "accept_redirects": 1, - "secure_redirects": 1, - "send_redirects": 1, - "shared_media": 1, - "rp_filter": 2, - "accept_source_route": 0, - "bootp_relay": 0, - "log_martians": 0, - "tag": 0, - "arpfilter": 0, - "medium_id": 0, - "noxfrm": 0, - "nopolicy": 0, - "force_igmp_version": 0, - "arp_announce": 0, - "arp_ignore": 0, - "promote_secondaries": 1, - "arp_accept": 0, - "arp_notify": 0, - "accept_local": 0, - "src_vmark": 0, - "proxy_arp_pvlan": 0, - "route_localnet": 0, - "igmpv2_unsolicited_report_interval": 10000, - "igmpv3_unsolicited_report_interval": 1000 - } - ], - [ - "AF_INET6", - { - "attrs": [ - [ - "IFLA_INET6_FLAGS", - 0 - ], - [ - "IFLA_INET6_CACHEINFO", - { - "max_reasm_len": 65535, - "tstamp": 13484027, - "reachable_time": 15309, - "retrans_time": 1000 - } - ], - [ - "IFLA_INET6_CONF", - { - "forwarding": 0, - "hop_limit": 64, - "mtu": 1500, - "accept_ra": 1, - "accept_redirects": 1, - "autoconf": 1, - "dad_transmits": 1, - "router_solicitations": 4294967295, - "router_solicitation_interval": 4000, - "router_solicitation_delay": 1000, - "use_tempaddr": 0, - "temp_valid_lft": 604800, - "temp_preferred_lft": 86400, - "regen_max_retry": 3, - "max_desync_factor": 600, - "max_addresses": 16, - "force_mld_version": 0, - "accept_ra_defrtr": 1, - "accept_ra_pinfo": 1, - "accept_ra_rtr_pref": 1, - "router_probe_interval": 60000, - "accept_ra_rt_info_max_plen": 0, - "proxy_ndp": 0, - "optimistic_dad": 0, - "accept_source_route": 0, - "mc_forwarding": 0, - "disable_ipv6": 0, - "accept_dad": 1, - "force_tllao": 0, - "ndisc_notify": 0 - } - ], - [ - "IFLA_INET6_STATS", - { - "num": 37, - "inpkts": 0, - "inoctets": 0, - "indelivers": 0, - "outforwdatagrams": 0, - "outpkts": 0, - "outoctets": 0, - "inhdrerrors": 0, - "intoobigerrors": 0, - "innoroutes": 0, - "inaddrerrors": 0, - "inunknownprotos": 0, - "intruncatedpkts": 0, - "indiscards": 0, - "outdiscards": 0, - "outnoroutes": 0, - "reasmtimeout": 0, - "reasmreqds": 0, - "reasmoks": 0, - "reasmfails": 0, - "fragoks": 0, - "fragfails": 0, - "fragcreates": 0, - "inmcastpkts": 0, - "outmcastpkts": 0, - "inbcastpkts": 0, - "outbcastpkts": 0, - "inmcastoctets": 0, - "outmcastoctets": 0, - "inbcastoctets": 0, - "outbcastoctets": 0, - "csumerrors": 0, - "noectpkts": 0, - "ect1pkts": 0, - "ect0pkts": 0, - "cepkts": 0 - } - ], - [ - "IFLA_INET6_ICMP6STATS", - { - "num": 7, - "inmsgs": 0, - "inerrors": 0, - "outmsgs": 0, - "outerrors": 0, - "csumerrors": 0 - } - ], - [ - "IFLA_INET6_TOKEN", - "::" - ], - [ - "IFLA_INET6_ADDR_GEN_MODE", - 0 - ] - ] - } - ] - ] - } - ], - [ - "UNKNOWN", - { - "header": { - "length": 4, - "type": 32830 - } - } - ] - ], - "header": { - "length": 1580, - "type": 16, - "flags": 2, - "sequence_number": 1710366817, - "pid": 778142 - }, - "state": "down" - } - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=725005, incl_len=36, orig_len=36, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "None", - "exception": "KeyError(3)", - "data": "14:00:00:00:03:00:02:00:61:20:f2:65:9e:df:0b:00:00:00:00:00" - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=725018, incl_len=36, orig_len=36, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "", - "exception": "error('unpack_from requires a buffer of at least 24 bytes for unpacking 4 bytes at offset 20 (actual buffer size is 20)')", - "data": "14:00:00:00:16:00:01:03:62:20:f2:65:00:00:00:00:00:00:00:00" - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=725029, incl_len=440, orig_len=440, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "", - "exception": null, - "data": { - "family": 2, - "prefixlen": 8, - "flags": 128, - "scope": 254, - "index": 1, - "attrs": [ - [ - "IFA_ADDRESS", - "127.0.0.1" - ], - [ - "IFA_LOCAL", - "127.0.0.1" - ], - [ - "IFA_LABEL", - "lo" - ], - [ - "IFA_FLAGS", - 128 - ], - [ - "IFA_CACHEINFO", - { - "ifa_preferred": 4294967295, - "ifa_valid": 4294967295, - "cstamp": 128, - "tstamp": 128 - } - ] - ], - "header": { - "length": 76, - "type": 20, - "flags": 2, - "sequence_number": 1710366818, - "pid": 778142 - } - } - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=725040, incl_len=304, orig_len=304, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "", - "exception": null, - "data": { - "family": 10, - "prefixlen": 128, - "flags": 128, - "scope": 254, - "index": 1, - "attrs": [ - [ - "IFA_ADDRESS", - "::1" - ], - [ - "IFA_CACHEINFO", - { - "ifa_preferred": 4294967295, - "ifa_valid": 4294967295, - "cstamp": 128, - "tstamp": 128 - } - ], - [ - "IFA_FLAGS", - 640 - ] - ], - "header": { - "length": 72, - "type": 20, - "flags": 2, - "sequence_number": 1710366818, - "pid": 778142 - } - } - }, - { - "pcap header": "PcapPacketHeader(ts_sec=1710366817, ts_usec=725048, incl_len=36, orig_len=36, header_len=16)", - "link layer header": "PcapLLHeader(pad0=4, addr_type=824, pad1=0, pad2=0, pad3=0, family=0, header_len=16)", - "message class": "None", - "exception": "KeyError(3)", - "data": "14:00:00:00:03:00:02:00:62:20:f2:65:9e:df:0b:00:00:00:00:00" - } -] diff --git a/tests/test_decoder/nl0.pcap b/tests/test_decoder/nl0.pcap deleted file mode 100644 index c1139ff6d482922b3d7a50a5e14581539a840ef1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20884 zcmeHPYiwLc6+YMd@b!9aZ&N3ALs~nnO(1oW&4Vf=A-E+H6cs6?C5oUG&BJvq632Ct zMpYwvivk1@N~8uUKon9@AVe*S5Q<708uR#39zh_g_6HSJ#hJGk13H z-rcp=vDdq9_h@I&%(*jXX3oruBe1#ao&KYrO~?xP=_DFi zN#sMS8=h{7MIQU+Kl+!sBG(}7N<!vHTt!y{6glr(~nXbL1fMP0B)PJpY4D-#zKzS%W+eSRt_gV!nYz*}FMuCwL2zZ3^6!u~aE>6KsOA%gZyTN`(E?Y1pHu%l z`8Y5kDK|)0!uVryqtv#|+;Z|a@r*D$Tx6)#t3pzW0+vh=S919hU9LI0dxz8YdbwQXC`6zy|9n5f1Ze&!)^hiJ>PFqO#UIQ=pd}?TH^tnbvAW z58JzaIK69G`hIHMpnm2Wg8G>;Y0JI}UaFr#Q2mU>wJTGnpV_<1)z7=NOiNVTyOLHs zQT@E>j|+Q0|D<5`)1dvA{h0bWelHXnVE^gWikA+)+`+DX_WR2>Dk1duomnA5jRdKm zSr^uWbqJP!s>&SH&k4CztW3M1t0$|Ui;2~H-^VKgF>Sy+I0`szt z`pxQRKAWt54%Wk#J4h?#XWoRceh$VrG4Nleetv$$r1)Fvd+K?8D9b@TXZ<O#)uC{hODX-# zl3gP!B@I}G-_jL})!=pI!!o>e5KXt%qb>akZ0;2|V80hGY;&){T#;{c8+#$*8w<_Q z?GFu(Y#CWfJ)N_=S>&MIZT0#((5cs{2d6`?BRkl0`F1zu612M+o~T~maJL-$)Xxg_ zI`exs0QGvMA*k1%c4lGc+mS6JgWCrBMUFb$Cr-&Tej&KeZ6S%3qlKhgE0=mqh&KC+ z@N$pZ;E4PQ)u3IG-a{NyT^mP#5RYwc>S%WG*-x55HZ7aPjBWkgu_Um66Jq|_EQURK zLISH<$NQD~lQ^>=kTZA9A5*+Q^Ns2Hk?9vP3h*4Cew~jyeERi1UZ~iWxoVs1c)_@H zWuDqsF827%IJ?Eiy~-zv{r+v5l6C5oLy#vSDP3}OtM$*9m&}qA^qaD4jwIHlA2QWN z`MNc7%(t>*@_r51@VtwbFzcds&{PL@j6LtNG93F;Iq#AmorS5GcP+0x95tcw)VX0Y1n1p?hvN`R&AUv-xwY8b$!0504J^cMfgev2!RicTyC= zb>}GhGWQcrDpiNz`Z9$zBhSmU&p=K!yoSl1?L6_%#d2tEp&n+Rw(AUwr*aL$ZbW{zJJr`P zRuZ*|E2)PI&)lzM*5XyVKL_;i-hoAfLpUr6u3;#tpdLO5dUzpU0RWuvad*oW4X2)(r+B9PQ4g~&tOx5*iSh^aa69yHEBemf!IA!fJ)1|k zPc@M@Np50se&vUKL;3o_9GW$4z>8zVdy-c1{t(VEO z0P1^71tPf`LM8Mv+lzZk2XQWye=d`Vyp3m`U(hDl#{(n%y2BgP%VxjW?4@&G{5K)J z%;eLhmz(rlX6LZU^JXpiWb|_Q9d+8v?-rQuJ*7j?)08LrknD%HwX%)-nm3OAMJWCL zu{m3@?}^0p76WYp7!R6`ZvgU)1jejluFx1ZeKYQ#t;BG?L$LP?4&f}Mmd^a`8LFh3 zW!@Z77d}Jn#c@;qSasw)>eOJW4EsdK>h8_MY1Fgy8R}_&tgcge6s~2BoGP=Ho$9d~ zes_`cnteBh^>{bHbdTA?7_&S>#h683!n;K^G;V8ny`8n|{&)7AoWtu_;dd$d?t-=F zEM{-5<#%<8j-R?+S5zz$C04?&V;S_!u*h6-0ndVSMgPdI;mz3i9BZyPS)VH=+N@QL zKizZ1Q~q3G`V!W;m@Dl2D6Y&_SroJRy6|MWZN8Z@cYI@eeeS60AesT0S90zcF4z;0 z$iE>I&zB2t;O^ctv^|aRSaZjV^*sS!VzrH?d(PO6Iim@d@Zj(uRu{OV7g%2ezYD-; z{7!+v=uCjp>wohOWp3SX-tCO*iEVAK`&Sg$YHsAesH?{c1GopbYj8J*TY2M6$K7)5 zc<@YldhWbT^}62L47NYi_u$z2liH+oLaKT>F;alW(bdW(bW zd-^AP7l%Al`1rHuP0Y(eDZhOlzKdh&dAPp5&lAVh1}bTz@xDz9_KLevIJ)p&F-Iux z(&z5Dn|twd$4P^m1d%P^O_l_v??IYp(Xr^l`P>OoJMv)~-lcDFy(^4Hee)=W19c2> z?oPd{^IAJJ1N$<;Lwve>;PQCk8Uxh}c|D!F@@4WeI7*x;yblWfgaqXMg&&E!2XJrh z^rm$VUvA3h%`f;E-lHcLXgLjz3UDU>K2l+s6F6?5E50y>L_ud5f`SmxF&187SOviRiqaD*|9J+8j zej=1#!ZYoN8!32_2TrlJ)4Cw;JTr%vA0X?e9R3AWR$Z<%(|?>!ac)8}bl z2=aO5`J+evwXpF>usq~LSF${P-es^yUvT7#XUa0t5dG5EF3uSZ@)jEj|%lA0OP6VQE0*^b^Z@i1qnSKY-9|oN+oPIxO zDMkb^J+3dhm?%JCg_ZTVQg^Iw!7jV4pJ28!)4raY7p9#C~RI5$$_s%vc(~ zT;`x>n|Pezd2fa^S`fLk=DT0aGe_h1EgZeyl%rAFv{_9&@@PkmNaMHBa`DXf3uv9> zXw4drWhKp|O)8DokaF=_l*SuaIU2vS=IBXbj@G6$$~Q+#DUBzKmZszCKx-*IoT47? zfllS5%{Gcc(At-fW_8xXuc;$eV}6tG5M`HhVQ`{D(g*PeGFy`^Ia#ls)3|R)A-4k;A+bev$YIQ zgN@C&4;&`?eW+kQeAmhX>tKN`Y9Xn;!i0ENzyjra_c(3Bv1;YZbWHQ?!B*jXMi*|A yH6Zm!j7^~J7z6G`AlxP#YitwS4pGD*?=#?iA9&D(d9MdarSJ283vDx*{QeKvb4_dj diff --git a/tests/test_decoder/test_pcap.py b/tests/test_decoder/test_pcap.py deleted file mode 100644 index 04c814775..000000000 --- a/tests/test_decoder/test_pcap.py +++ /dev/null @@ -1,51 +0,0 @@ -import json -import subprocess - - -def test_pcap_rtnl(): - decoder = subprocess.Popen( - [ - "pyroute2-decoder", - "-c", - "pyroute2/netlink/rtnl/marshal.MarshalRtnl", - "-d", - "test_decoder/nl0.pcap", - "-m", - "ll_header{family=0}", - ], - stdout=subprocess.PIPE, - ) - dump = json.loads(decoder.communicate()[0]) - decoder.wait() - with open("test_decoder/nl0.json", 'r') as f: - ref = json.load(f) - assert len(ref) == len(dump) - for i in range(len(ref)): - assert ref[i]["pcap header"] == dump[i]["pcap header"] - assert ref[i]["message class"] == dump[i]["message class"] - - -def test_pcap_ipvs(): - decoder = subprocess.Popen( - [ - "pyroute2-decoder", - "-c", - "pyroute2/netlink/generic/ipvs.ipvsmsg", - "-d", - "test_decoder/nl0.pcap", - "-m", - ( - "ll_header{family=16}" - " AND data{fmt='H', offset=4, value=37}" - " AND data{fmt='B', offset=16, value=1}" - ), - ], - stdout=subprocess.PIPE, - ) - dump = json.loads(decoder.communicate()[0]) - decoder.wait() - assert len(dump) == 1 - assert dump[0]["data"]["cmd"] == 1 - assert dump[0]["data"]["header"]["type"] == 37 - assert dump[0]["link layer header"].find("family=16") > 0 - assert dump[0]["data"]["attrs"][0][0] == "IPVS_CMD_ATTR_SERVICE" diff --git a/tests/test_integration/conftest.py b/tests/test_integration/conftest.py deleted file mode 100644 index 10dc93a9a..000000000 --- a/tests/test_integration/conftest.py +++ /dev/null @@ -1 +0,0 @@ -pytest_plugins = ['pyroute2.fixtures.iproute', 'pyroute2.fixtures.ndb'] diff --git a/tests/test_integration/test_kuryr.py b/tests/test_integration/test_kuryr.py index bc01a755e..12423a399 100644 --- a/tests/test_integration/test_kuryr.py +++ b/tests/test_integration/test_kuryr.py @@ -1,90 +1,7 @@ -import pytest -from net_tools import interface_exists - import pyroute2 -from pyroute2 import IPDB -from pyroute2.common import uifname -from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg - -# from pyroute2.common import uifname -TADDR = '00:11:22:33:44:55' -KIND = 'ipvlan' -IPVLAN_MODE_L2 = ifinfmsg.ifinfo.data_map['ipvlan'].modes['IPVLAN_MODE_L2'] - - -@pytest.fixture -def ipdb(nsname): - with IPDB(sources=[{'target': 'localhost', 'netns': nsname}]) as ip: - yield ip - - -@pytest.mark.parametrize( - 'exc', - ( - pyroute2.NetlinkError, - pyroute2.CreateException, - pyroute2.CommitException, - ), -) -def test_exception_types(exc): - assert issubclass(exc, Exception) - - -def test_ipdb_create_exception(test_link_ifname, ipdb): - with pytest.raises(pyroute2.CreateException): - ipdb.create(ifname=test_link_ifname, kind='dummy').commit() - - -def test_ipdb_create_reuse(test_link_ifname, ipdb): - ipdb.create(ifname=test_link_ifname, kind='dummy', reuse=True).commit() - - -@pytest.mark.parametrize( - 'method,argv,check', - ( - ('set_mtu', [1000], lambda x: x['mtu'] == 1000), - ('set_address', [TADDR], lambda x: x['address'] == TADDR), - ('add_ip', ['10.1.2.3', 24], lambda x: '10.1.2.3/24' in x.ipaddr), - ('up', [], lambda x: x['flags'] & 1), - ), -) -def test_ipdb_iface_methods(test_link_ifname, ipdb, method, argv, check): - iface = ipdb.interfaces[test_link_ifname] - with iface: - getattr(iface, method)(*argv) - assert check(iface) - - -def test_utils_remove(nsname, test_link_index, test_link_ifname, ipdb): - index = ipdb.interfaces.get(test_link_ifname, {}).get('index', None) - assert isinstance(index, int) - assert index == test_link_index - with ipdb.interfaces[test_link_ifname] as iface: - iface.remove() - assert test_link_ifname not in ipdb.interfaces - assert test_link_index not in ipdb.interfaces - assert not interface_exists(test_link_ifname, netns=nsname, timeout=0.1) - - -def test_get_iface(test_link_ifname, ipdb): - with ipdb.interfaces[test_link_ifname] as link: - link.set_address(TADDR) - target = None - for name, data in ipdb.interfaces.items(): - if data['address'] == TADDR: - target = data['ifname'] - assert target == test_link_ifname -def test_create_ipvlan(test_link_index, test_link_ifname, ipdb): - ipvlname = uifname() - with ipdb.create( - ifname=ipvlname, - kind=KIND, - link=ipdb.interfaces[test_link_ifname], - ipvlan_mode=IPVLAN_MODE_L2, - ) as iface: - assert iface['mode'] == IPVLAN_MODE_L2 - assert iface['ifname'] == ipvlname - assert iface['link'] == test_link_index - assert iface['kind'] == KIND +def test_exceptions(): + assert issubclass(pyroute2.NetlinkError, Exception) + assert issubclass(pyroute2.CreateException, Exception) + assert issubclass(pyroute2.CommitException, Exception) diff --git a/tests/test_integration/test_lnst.py b/tests/test_integration/test_lnst.py index acefe7c1b..363155f3a 100644 --- a/tests/test_integration/test_lnst.py +++ b/tests/test_integration/test_lnst.py @@ -2,43 +2,107 @@ import pytest -from pyroute2 import IPRoute, IPRSocket -from pyroute2.netlink import NLM_F_DUMP, NLM_F_REQUEST -from pyroute2.netlink.rtnl import RTM_GETADDR, RTM_GETLINK +# NetlinkError, NetlinkDecodeError +# IPRoute +# IPRSocket +from pyroute2 import IPRoute as IPRoute0 +from pyroute2 import IPRSocket as IPRSocket0 +from pyroute2 import NetlinkDecodeError as NetlinkDecodeError0 +from pyroute2 import NetlinkError as NetlinkError0 +from pyroute2.iproute import IPRoute as IPRoute1 - -@pytest.mark.parametrize( - 'msg_type,dump_method,field,event', - [ - (RTM_GETLINK, 'link', 'ifname', 'RTM_NEWLINK'), - (RTM_GETADDR, 'addr', 'address', 'RTM_NEWADDR'), - ], +# nlmsg +# flags +from pyroute2.netlink import ( + NLM_F_DUMP, + NLM_F_MATCH, + NLM_F_REQUEST, + NLM_F_ROOT, + NLMSG_DONE, + NLMSG_ERROR, +) +from pyroute2.netlink import NetlinkDecodeError as NetlinkDecodeError1 +from pyroute2.netlink import NetlinkError as NetlinkError1 +from pyroute2.netlink import nlmsg +from pyroute2.netlink.rtnl import ( + RTM_DELADDR, + RTM_DELLINK, + RTM_GETADDR, + RTM_GETLINK, + RTM_NEWADDR, + RTM_NEWLINK, + RTMGRP_IPV4_IFADDR, + RTMGRP_IPV6_IFADDR, + RTMGRP_LINK, ) -def test_interface_manager_dump_link( - nsname, msg_type, dump_method, field, event -): - with IPRSocket(netns=nsname) as iprsock, IPRoute(netns=nsname) as ipr: - - # bring up loopback - ipr.link('set', index=1, state='up') - ipr.poll(ipr.addr, 'dump', address='127.0.0.1', timeout=1) - - # init dump: - # InterfaceManager.request_netlink_dump() - iprsock.put(None, msg_type, msg_flags=NLM_F_REQUEST | NLM_F_DUMP) - - # collect responses: - # InterfaceManager.pull_netlink_messages_into_queue() - ret = [] - while True: - rl, wl, xl = select.select([iprsock], [], [], 0) - if not len(rl): - break - ret.extend(iprsock.get()) - - links = [x for x in getattr(ipr, dump_method)('dump')] - ifnames_ipr = set([x.get(field) for x in links]) - ifnames_iprsock = set( - [x.get(field) for x in ret if x.get('event') == event] - ) - assert ifnames_iprsock == ifnames_ipr +from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg +from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg +from pyroute2.netlink.rtnl.iprsocket import IPRSocket as IPRSocket1 +from pyroute2.netlink.rtnl.rtmsg import rtmsg + + +def test_exceptions_compat(): + with pytest.raises(NetlinkError1): + raise NetlinkError1(code=99) + + with pytest.raises(NetlinkDecodeError1): + raise NetlinkDecodeError1(exception=Exception()) + + +def test_exceptions(): + with pytest.raises(NetlinkError0): + raise NetlinkError0(code=99) + + with pytest.raises(NetlinkDecodeError0): + raise NetlinkDecodeError0(exception=Exception()) + + +def test_constants(): + assert issubclass(ifinfmsg, nlmsg) + assert NLM_F_REQUEST == 1 + assert NLM_F_ROOT == 0x100 + assert NLM_F_MATCH == 0x200 + assert NLM_F_DUMP == (NLM_F_ROOT | NLM_F_MATCH) + assert NLMSG_DONE == 0x3 + assert NLMSG_ERROR == 0x2 + assert RTM_NEWLINK == 0x10 + assert RTM_DELLINK == 0x11 + assert RTM_GETLINK == 0x12 + assert RTM_NEWADDR == 0x14 + assert RTM_DELADDR == 0x15 + assert RTM_GETADDR == 0x16 + assert RTMGRP_LINK == 0x1 + assert RTMGRP_IPV4_IFADDR == 0x10 + assert RTMGRP_IPV6_IFADDR == 0x100 + + +@pytest.mark.parametrize('socket_class', (IPRSocket0, IPRSocket1)) +def test_basic(socket_class): + ip = socket_class() + ip.bind() + + # check the `socket` interface compliance + poll = select.poll() + poll.register(ip, select.POLLIN | select.POLLPRI) + poll.unregister(ip) + ip.close() + + +@pytest.mark.parametrize('iproute_class', (IPRoute0, IPRoute1)) +def test_iproute_message_classes(iproute_class): + with iproute_class() as ip: + assert {ifaddrmsg, ifinfmsg, rtmsg} < {type(x) for x in ip.dump()} + + +@pytest.mark.parametrize('iproute_class', (IPRoute0, IPRoute1)) +def test_iproute_message_subclass(iproute_class): + with iproute_class() as ip: + assert all([issubclass(type(x), nlmsg) for x in ip.dump()]) + + +@pytest.mark.parametrize('iprsocket_class', (IPRSocket0, IPRSocket1)) +def test_iprsocket_put(iprsocket_class): + NL_GROUPS = RTMGRP_IPV4_IFADDR | RTMGRP_IPV6_IFADDR | RTMGRP_LINK + with iprsocket_class() as iprs: + iprs.bind(groups=NL_GROUPS) + iprs.put(None, RTM_GETLINK, msg_flags=NLM_F_REQUEST | NLM_F_DUMP) diff --git a/tests/test_integration/test_octavia.py b/tests/test_integration/test_octavia.py deleted file mode 100644 index 91c1ce146..000000000 --- a/tests/test_integration/test_octavia.py +++ /dev/null @@ -1,98 +0,0 @@ -import errno -import os - -import pytest -from net_tools import interface_exists - -import pyroute2 -from pyroute2.common import uifname -from pyroute2.netlink.exceptions import NetlinkError -from pyroute2.netlink.rtnl import ifaddrmsg, rt_proto - - -def test_amphora_info(nsname): - with pyroute2.NetNS(netns=nsname) as ns: - for interface in ns.get_links(): - for item in interface['attrs']: - if item[0] == 'IFLA_IFNAME': - assert isinstance(item[1], str) - if item[0] == 'IFLA_STATS64': - assert isinstance(item[1]['tx_bytes'], int) - assert isinstance(item[1]['rx_bytes'], int) - - -def test_api_server_plug(nsname, test_link_ifname, test_link_address): - newname = uifname() - with pyroute2.IPRoute(netns=nsname) as ipr: - idx = ipr.link_lookup(address=test_link_address)[0] - ipr.link('set', index=idx, net_ns_fd=nsname, IFLA_IFNAME=newname) - assert not interface_exists( - ifname=test_link_ifname, netns=nsname, timeout=0.1 - ) - assert interface_exists(ifname=newname, netns=nsname) - - -def test_api_server_mac(nsname, test_link_address, test_link_ifname): - with pyroute2.IPRoute(netns=nsname) as ipr: - idx = ipr.link_lookup(address=test_link_address)[0] - addr = ipr.get_links(idx)[0] - for attr in addr['attrs']: - if attr[0] == 'IFLA_IFNAME': - assert attr[1] == test_link_ifname - - -def test_api_server_attrs( - test_link_ifinfmsg, test_link_address, test_link_ifname -): - attr_dict = dict(test_link_ifinfmsg['attrs']) - assert attr_dict.get('IFLA_ADDRESS') == test_link_address - assert attr_dict.get('IFLA_IFNAME') == test_link_ifname - - -def test_api_server_netns_flags(nsname): - with pyroute2.NetNS(nsname, flags=os.O_CREAT) as netns: - for link in netns.get_links(): - assert isinstance(link.get('address'), str) - assert isinstance(link.get('ifname'), str) - - -def test_utils_exception_ref(): - assert pyroute2.NetlinkError is NetlinkError - e = NetlinkError(errno.EINVAL, 'message') - assert e.code == errno.EINVAL - - -def test_utils_link_attr(test_link_ifname, sync_ipr): - idx = sync_ipr.link_lookup(ifname=test_link_ifname)[0] - ref = sync_ipr.get_links(idx)[0] - assert ref.get('state') == 'up' - sync_ipr.link('set', index=idx, state='down', mtu=1000) - sync_ipr.poll( - sync_ipr.link, 'dump', index=idx, state='down', mtu=1000, timeout=5 - ) - with pytest.raises(TimeoutError): - sync_ipr.poll( - sync_ipr.link, 'dump', index=idx, state='up', timeout=0.1 - ) - with pytest.raises(TimeoutError): - sync_ipr.poll(sync_ipr.link, 'dump', index=idx, mtu=1500, timeout=0.1) - - -def test_utils_addr_flags(test_link_index, test_link_ifname, sync_ipr, ndb): - with ndb.interfaces[test_link_ifname] as i: - i.set('state', 'up') - i.add_ip('10.1.2.3/24') - - for addr in sync_ipr.get_addr(index=test_link_index): - attrs = dict(addr['attrs']) - if attrs['IFA_FLAGS'] & ifaddrmsg.IFA_F_PERMANENT: - break - else: - raise NetlinkError(errno.ENOENT, 'no static addresses') - - -def test_utils_route_proto(ndb, sync_ipr): - with ndb.interfaces['lo'] as i: - i.set('state', 'up') - for route in sync_ipr.get_routes(oif=1): - assert route['proto'] != rt_proto['static'] diff --git a/tests/test_integration/test_os_vif.py b/tests/test_integration/test_os_vif.py deleted file mode 100644 index fe68a7830..000000000 --- a/tests/test_integration/test_os_vif.py +++ /dev/null @@ -1,42 +0,0 @@ -from pyroute2.common import uifname - - -def test_impl_lookup(sync_ipr, test_link_ifname): - assert len(sync_ipr.link_lookup(ifname=test_link_ifname)) == 1 - - -def test_impl_add_bridge(sync_ipr): - brname = uifname() - args = { - 'ifname': brname, - 'kind': 'bridge', - 'IFLA_BR_FORWARD_DELAY': 0, - 'IFLA_BR_STP_STATE': 0, - 'IFLA_BR_MCAST_SNOOPING': 0, - 'IFLA_BR_AGEING_TIME': 1500, - } - sync_ipr.link('add', **args) - link = [ - x - for x in sync_ipr.poll(sync_ipr.link, 'dump', ifname=brname, timeout=5) - ][0] - assert link.get(('linkinfo', 'data', 'br_forward_delay')) == 0 - assert link.get(('linkinfo', 'data', 'br_stp_state')) == 0 - assert link.get(('linkinfo', 'data', 'br_mcast_snooping')) == 0 - assert link.get(('linkinfo', 'data', 'br_ageing_time')) == 1500 - - -def test_impl_add_vlan(sync_ipr, test_link_index): - vname = uifname() - args = { - 'ifname': vname, - 'kind': 'vlan', - 'vlan_id': 1001, - 'link': test_link_index, - } - sync_ipr.link('add', **args) - link = [ - x - for x in sync_ipr.poll(sync_ipr.link, 'dump', ifname=vname, timeout=5) - ][0] - assert link.get(('linkinfo', 'data', 'vlan_id')) == 1001 diff --git a/tests/test_integration/test_ovn_bgp_agent.py b/tests/test_integration/test_ovn_bgp_agent.py deleted file mode 100644 index f2c8c7924..000000000 --- a/tests/test_integration/test_ovn_bgp_agent.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest - -from pyroute2 import iproute -from pyroute2.common import uifname - - -@pytest.mark.parametrize( - 'method,argv', - (('get_rules', []), ('route', ['show']), ('brport', ['show'])), -) -def test_iproute_call(method, argv): - with iproute.IPRoute() as ip: - iter(getattr(ip, method)(*argv)) - - -@pytest.mark.parametrize( - 'param,value', (('neigh_suppress', True), ('learning', False)) -) -def test_brport_set( - sync_ipr, ndb, test_link_index, test_link_ifname, param, value -): - brname = uifname() - ndb.interfaces.create(ifname=brname, kind='bridge').add_port( - test_link_ifname - ).commit() - kwarg = {param: value} - sync_ipr.brport('set', index=test_link_index, **kwarg) - sync_ipr.poll( - sync_ipr.brport, 'dump', index=test_link_index, timeout=5, **kwarg - ) diff --git a/tests/test_lab/conftest.py b/tests/test_lab/conftest.py new file mode 100644 index 000000000..694d7d58d --- /dev/null +++ b/tests/test_lab/conftest.py @@ -0,0 +1 @@ +pytest_plugins = "pytester" diff --git a/tests/test_lab/test_code_blocks.py b/tests/test_lab/test_code_blocks.py new file mode 100644 index 000000000..03aa132d2 --- /dev/null +++ b/tests/test_lab/test_code_blocks.py @@ -0,0 +1,26 @@ +import os +import pathlib +import sys + +import pytest + + +def get_examples(*argv): + root = pathlib.Path(os.environ['WORKSPACE']) + examples = [ + example + for example in root.joinpath(*argv).iterdir() + if example.is_dir() + ] + return { + 'argnames': 'example', + 'argvalues': examples, + 'ids': [x.name for x in examples], + } + + +@pytest.mark.parametrize(**get_examples('examples', 'lab')) +def test_block(example, pytester): + os.chdir(example.as_posix()) + result = pytester.run(sys.executable, 'check.py') + assert result.ret == 0 diff --git a/tests/test_limits/conftest.py b/tests/test_limits/conftest.py deleted file mode 100644 index 10dc93a9a..000000000 --- a/tests/test_limits/conftest.py +++ /dev/null @@ -1 +0,0 @@ -pytest_plugins = ['pyroute2.fixtures.iproute', 'pyroute2.fixtures.ndb'] diff --git a/tests/test_limits/test_nl.py b/tests/test_limits/test_nl.py index 748fae387..fcf091980 100644 --- a/tests/test_limits/test_nl.py +++ b/tests/test_limits/test_nl.py @@ -1,15 +1,12 @@ -import errno import resource import socket import pytest -from pyroute2 import AsyncIPRoute, IPRoute -from pyroute2.common import uifname from pyroute2.netlink.nlsocket import NetlinkSocket -def _test_ports_auto(): +def test_ports_auto(): # create two sockets s1 = NetlinkSocket() s2 = NetlinkSocket() @@ -25,7 +22,7 @@ def _test_ports_auto(): s2.close() -def _test_ports_fail(): +def test_ports_fail(): s1 = NetlinkSocket(port=0x10) s2 = NetlinkSocket(port=0x10) @@ -49,82 +46,24 @@ def _test_ports_fail(): def test_no_free_ports(): soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) try: - resource.setrlimit(resource.RLIMIT_NOFILE, (10384, 10384)) + resource.setrlimit(resource.RLIMIT_NOFILE, (4096, 4096)) except ValueError: pytest.skip('cannot set RLIMIT_NOFILE') # create and bind 1024 sockets - ports = [NetlinkSocket() for x in range(1025)] - counter = 0 - with pytest.raises(KeyError): - for port in ports: - port.bind() - counter += 1 + ports = [NetlinkSocket() for x in range(1024)] + for port in ports: + port.bind() - assert 0 < counter <= 180 - assert 20 < ports[counter].status['port'] < 200 + # create an extra socket + fail = NetlinkSocket() + # bind must fail with KeyError: no free ports available + with pytest.raises(KeyError): + fail.bind() # cleanup for port in ports: port.close() + fail.close() resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) - - -@pytest.mark.asyncio -async def test_enobufs_async(async_ipr, nsname): - ifname: str = uifname() - ifaddr: str = '172.16.3.2' - dst: str = '10.3.4.0' - gateway: str = '172.16.3.10' - # Create a socket and start receiving broadcasts - ipr = AsyncIPRoute(netns=nsname, rcvbuf=1024) - await ipr.bind() - # Make some netlink trafic - await async_ipr.link('add', ifname=ifname, kind='dummy', state='up') - (link,) = await async_ipr.poll( - async_ipr.link, 'dump', ifname=ifname, state='up', timeout=2 - ) - await async_ipr.addr( - 'add', index=link.get('index'), address=f'{ifaddr}/24' - ) - (addr,) = await async_ipr.poll( - async_ipr.addr, 'dump', address=f'{ifaddr}', timeout=2 - ) - await async_ipr.route('add', dst=f'{dst}/24', gateway=f'{gateway}') - (route,) = await async_ipr.poll( - async_ipr.route, 'dump', dst=f'{dst}', timeout=2 - ) - assert route.get('oif') == link.get('index') - await async_ipr.link('del', index=link.get('index')) - - with pytest.raises(OSError) as e: - [x async for x in ipr.get()] - assert e.value.errno == errno.ENOBUFS - - -def test_enobufs_sync(sync_ipr, nsname): - ifname: str = uifname() - ifaddr: str = '172.16.3.2' - dst: str = '10.3.4.0' - gateway: str = '172.16.3.10' - # Create a socket and start receiving broadcasts - ipr = IPRoute(netns=nsname, rcvbuf=1024) - ipr.bind() - # Make some netlink trafic - sync_ipr.link('add', ifname=ifname, kind='dummy', state='up') - (link,) = sync_ipr.poll( - sync_ipr.link, 'dump', ifname=ifname, state='up', timeout=2 - ) - sync_ipr.addr('add', index=link.get('index'), address=f'{ifaddr}/24') - (addr,) = sync_ipr.poll( - sync_ipr.addr, 'dump', address=f'{ifaddr}', timeout=2 - ) - sync_ipr.route('add', dst=f'{dst}/24', gateway=f'{gateway}') - (route,) = sync_ipr.poll(sync_ipr.route, 'dump', dst=f'{dst}', timeout=2) - assert route.get('oif') == link.get('index') - sync_ipr.link('del', index=link.get('index')) - - with pytest.raises(OSError) as e: - ipr.get() - assert e.value.errno == errno.ENOBUFS diff --git a/tests/test_limits/test_stress.py b/tests/test_limits/test_stress.py index 5718aa652..5d3a823ef 100644 --- a/tests/test_limits/test_stress.py +++ b/tests/test_limits/test_stress.py @@ -1,3 +1,4 @@ +import atexit import errno import gc import getpass @@ -6,32 +7,18 @@ import pytest -from pyroute2 import AsyncIPRoute, IPRoute, netns +from pyroute2 import NDB, IPRoute, NetNS +from pyroute2.common import uifname -RESPAWNS = 1024 -USAGE_RSS = set() +RESPAWNS = 200 pytestmark = [ pytest.mark.skipif(getpass.getuser() != 'root', reason='no root access') ] -def reset_rss_usage(): - global USAGE_RSS - USAGE_RSS = set() - - -def max_rss_usage(): - global USAGE_RSS - usage = resource.getrusage(resource.RUSAGE_SELF) - USAGE_RSS.add(usage.ru_maxrss // 10) - return (max(USAGE_RSS) - min(USAGE_RSS)) == 0 - - @pytest.fixture -def resources(): - # current file limits +def fds(): soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - # current usage try: resource.setrlimit( resource.RLIMIT_NOFILE, @@ -43,52 +30,73 @@ def resources(): yield fds_before gc.collect() fds_after = os.listdir(f'/proc/{os.getpid()}/fd/') - # restore limits - resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) assert len(fds_after) <= len(fds_before) -def test_respawn_iproute_sync(resources): - reset_rss_usage() +def test_respawn_iproute_sync(fds): for _ in range(RESPAWNS): with IPRoute() as i: i.bind() i.link_lookup(ifname='lo') - assert max_rss_usage() - gc.collect() -@pytest.mark.asyncio -async def test_respawn_iproute_async(resources): - reset_rss_usage() +def test_respawn_iproute_async(fds): for _ in range(RESPAWNS): - async with AsyncIPRoute() as i: - await i.bind() - await i.link_lookup(ifname='lo') - assert max_rss_usage() - gc.collect() + with IPRoute() as i: + i.bind(async_cache=True) + i.link_lookup(ifname='lo') -def test_fd_leaks_netns(resources): - reset_rss_usage() +def test_respawn_ndb(fds): + for _ in range(RESPAWNS): + with NDB() as i: + assert i.interfaces.count() > 0 + assert i.addresses.count() > 0 + assert i.routes.count() > 0 + assert i.neighbours.count() > 0 + + +def test_bridge_fd_leaks(fds): + ifs = [] + for _ in range(RESPAWNS): + ifs.append(uifname()) + with NDB() as ndb: + for name in ifs: + ndb.interfaces.create(ifname=name, kind='bridge').apply() + with NDB() as ndb: + for name in ifs: + ndb.interfaces[name].remove().apply() + + +def test_tuntap_fd_leaks(fds): + ifs = [] + for _ in range(RESPAWNS): + ifs.append(uifname()) + with NDB() as ndb: + for name in ifs: + ndb.interfaces.create( + ifname=name, kind='tuntap', mode='tun' + ).apply() + with NDB() as ndb: + for name in ifs: + ndb.interfaces[name].remove().apply() + + +def test_fd_leaks(fds): for i in range(RESPAWNS): nsid = 'leak_%i' % i - ipr = IPRoute(netns=nsid) - ipr.link_lookup(ifname='lo') - ipr.close() - assert max_rss_usage() - gc.collect() - netns.remove(nsid) + ns = NetNS(nsid) + ns.close() + ns.remove() + if hasattr(atexit, '_exithandlers'): + assert ns.close not in atexit._exithandlers -def test_fd_leaks_netns_enoent(resources): - reset_rss_usage() +def test_fd_leaks_nonexistent_ns(fds): for i in range(RESPAWNS): nsid = 'non_existent_leak_%i' % i try: - with IPRoute(netns=nsid, flags=0): + with NetNS(nsid, flags=0): pass except OSError as e: - assert e.errno == errno.ENOENT - assert max_rss_usage() - gc.collect() + assert e.errno in (errno.ENOENT, errno.EPIPE) diff --git a/tests/test_linux/conftest.py b/tests/test_linux/conftest.py index 1f379d9fd..df00a9c68 100644 --- a/tests/test_linux/conftest.py +++ b/tests/test_linux/conftest.py @@ -2,12 +2,6 @@ from uuid import uuid4 import pytest -from fixtures.dhcp_servers.dnsmasq import dnsmasq, dnsmasq_config # noqa: F401 -from fixtures.dhcp_servers.mock import mock_dhcp_server # noqa: F401 -from fixtures.dhcp_servers.udhcpd import udhcpd, udhcpd_config # noqa: F401 -from fixtures.interfaces import dhcp_range, veth_pair # noqa: F401 -from fixtures.pcap_files import pcap # noqa: F401 -from fixtures.rt_file import CreateRtFile # noqa: F401 from pr2test.context_manager import NDBContextManager, SpecContextManager from utils import require_user @@ -16,11 +10,7 @@ from pyroute2.wiset import COUNT config.nlm_generator = True -pytest_plugins = [ - 'pyroute2.fixtures.iproute', - 'pyroute2.fixtures.ndb', - 'pytester', -] +pytest_plugins = "pytester" @pytest.fixture @@ -72,15 +62,7 @@ def wiset_sock(request): if request.param is None: yield None else: - before_count = COUNT['count'] + before_count = COUNT["count"] with IPSet() as sock: yield sock assert before_count == COUNT['count'] - - -@pytest.fixture -def fake_rt_file(tmpdir): - ''' - A simple fixture with only some variables set - ''' - yield CreateRtFile(tmpdir) diff --git a/tests/test_linux/fixtures/dhcp_servers/__init__.py b/tests/test_linux/fixtures/dhcp_servers/__init__.py deleted file mode 100644 index daa78226d..000000000 --- a/tests/test_linux/fixtures/dhcp_servers/__init__.py +++ /dev/null @@ -1,162 +0,0 @@ -import abc -import asyncio -from argparse import ArgumentParser -from collections import defaultdict -from dataclasses import dataclass -from ipaddress import IPv4Address -from typing import ClassVar, DefaultDict, Generic, Literal, Optional, TypeVar - -import pytest - -from ..interfaces import DHCPRangeConfig - - -@pytest.fixture -def lease_time() -> int: - '''Configures the lease time used by dhcp servers. - - Can be overridden in individual tests with - `@pytest.mark.parametrize('lease_time', [5])` - ''' - return 120 - - -@dataclass -class DHCPServerConfig: - range: DHCPRangeConfig - interface: str - lease_time: int = 120 # in seconds - max_leases: int = 50 - - -DHCPServerConfigT = TypeVar("DHCPServerConfigT", bound=DHCPServerConfig) - - -class DHCPServerFixture(abc.ABC, Generic[DHCPServerConfigT]): - - BINARY_PATH: ClassVar[Optional[str]] = None - - @classmethod - def get_config_class(cls) -> type[DHCPServerConfigT]: - return cls.__orig_bases__[0].__args__[0] - - def __init__(self, config: DHCPServerConfigT) -> None: - self.config = config - self.stdout: list[str] = [] - self.stderr: list[str] = [] - self.process: Optional[asyncio.subprocess.Process] = None - self.output_poller: Optional[asyncio.Task] = None - self.expected_logs: DefaultDict[str, asyncio.Event] = defaultdict( - asyncio.Event - ) - - async def _read_output(self, name: Literal['stdout', 'stderr']): - '''Read stdout or stderr until the process exits.''' - stream: asyncio.StreamReader = getattr(self.process, name) - output: list[str] = getattr(self, name) - while line := await stream.readline(): - line = line.decode().strip() - # Trigger events for any log substring we're looking for - # that will wake up `wait_for_log` - for sublog in filter(line.__contains__, self.expected_logs): - self.expected_logs[sublog].set() - output.append(line) - - async def _read_outputs(self): - '''Read stdout & stderr until the process exits.''' - assert self.process - await asyncio.gather( - self._read_output('stderr'), self._read_output('stdout') - ) - - @abc.abstractmethod - def get_cmdline_options(self) -> tuple[str]: - '''All commandline options passed to the server.''' - - async def __aenter__(self): - '''Start the server process and start polling its output.''' - if not self.BINARY_PATH: - raise RuntimeError( - f"server binary is missing for {type(self).__name__}" - ) - self.process = await asyncio.create_subprocess_exec( - self.BINARY_PATH, - *self.get_cmdline_options(), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env={'LANG': 'C'}, # usually ensures the output is in english - ) - self.output_poller = asyncio.Task(self._read_outputs()) - return self - - async def __aexit__(self, *_): - if self.process: - if self.process.returncode is None: - self.process.terminate() - await self.process.wait() - await self.output_poller - - async def wait_for_log(self, substr: str): - '''Wait for a string to appear in logs, then return.''' - await self.expected_logs[substr].wait() - # wait a tiny bit more so the client has time to react - await asyncio.sleep(0.1) - - -def get_psr() -> ArgumentParser: - psr = ArgumentParser() - psr.add_argument('interface', help='Interface to listen on') - psr.add_argument( - '--router', type=IPv4Address, default=None, help='Router IPv4 address.' - ) - psr.add_argument( - '--range-start', - type=IPv4Address, - default=IPv4Address('192.168.186.10'), - help='Start of the DHCP client range.', - ) - psr.add_argument( - '--range-end', - type=IPv4Address, - default=IPv4Address('192.168.186.100'), - help='End of the DHCP client range.', - ) - psr.add_argument( - '--lease-time', - default=120, - type=int, - help='DHCP lease time in seconds (minimum 2 minutes)', - ) - psr.add_argument( - '--netmask', type=IPv4Address, default=IPv4Address("255.255.255.0") - ) - psr.add_argument( - '--broadcast', type=IPv4Address, default=IPv4Address('192.168.186.255') - ) - return psr - - -async def run_fixture_as_main(fixture_cls: type[DHCPServerFixture]): - config_cls = fixture_cls.get_config_class() - args = get_psr().parse_args() - range_config = DHCPRangeConfig( - start=args.range_start, - end=args.range_end, - router=args.router, - netmask=args.netmask, - broadcast=args.broadcast, - ) - conf = config_cls( - range=range_config, - interface=args.interface, - lease_time=args.lease_time, - ) - read_lines: int = 0 - async with fixture_cls(conf) as dhcp_server: - # quick & dirty stderr polling - while True: - if len(dhcp_server.stderr) > read_lines: - read_lines += len(lines := dhcp_server.stderr[read_lines:]) - print(*lines, sep='\n') - else: - await asyncio.sleep(0.2) diff --git a/tests/test_linux/fixtures/dhcp_servers/dnsmasq.py b/tests/test_linux/fixtures/dhcp_servers/dnsmasq.py deleted file mode 100644 index d52c73ae1..000000000 --- a/tests/test_linux/fixtures/dhcp_servers/dnsmasq.py +++ /dev/null @@ -1,79 +0,0 @@ -import asyncio -from dataclasses import dataclass -from shutil import which -from typing import AsyncGenerator, ClassVar, Optional - -import pytest -import pytest_asyncio -from fixtures.interfaces import DHCPRangeConfig - -from . import DHCPServerConfig, DHCPServerFixture, run_fixture_as_main - - -@dataclass -class DnsmasqConfig(DHCPServerConfig): - '''Options for the dnsmasq server.''' - - # Respond to REQUESTs even after RELEASEs - # TODO: test both cases - authoritative: bool = True - - def __iter__(self): - lease_time = 'infinite' if self.lease_time == -1 else self.lease_time - opts = [ - f'--interface={self.interface}', - f'--dhcp-range={self.range.start},' - f'{self.range.end},{lease_time}', - f'--dhcp-lease-max={self.max_leases}', - ] - if self.authoritative: - opts.append('--dhcp-authoritative') - if router := self.range.router: - opts.append(f'--dhcp-option=option:router,{router}') - return iter(opts) - - -class DnsmasqFixture(DHCPServerFixture[DnsmasqConfig]): - '''Runs the dnsmasq server as an async context manager.''' - - BINARY_PATH: ClassVar[Optional[str]] = which('dnsmasq') - - def _get_base_cmdline_options(self) -> tuple[str]: - '''The base commandline options for dnsmasq.''' - return ( - '--no-daemon', # keep in foreground - '--no-resolv', # don't mess w/ resolv.conf - '--log-facility=-', # log to stdout - '--no-hosts', # don't read /etc/hosts - '--bind-interfaces', # don't bind on wildcard - '--no-ping', # don't ping to check if ips are attributed - '--log-dhcp', - '--log-debug', - ) - - def get_cmdline_options(self) -> tuple[str]: - '''All commandline options passed to dnsmasq.''' - return (*self._get_base_cmdline_options(), *self.config) - - -@pytest.fixture -def dnsmasq_config( - veth_pair: tuple[str, str], dhcp_range: DHCPRangeConfig, lease_time: int -) -> DnsmasqConfig: - '''dnsmasq options useful for test purposes.''' - return DnsmasqConfig( - range=dhcp_range, interface=veth_pair[0], lease_time=lease_time - ) - - -@pytest_asyncio.fixture -async def dnsmasq( - dnsmasq_config: DnsmasqConfig, -) -> AsyncGenerator[DnsmasqFixture, None]: - '''A dnsmasq instance running for the duration of the test.''' - async with DnsmasqFixture(config=dnsmasq_config) as dnsf: - yield dnsf - - -if __name__ == '__main__': - asyncio.run(run_fixture_as_main(DnsmasqFixture)) diff --git a/tests/test_linux/fixtures/dhcp_servers/mock.py b/tests/test_linux/fixtures/dhcp_servers/mock.py deleted file mode 100644 index dad9ddf7b..000000000 --- a/tests/test_linux/fixtures/dhcp_servers/mock.py +++ /dev/null @@ -1,61 +0,0 @@ -import asyncio - -import pytest -from fixtures.pcap_files import PcapFile - -from pyroute2.dhcp.dhcp4socket import AsyncDHCP4Socket -from pyroute2.dhcp.messages import SentDHCPMessage - - -class MockDHCPServerFixture: - '''A fixture that's used to avoid a real dhcp server in unit tests. - - By replacing `AsyncDHCP4Socket.loop` with an instance of this class, - it will wait for messages sent by `AsyncDHCPClient` and answer - with data from `responses`, in order. - - The requests made by the client will be stored in `decoded_requests`. - ''' - - def __init__(self, responses: list[bytes]): - self.responses: list[bytes] = responses - self.requests: list[bytes] = [] - self.decoded_requests: list[SentDHCPMessage] = [] - # save the event_loop fixture, 'cause pytest-asyncio on Python < 3.10 - # drops the loop in the middle of the test - self.loop = asyncio.new_event_loop() - self._request_received = asyncio.Event() - self.truncate_at: int = 0 - - async def sock_sendall(self, sock, data: bytes): - self.requests.append(data) - self.decoded_requests.append(AsyncDHCP4Socket._decode_msg(data)) - self._request_received.set() - - async def sock_recv(self, sock, size: int) -> bytes: - # wait for a request to be received to send a response - await self._request_received.wait() - self._request_received.clear() - if self.responses: - data = self.responses.pop(0) - if self.truncate_at: - return data[: self.truncate_at] - return data - # make the client timeout, the server is supposed to answer nothing - await asyncio.sleep(9999) - - -@pytest.fixture -def mock_dhcp_server( - pcap: PcapFile, monkeypatch: pytest.MonkeyPatch -) -> MockDHCPServerFixture: - '''Monkey patches the client to respond to requests with pcap data. - - The `pcap` fixture is used which means the pcap file must be named - after the test. - ''' - responder = MockDHCPServerFixture(responses=pcap) - monkeypatch.setattr( - 'pyroute2.dhcp.dhcp4socket.AsyncDHCP4Socket.loop', responder - ) - return responder diff --git a/tests/test_linux/fixtures/dhcp_servers/udhcpd.py b/tests/test_linux/fixtures/dhcp_servers/udhcpd.py deleted file mode 100644 index 4ba598c54..000000000 --- a/tests/test_linux/fixtures/dhcp_servers/udhcpd.py +++ /dev/null @@ -1,101 +0,0 @@ -import asyncio -from dataclasses import dataclass -from pathlib import Path -from shutil import which -from tempfile import TemporaryDirectory -from typing import AsyncGenerator, ClassVar, Optional - -import pytest -import pytest_asyncio - -from ..interfaces import DHCPRangeConfig -from . import DHCPServerConfig, DHCPServerFixture, run_fixture_as_main - - -@dataclass -class UdhcpdConfig(DHCPServerConfig): - arp_ping_timeout_ms: int = 200 # default is 2000 - - -class UdhcpdFixture(DHCPServerFixture[UdhcpdConfig]): - '''Runs the udhcpd server as an async context manager.''' - - BINARY_PATH: ClassVar[Optional[str]] = which('busybox') - - def __init__(self, config): - super().__init__(config) - self._temp_dir: Optional[TemporaryDirectory[str]] = None - - @property - def workdir(self) -> Path: - '''A temporary directory for udhcpd's files.''' - assert self._temp_dir - return Path(self._temp_dir.name) - - @property - def config_file(self) -> Path: - '''The udhcpd config file path.''' - return self.workdir.joinpath("udhcpd.conf") - - async def __aenter__(self): - self._temp_dir = TemporaryDirectory(prefix=type(self).__name__) - self._temp_dir.__enter__() - self.config_file.write_text(self.generate_config()) - return await super().__aenter__() - - def generate_config(self) -> str: - '''Generate the contents of udhcpd's config file.''' - cfg = self.config - base_workfile = self.workdir.joinpath(self.config.interface) - lease_file = base_workfile.with_suffix(".leases") - pidfile = base_workfile.with_suffix(".pid") - lines = [ - ("start", cfg.range.start), - ("end", cfg.range.end), - ("max_leases", cfg.max_leases), - ("interface", cfg.interface), - ("lease_file", lease_file), - ("pidfile", pidfile), - ("opt lease", cfg.lease_time), - ("opt subnet", cfg.range.netmask), - ] - if router := cfg.range.router: - lines.append(("opt router", router)) - return "\n".join(f"{opt}\t{value}" for opt, value in lines) - - async def __aexit__(self, *_): - await super().__aexit__(*_) - self._temp_dir.__exit__(*_) - - def get_cmdline_options(self) -> tuple[str]: - '''All commandline options passed to udhcpd.''' - return ( - 'udhcpd', - '-f', # run in foreground - '-a', - str(self.config.arp_ping_timeout_ms), - str(self.config_file), - ) - - -@pytest.fixture -def udhcpd_config( - veth_pair: tuple[str, str], dhcp_range: DHCPRangeConfig, lease_time: int -) -> UdhcpdConfig: - '''udhcpd options useful for test purposes.''' - return UdhcpdConfig( - range=dhcp_range, interface=veth_pair[0], lease_time=lease_time - ) - - -@pytest_asyncio.fixture -async def udhcpd( - udhcpd_config: UdhcpdConfig, -) -> AsyncGenerator[UdhcpdFixture, None]: - '''An udhcpd instance running for the duration of the test.''' - async with UdhcpdFixture(config=udhcpd_config) as dhcp_server: - yield dhcp_server - - -if __name__ == '__main__': - asyncio.run(run_fixture_as_main(UdhcpdFixture)) diff --git a/tests/test_linux/fixtures/interfaces.py b/tests/test_linux/fixtures/interfaces.py deleted file mode 100644 index fa91277c9..000000000 --- a/tests/test_linux/fixtures/interfaces.py +++ /dev/null @@ -1,85 +0,0 @@ -import random -from ipaddress import IPv4Address -from pathlib import Path -from typing import AsyncGenerator, NamedTuple - -import pytest -import pytest_asyncio - -from pyroute2.common import uifname -from pyroute2.fixtures.iproute import SetNSContext, TestContext -from pyroute2.iproute.linux import AsyncIPRoute - - -class DHCPRangeConfig(NamedTuple): - start: IPv4Address - end: IPv4Address - router: IPv4Address - broadcast: IPv4Address - netmask: IPv4Address - - -@pytest.fixture -def dhcp_range() -> DHCPRangeConfig: - '''An IPv4 DHCP range configuration.''' - rangeidx = random.randint(1, 254) - return DHCPRangeConfig( - start=IPv4Address(f'10.{rangeidx}.0.10'), - end=IPv4Address(f'10.{rangeidx}.0.20'), - router=IPv4Address(f'10.{rangeidx}.0.1'), - broadcast=IPv4Address(f'10.{rangeidx}.0.255'), - netmask=IPv4Address('255.255.255.0'), - ) - - -class VethPair(NamedTuple): - '''A pair of veth interfaces.''' - - server: str - client: str - server_idx: int - client_idx: int - - -def accept_local(ifname: str): - '''Turn on the accept_local sysctl for the given interface. - - This seems to be required for arp to work on veth pairs. - ''' - Path('/proc/sys/net/ipv4/conf', ifname, 'accept_local').write_text('1') - - -@pytest_asyncio.fixture -async def veth_pair( - dhcp_range: DHCPRangeConfig, - async_context: TestContext[AsyncIPRoute], - setns_context: SetNSContext, -) -> AsyncGenerator[VethPair, None]: - '''Fixture that creates & removes a temporary veth pair.''' - base_ifname = uifname() - server_ifname = f'{base_ifname}-srv' - client_ifname = f'{base_ifname}-cli' - try: - await async_context.ipr.link( - 'add', ifname=server_ifname, kind="veth", peer=client_ifname - ) - srv_id = (await async_context.ipr.link_lookup(ifname=server_ifname))[0] - cli_id = (await async_context.ipr.link_lookup(ifname=client_ifname))[0] - await async_context.ipr.addr( - 'add', - index=srv_id, - address=str(dhcp_range.router), - prefixlen=24, # FIXME - ) - await async_context.ipr.link("set", index=srv_id, state="up") - await async_context.ipr.link("set", index=cli_id, state="up") - accept_local(server_ifname) - accept_local(client_ifname) - yield VethPair( - server=server_ifname, - client=client_ifname, - server_idx=srv_id, - client_idx=cli_id, - ) - finally: - await async_context.ipr.link("del", index=srv_id) diff --git a/tests/test_linux/fixtures/pcap_files.py b/tests/test_linux/fixtures/pcap_files.py deleted file mode 100644 index 63981962e..000000000 --- a/tests/test_linux/fixtures/pcap_files.py +++ /dev/null @@ -1,79 +0,0 @@ -from collections import UserList -from io import BufferedReader -from pathlib import Path -from struct import Struct -from typing import Iterator, NamedTuple - -import pytest - -FILE_HEADER_FORMAT = Struct('IHHiIII') -PKT_HEADER_FORMAT = Struct('IIII') - - -class PcapFileHeader(NamedTuple): - '''The header in front of pcap files.''' - - magic_number: int - version_major: int - version_minor: int - thiszone: int - sigfigs: int - snaplen: int - network: int - - -class PcapPacketHeader(NamedTuple): - '''The header in front of each packet in a pcap.''' - - ts_sec: int - ts_usec: int - incl_len: int - orig_len: int - - -class PcapFile(UserList): - '''Reads & stores raw packets from a pcap file.''' - - def __init__(self, filename: Path): - self.filename = filename - with self.filename.open('rb') as fp: - super().__init__(self._parse_pcap(fp)) - - @classmethod - def _parse_pcap(cls, fp: BufferedReader) -> Iterator[bytes]: - '''Read & yield the raw data for every packet in `fp`.''' - header_data = fp.read(FILE_HEADER_FORMAT.size) - cls._validate_file_header(header_data) - pkt_hdr_size = PKT_HEADER_FORMAT.size - while next_header := fp.read(pkt_hdr_size): - pkt_size = cls._parse_pkt_header(next_header) - pkt_data = fp.read(pkt_size) - assert len(pkt_data) == pkt_size, 'truncated packet' - yield pkt_data - - @classmethod - def _validate_file_header(cls, data: bytes): - '''Parse & check the pcap file header.''' - header = PcapFileHeader(*FILE_HEADER_FORMAT.unpack(data)) - # only support v2.4 big endian pcaps, this is a test fixture after all - assert header.magic_number == 0xA1B2C3D4 - assert (header.version_major, header.version_minor) == (2, 4) - assert header.network == 1 # ethernet - - @classmethod - def _parse_pkt_header(cls, data: bytes) -> int: - '''Read the pcap header for a single packet and return its length.''' - header = PcapPacketHeader(*PKT_HEADER_FORMAT.unpack(data)) - assert header.incl_len == header.orig_len - return header.incl_len - - -@pytest.fixture -def pcap(request: pytest.FixtureRequest) -> PcapFile: - '''Fixture that loads a pcap file named after the test.''' - pcap_path = request.path.parent.joinpath( - 'captures', - request.path.stem, # test file name without the extension - request.node.originalname, # name of the test - ).with_suffix('.pcap') - return PcapFile(pcap_path) diff --git a/tests/test_linux/fixtures/rt_file.py b/tests/test_linux/fixtures/rt_file.py deleted file mode 100644 index ae5dcd501..000000000 --- a/tests/test_linux/fixtures/rt_file.py +++ /dev/null @@ -1,15 +0,0 @@ -from pathlib import Path - -from pyroute2.netlink import rt_files - - -class CreateRtFile: - - def __init__(self, tmpdir): - self.tmpdir = tmpdir - rt_files.IPRouteRtFile.DIRECTORIES = [Path(self.tmpdir)] - - def create(self, rt_file, file_as_dict): - with (self.tmpdir / rt_file.get_rt_filename()).open("w+") as fp: - for key, value in file_as_dict.items(): - fp.write(f"{key} {value}\n") diff --git a/tests/test_linux/pr2test/context_manager.py b/tests/test_linux/pr2test/context_manager.py index 400c4482c..8e16dea32 100644 --- a/tests/test_linux/pr2test/context_manager.py +++ b/tests/test_linux/pr2test/context_manager.py @@ -4,6 +4,7 @@ import itertools import logging import os +import sys import uuid from collections import namedtuple from socket import AF_INET, AF_INET6 @@ -11,10 +12,13 @@ import pytest from utils import allocate_network, free_network -from pyroute2 import NDB, IPRoute, NetNS, netns +from pyroute2 import netns from pyroute2.common import basestring, uifname +from pyroute2.iproute.linux import IPRoute +from pyroute2.ndb.main import NDB from pyroute2.netlink.exceptions import NetlinkError from pyroute2.netlink.generic.wireguard import WireGuard +from pyroute2.nslink.nslink import NetNS def skip_if_not_implemented(func): @@ -175,10 +179,7 @@ def __init__(self, request, tmpdir, **kwarg): # in utility methods self.db_provider = kwarg['db_provider'] self.ndb = NDB(**kwarg) - if self.netns is None: - self.ipr = IPRoute() - else: - self.ipr = IPRoute(netns=self.netns) + self.ipr = self.ndb.sources['localhost'].nl.clone() self.wg = WireGuard() # # IPAM @@ -312,8 +313,8 @@ def teardown(self): 2. remove the registered interfaces, ignore not existing ''' # save postmortem DB for SQLite3 - if self.db_provider == 'sqlite3': - self.ndb.db.backup(f'{self.spec.uid}-post.db') + if self.db_provider == 'sqlite3' and sys.version_info >= (3, 7): + self.ndb.backup(f'{self.spec.uid}-post.db') self.ndb.close() self.ipr.close() self.wg.close() diff --git a/tests/test_linux/pr2test/tools.py b/tests/test_linux/pr2test/tools.py index 148fdcee9..53a5b7369 100644 --- a/tests/test_linux/pr2test/tools.py +++ b/tests/test_linux/pr2test/tools.py @@ -1,4 +1,5 @@ -from pyroute2 import IPRoute, NetNS +from pyroute2.iproute.linux import IPRoute +from pyroute2.nslink.nslink import NetNS def interface_exists(netns=None, *argv, **kwarg): diff --git a/tests/test_linux/test_devlink.py b/tests/test_linux/test_devlink.py index ee0473159..9043d1e81 100644 --- a/tests/test_linux/test_devlink.py +++ b/tests/test_linux/test_devlink.py @@ -1,37 +1,10 @@ -import errno - -import pytest from pr2test.context_manager import skip_if_not_supported from pyroute2 import DL -@pytest.fixture -def netdevsim(): - # Create a netdevsim device and cleanup upon exit, otherwise - # even an unused netdevsim affects RTNL tests - # - own_device = False - try: - # Trying to create an existing netdevsim device will - # result in errno.ENOSPC, then just ignore it: we only - # need a device to list - with open('/sys/bus/netdevsim/new_device', 'w') as f: - f.write('1 0') - own_device = True - except OSError as e: - if e.errno != errno.ENOSPC: - raise - try: - yield - finally: - if own_device: - with open('/sys/bus/netdevsim/del_device', 'w') as f: - f.write('1 0') - - @skip_if_not_supported -def test_list(netdevsim): +def test_list(): with DL() as dl: dls = dl.get_dump() if not dls: diff --git a/tests/test_linux/test_dhcp/captures/test_parser/test_android_reboot_request.pcap b/tests/test_linux/test_dhcp/captures/test_parser/test_android_reboot_request.pcap deleted file mode 100644 index 2a08ba74945cdabba79b17cfea8b97622febdd71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 386 zcmca|c+)~A1{MYcU}0bcaw<1ZPoI8=fguXW0pb5(U{uwxYWZ;v23G+_9|i^o1_wdQ zNg$mdNd^}NXGRNd9Y#hrhSq2D#}OtXSP=aP{!nKnHw7n~GBVq;GeUf3#ByLo+>+NEx;_zx<~)qdZ2VF(##YwaMLGW&0G4DiBme*a diff --git a/tests/test_linux/test_dhcp/captures/test_parser/test_android_tethering_renew.pcap b/tests/test_linux/test_dhcp/captures/test_parser/test_android_tethering_renew.pcap deleted file mode 100644 index e9ea91767cce7b4dfb9046c8890ec041d9f1d5ff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 754 zcmca|c+)~A1{MYcU}0bca!f+Dq^G#DF!%sDAl%k@SH|o0f+a^MuSwuwaAjb$+acn> z;2@Z{`M?T~6+rxs!G*z@QR}4&BO@C_L-=n_kTD=RgvksH5dFv^Lz|u46r60z$ZX5b z2=SS@AR`kS55JU5C9|T2rdIjSKZ2G_|Nrywlw@S)Wfvr-=%(i7{|ANbr!*FZJYWcc zFhnmTgu!8(5WWr+w&%Ws!WA5b&I~S$J_|N7!NL|8Y#?zceg_K0;Tg89W-OpUFb1Yj zK7K18$28OaE`+*g4|3GpK&I~S0I%k8KV4=*= zzyJb3`6e&}NDR+VW-?;|1%fd!t*$u02nulqX0Qkwm?SCOVIeiVX;A8#>A;7Ux)6Do z$$Uw)_}qRoVtK{m`P_!(a6ig%toM8)P$L?An6T~w;?Hzhu_E3EkeOLKPSIP j!O+0KU}Pq}I$-MRPUS^QU4+Z7QzNkKVjV5JNGxgr9MY<` diff --git a/tests/test_linux/test_dhcp/captures/test_parser/test_huawei_discover_option_148.pcap b/tests/test_linux/test_dhcp/captures/test_parser/test_huawei_discover_option_148.pcap deleted file mode 100644 index b2ce0a35624cc71b5415549437a07e982712c826..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 833 zcmca|c+)~A1{MYcU}0bca#FS&O*bgyWC#OtK=?lxEU7uUN@RB+uk zkWP>!gA0Q*qX`28BO@C_T*Rb%3^E8aVJwJ#7;os)$xXq@ri_fX?2HhfSu(Niuo2HF zO)O8%RB+5Ja?MN6%u6*jH#5{VG&JXCWM<=+QPkF{oU-FT*z=%JE-dC`*bEF|5Qdnt z!7-mrO#~duOAj1mU~mvDd(X8)fRSs5bUK4GgA3!te-oI{LK!3nQUfw}c!n~Q84JjR z#w-k+Rt;7x3@mdUtXUX1gbf&3{{R2LAOn<@|IZ9o!v-d&Bqye4=A`E4r=;3~S-QzN z`K2k=5YfcE#Pn2Muw-Tdidag1ZenJht%-%6p{0qQk*SHEp|LrNih}&25?ez910zH0 F{{RK^c(DKg diff --git a/tests/test_linux/test_dhcp/captures/test_parser/test_invalid_client_id_option.pcap b/tests/test_linux/test_dhcp/captures/test_parser/test_invalid_client_id_option.pcap deleted file mode 100644 index 8e7d4d804c3ded7a004c99b1bc96edde54d09b72..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 382 zcmca|c+)~A1{MYcU}0bca+deaNPlR;&JYIVfbf4XSknA_=Ph{-23H0~kL+ncQ9*xx zkWP>!gA0Q*qsaw-Mn*OUZN88egoy|iL_dN*)LF?*!O5nK%;r3dmCS7XIYs4v;&t>a rnON`Gaxg;tYQ%D21%r+;3q#p*1|D8trvTlQR9&Cc%rsr&{|pQODIGCT diff --git a/tests/test_linux/test_dhcp/captures/test_parser/test_invalid_router_option.pcap b/tests/test_linux/test_dhcp/captures/test_parser/test_invalid_router_option.pcap deleted file mode 100644 index 0b5d2e41e375538415821311ff2ce02f1d6bbe98..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 395 zcmca|c+)~A1{MYcU}0bca&kmwq;IX_Wk?2cKv*H|JadOhPzZyT{7w!A*8_~9i`f|% z90VBz4y@4P0^&3VX9gEW$LqpOjBE@k>8X2RCW7QZYCwF5DKMF#PbW78Cz~>|nz4X9 zXw1UEY1LrG!oV`u!J37EL)d_k<^TWx3^E|u|7>72{K6&qr8)Vf#i@E}`K3k4`6YU3 NMa=&hR%reE4*-<1GT8tC diff --git a/tests/test_linux/test_dhcp/captures/test_parser/test_netatmo_discover_request.pcap b/tests/test_linux/test_dhcp/captures/test_parser/test_netatmo_discover_request.pcap deleted file mode 100644 index cd5bde9d5c005a4fdff7056de51188c362c3b7e3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1122 zcmca|c+)~A1{MYcU}0bca?WzjNdGdMogog$0pb5(Q1C8*xuB1O!IgnAKzBP(RIuk6 zNGC{=!G*z@(dM5QBO@EbTY>*G5hfy75d8@LP-i7K1t*&_GFmdR-Z5ukWR_w3Po9s? z14DZ6}NwFSR7GBsX6-AhoDCKQA#yH#{}5BqOy*Hy9|MnV$#G2>|T#a})po diff --git a/tests/test_linux/test_dhcp/captures/test_parser/test_truncated_packet.pcap b/tests/test_linux/test_dhcp/captures/test_parser/test_truncated_packet.pcap deleted file mode 100644 index 44abffc0259f3fab37bbee5d5fa14c7dd11af1f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 630 zcmca|c+)~A1{MYcU}0bcayX-Br0WT?Gx#wvfboAY_)y@_B=DSr!Igo@0jP+~=^Q=F$jHXP6m@eINE{tN^rMRoU4C*?aIz^QvkgyaN=9-)vZ0=lp`M`; w%YhYIsb(OO%aV!7!JLDUS?Z6r78{Q(J0m0nc-VszbBjy!(*F-#zf z(g~7eaA9y}v=P|B$jHVZDle}9G6fw#^rMRoU4C*?aIz^Qvn3Pj9V3si8QB=jbUaoeOhjUV^&|0zI5W8^IN6ku(UzSN5;8_C2Uf&Q=3x%c R%rs|XWM<=+QT&h3ivVr-C8q!Y diff --git a/tests/test_linux/test_dhcp/captures/test_unit/test_ack_invalid_request_state.pcap b/tests/test_linux/test_dhcp/captures/test_unit/test_ack_invalid_request_state.pcap deleted file mode 100644 index 878ac18727c8ca6a43f93dc793bb539cab201b59..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 740 zcmca|c+)~A1{MYcU}0bca^_#3k=`?jfgudY0pX&9q4{OadUdsZ@$noCt_K)B8n-hr zI0!OMIQQUl^cOo7P^eLA@*IN6ku$&3Z$ zL1R!TR#<^Z8*31m$;k5m|9=J`qG?eQZ7#hHa0p*8zD6?XOG9!_p3;>>&WOM)k diff --git a/tests/test_linux/test_dhcp/captures/test_unit/test_get_and_renew_lease.pcap b/tests/test_linux/test_dhcp/captures/test_unit/test_get_and_renew_lease.pcap deleted file mode 100644 index b259644a7ea859fd2fc56e2db352b60430df1139..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1098 zcmca|c+)~A1{MYcU}0bca^_#3k=`?jfgudY0pX&9q4{OadUdsZ@$noCt_K)B8n-hr zI0!OMIQQUl^cOo7P^eLA@*IN6ku$&3Z$ zL1R!TR#<^Z8*31m$;k5m|9=J`qG?Ysi7#hHa0p*8zD6?XOG9!_pd>a_b_gIDO<>enp59NK2prNcX LazdGv>`(>(ai(w| diff --git a/tests/test_linux/test_dhcp/captures/test_unit/test_offer_wrong_xid.pcap b/tests/test_linux/test_dhcp/captures/test_unit/test_offer_wrong_xid.pcap deleted file mode 120000 index dac46c283..000000000 --- a/tests/test_linux/test_dhcp/captures/test_unit/test_offer_wrong_xid.pcap +++ /dev/null @@ -1 +0,0 @@ -test_requesting_timeout.pcap \ No newline at end of file diff --git a/tests/test_linux/test_dhcp/captures/test_unit/test_requesting_timeout.pcap b/tests/test_linux/test_dhcp/captures/test_unit/test_requesting_timeout.pcap deleted file mode 100644 index e746f73277ef9135ddfdd2dd3464d5d1def2cba9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 382 zcmca|c+)~A1{MYcU}0bca$cNYnEqFkmmv(u0pb5(pjTJh7az~T;L5=00n*_h`0w|D z6}uQgk_^raE{rAzA2KnrG2C^IQe|Lh04ZSr%7=m(K%(eiXnt8Uj5qY@0dVaNk=KzPBDqm$Ppv~}K<@p{d{;L5<55WdcV z!9noc_X8_D-U0Cn24@BrMxO;6nHbp^8p3~bf=mR7L-9KhA7ToKH9SCaQ*g2=BdZw; zC=iT+p~%N?1ti(ot$`$uFC)wU|Nj|eK(hat!D`sRBo9wXMrK}iL1Kz-YF@rJzoVZ^ SkiVx(ysvABYmlqUe+B@C!$C6u diff --git a/tests/test_linux/test_dhcp/captures/test_unit/test_unexpected_dhcp_message.pcap b/tests/test_linux/test_dhcp/captures/test_unit/test_unexpected_dhcp_message.pcap deleted file mode 100644 index 7fb3fd27cebc853f17b7ae3805e394b53c4311b6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 382 zcmca|c+)~A1{MYcU}0bca^_#3k#4zwfgudY0pb5(pjTJh7az~T;Cg`3qi_cUgM(mC z^MMt+7(tQ@&I~S$Ce`nm7}*$vOu|YS7#hHa0_8nHYCxQ#gQ59l&0yhSK_oW?Cz~>| mnz4X9XbcL)3M&xFXbmDW8CgK#A_J2B&kR<>1}6W*JP!Z?h&BEI diff --git a/tests/test_linux/test_dhcp/captures/test_unit/test_unknown_message.pcap b/tests/test_linux/test_dhcp/captures/test_unit/test_unknown_message.pcap deleted file mode 100644 index 29cf022925af10e787cf5c1b52f2ff3370d1a48f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 414 zcmca|c+)~A1{MYcU}0bca@HM~o__l|Cqo&K1H%8oAY*Zb!iH-c46Xu<84L^#3=V>p zWgwj(Nd^}NXU34@wv3Ey4E+K57Z4^QSP=aP{!nKnHw7n~GBVq;GeUf3#ByK-gPa+N z{AbC;ddEg6F)yVkKQl!)B_p{&*Ids~&yYvfttd4$DZfJ3H#09&*T^#2(8SElG}$uA WB-JF**qn=znT=mc#@NbQ`#%7CcsjNK diff --git a/tests/test_linux/test_dhcp/conftest.py b/tests/test_linux/test_dhcp/conftest.py deleted file mode 100644 index d01d90d82..000000000 --- a/tests/test_linux/test_dhcp/conftest.py +++ /dev/null @@ -1,100 +0,0 @@ -'''Fixtures only relevant for dhcp tests.''' - -import json -import re -import socket -from typing import Awaitable, Callable, Optional - -import pytest -from fixtures.dhcp_servers import lease_time # noqa: F401 -from fixtures.interfaces import VethPair - -from pyroute2.common import uifname -from pyroute2.dhcp.client import ClientConfig -from pyroute2.fixtures.iproute import TestContext -from pyroute2.iproute.linux import AsyncIPRoute -from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg - - -@pytest.fixture -def dhcp_client_host_name() -> str: - '''The hostname option sent in dhcp tests.''' - return f'test-hostname-{uifname()}' - - -@pytest.fixture -def dhcp_client_vendor_id() -> str: - '''The vendor id option sent in dhcp tests.''' - return f'vendor-id-{uifname()}' - - -@pytest.fixture -def client_config( - veth_pair: VethPair, - dhcp_client_host_name: Optional[str], - dhcp_client_vendor_id: Optional[str], -) -> ClientConfig: - '''Fixture that returns a ClientConfig for the veth_pair. - - Signal handlers are disabled. - ''' - return ClientConfig( - interface=veth_pair.client, - vendor_id=dhcp_client_vendor_id, - host_name=dhcp_client_host_name, - handle_signals=False, - ) - - -@pytest.fixture -def set_fixed_xid(monkeypatch: pytest.MonkeyPatch) -> Callable[[int], None]: - '''Set a static value to use instead of randomly generated xids.''' - - def _set_fixed_xid(xid: int): - monkeypatch.setattr( - "pyroute2.dhcp.xids.random_xid_prefix", lambda: xid - ) - - return _set_fixed_xid - - -async def _get_ipv4_addrs(ipr: AsyncIPRoute, index: int) -> list[str]: - return [ - i - async for i in await ipr.addr( - 'dump', index=index, family=socket.AF_INET - ) - ] - - -GetIPv4Addrs = Callable[[None], Awaitable[list[ifaddrmsg]]] - - -@pytest.fixture -def get_ipv4_addrs(async_context: TestContext[AsyncIPRoute]) -> GetIPv4Addrs: - '''Callable fixture that returns the test interface's ipv4 addresses.''' - - async def _wrapped() -> list[str]: - return await _get_ipv4_addrs( - ipr=async_context.ipr, index=async_context.test_link.index - ) - - return _wrapped - - -GetIPv4AddrsFor = Callable[[int], Awaitable[list[ifaddrmsg]]] - - -@pytest.fixture -def get_ipv4_addrs_for(async_ipr: AsyncIPRoute) -> GetIPv4AddrsFor: - '''Callable fixture that returns an interface's ipv4 addresses.''' - - async def _wrapped(index: int) -> list[str]: - return await _get_ipv4_addrs(ipr=async_ipr, index=index) - - return _wrapped - - -def parse_stdout_leases(data: bytes) -> list[dict]: - '''Parses leases written by the client on stdout.''' - return [json.loads(i) for i in re.split(b'(?<=\n)(?=\\{\n)', data)] diff --git a/tests/test_linux/test_dhcp/test_cli.py b/tests/test_linux/test_dhcp/test_cli.py deleted file mode 100644 index f63d6e5a3..000000000 --- a/tests/test_linux/test_dhcp/test_cli.py +++ /dev/null @@ -1,248 +0,0 @@ -import asyncio -import signal -from ipaddress import IPv4Address - -import pytest -from fixtures.dhcp_servers.dnsmasq import DnsmasqFixture -from fixtures.dhcp_servers.udhcpd import UdhcpdFixture -from fixtures.interfaces import VethPair -from pr2test.marks import require_root -from test_dhcp.conftest import GetIPv4AddrsFor, parse_stdout_leases - -from pyroute2.iproute.linux import AsyncIPRoute - -pytestmark = [ - pytest.mark.asyncio, - require_root(), - pytest.mark.usefixtures('setns_context'), -] - - -async def test_client_console(dnsmasq: DnsmasqFixture, veth_pair: VethPair): - '''The commandline client can get a lease, print it to stdout and exit.''' - process = await asyncio.create_subprocess_exec( - 'pyroute2-dhcp-client', - veth_pair.client, - '--lease-type', - 'pyroute2.dhcp.leases.JSONStdoutLease', - '--exit-on-timeout=5', - '--log-level=DEBUG', - stdout=asyncio.subprocess.PIPE, - ) - - asyncio.get_running_loop().call_later( - 2, process.send_signal, signal.SIGINT - ) - try: - stdout, _ = await asyncio.wait_for(process.communicate(), timeout=5) - except TimeoutError: - raise AssertionError(f'Timed out. dnsmasq output: {dnsmasq.stderr}') - assert process.returncode == 0 - assert stdout - json_leases = parse_stdout_leases(stdout) - assert len(json_leases) == 1 - json_lease = json_leases[0] - assert json_lease['interface'] == veth_pair.client - assert ( - dnsmasq.config.range.start - <= IPv4Address(json_lease['ack']['yiaddr']) - <= dnsmasq.config.range.end - ) - - -async def test_interface_flaps(dnsmasq: DnsmasqFixture, veth_pair: VethPair): - # Run a dhcp client - process = await asyncio.create_subprocess_exec( - 'pyroute2-dhcp-client', - veth_pair.client, - '--lease-type', - 'pyroute2.dhcp.leases.JSONStdoutLease', - '--log-level=INFO', - '--hook', - 'pyroute2.dhcp.hooks.configure_ip', - '--hook', - 'pyroute2.dhcp.hooks.remove_ip', - stderr=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - ) - # TODO: check the interface has an IP - res_tsk = asyncio.Task(process.communicate()) - await asyncio.sleep(3) - # put iface down - async with AsyncIPRoute() as ipr: - await ipr.link('set', index=veth_pair.client_idx, state='down') - await asyncio.sleep(0.5) - # TODO: check the interface has no IP anymore - # up again - async with AsyncIPRoute() as ipr: - await ipr.link('set', index=veth_pair.client_idx, state='up') - - # stop client - await asyncio.sleep(2) - # TODO: check the interface has an IP again - process.send_signal(signal.SIGINT) - stdout, stderr = await asyncio.wait_for(res_tsk, timeout=5) - assert process.returncode == 0 - # TODO: check the interface has no IP anymore - - # check the logs mention the interface flapping - logs = stderr.decode() - assert logs, 'not a single lease' - assert logs.index(f'{veth_pair.client} went down') < logs.index( - f'Waiting for {veth_pair.client} to go up...' - ) - - # check we got 2 leases - leases = parse_stdout_leases(stdout) - assert len(leases) == 2 - first_json_lease, second_json_lease = leases - assert ( - first_json_lease['ack']['options'] - == second_json_lease['ack']['options'] - ) - - -@pytest.mark.parametrize( - ('switch', 'value', 'err_msg'), - ( - ( - '--lease-type', - 'meublé', - '\'meublé\' must point to a Lease subclass.', - ), - ( - '--hook', - 'captain.hook', - '\'captain.hook\' must point to a valid hook.', - ), - ), -) -async def test_wrong_custom_hook_or_lease( - switch: str, value: str, err_msg: str -): - process = await asyncio.create_subprocess_exec( - 'pyroute2-dhcp-client', - 'irrelevantIface', - switch, - value, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await process.communicate() - assert process.returncode > 0 - assert stderr - assert stderr.splitlines()[-1].decode().endswith(err_msg) - - -async def test_exit_timeout( - udhcpd: UdhcpdFixture, - veth_pair: VethPair, - get_ipv4_addrs_for: GetIPv4AddrsFor, -): - process = await asyncio.create_subprocess_exec( - 'pyroute2-dhcp-client', - veth_pair.client, - '--lease-type', - 'pyroute2.dhcp.leases.JSONStdoutLease', - '--exit-on-timeout=3', - '--no-release', - stderr=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - ) - try: - stdout, _ = await asyncio.wait_for(process.communicate(), timeout=5) - except TimeoutError: - raise AssertionError(f'Timed out. udhcpd output: {udhcpd.stderr}') - assert process.returncode == 0 - # check the lease - assert stdout - json_leases = parse_stdout_leases(stdout) - assert len(json_leases) == 1 - json_lease = json_leases[0] - assert json_lease['interface'] == veth_pair.client - ip = json_lease['ack']['yiaddr'] - assert udhcpd.stderr[-2:] == [ - f'udhcpd: sending OFFER to {ip}', - f'udhcpd: sending ACK to {ip}', - ] - - # since we passed --no-release, the ip is still there on exit - ips = await get_ipv4_addrs_for(veth_pair.client_idx) - assert len(ips) == 1 - assert ips[0].get('address') == ip - - -@pytest.mark.parametrize( - ('signum', 'expected_signal_log', 'expected_state_change'), - ( - ( - signal.SIGUSR1, - 'SIGUSR1 received, renewing lease', - 'BOUND -> RENEWING', - ), - ( - signal.SIGUSR2, - 'SIGUSR2 received, rebinding lease', - 'BOUND -> REBINDING', - ), - (signal.SIGHUP, 'SIGHUP received, resetting', 'BOUND -> INIT'), - ), -) -async def test_signals( - dnsmasq: DnsmasqFixture, - veth_pair: VethPair, - signum: int, - expected_signal_log: str, - expected_state_change: str, -): - '''Signals can be sent to the client to rene/rebind/reset its lease''' - # Run a dhcp client - process = await asyncio.create_subprocess_exec( - 'pyroute2-dhcp-client', - veth_pair.client, - '--lease-type', - 'pyroute2.dhcp.leases.JSONStdoutLease', - '--log-level=DEBUG', - '--hook', - 'pyroute2.dhcp.hooks.configure_ip', - '--hook', - 'pyroute2.dhcp.hooks.remove_ip', - stderr=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - ) - # wait till the client is bound - await asyncio.wait_for(dnsmasq.wait_for_log('DHCPACK'), timeout=3) - dnsmasq.expected_logs.clear() - await asyncio.sleep(0.2) - # send a signal to trigger a renewal - process.send_signal(signum) - # wait till the client is bound again - await asyncio.wait_for(dnsmasq.wait_for_log('DHCPACK'), timeout=2) - await asyncio.sleep(0.2) - # stop the client - process.send_signal(signal.SIGINT) - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=2) - # check the client received the signal - assert expected_signal_log.encode() in stderr - assert expected_state_change.encode() in stderr - - # check there are two leases - leases = parse_stdout_leases(stdout) - assert len(leases) == 2 - first_lease, second_lease = leases - first_lease['ack']['yiaddr'] == second_lease['ack']['yiaddr'] - - -async def test_interface_does_not_exist(): - '''The client raises a meaninfgul error - if the interface does not exist.''' - - process = await asyncio.create_subprocess_exec( - 'pyroute2-dhcp-client', 'doesn0texist', stderr=asyncio.subprocess.PIPE - ) - _, stderr = await process.communicate() - assert process.returncode and process.returncode > 0 - assert stderr - assert ( - stderr.splitlines()[-1].decode() - == 'pyroute2-dhcp-client: error: Interface not found: doesn0texist' - ) diff --git a/tests/test_linux/test_dhcp/test_encode.py b/tests/test_linux/test_dhcp/test_encode.py deleted file mode 100644 index bf228a59d..000000000 --- a/tests/test_linux/test_dhcp/test_encode.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Any - -import pytest - -from pyroute2.dhcp.dhcp4msg import dhcp4msg -from pyroute2.dhcp.enums import dhcp - - -@pytest.mark.parametrize( - ('option_name', 'option_value'), - ( - ('name_server', ['1.1.1.2', '2.2.2.2']), - ('lease_time', -1), - ('host_name', 'some computer'), - ('max_msg_size', 1500), - ('subnet_mask', '255.255.255.0'), - ('client_id', {'type': 1, 'key': '16:f4:cb:71:09:a1'}), - ('client_id', {'type': 0, 'key': b'some-client-identifier'}), - ('perform_mask_discovery', True), - ('tcp_keepalive_garbage', False), - ('host_name', [104, 97, 153, 104, 97]), - ), -) -def test_encode_decode_options(option_name: str, option_value: Any): - msg = dhcp4msg( - { - 'options': { - 'message_type': dhcp.MessageType.ACK, - option_name: option_value, - } - } - ) - data = msg.encode().buf - decoded_msg = dhcp4msg(buf=data).decode() - assert ( - decoded_msg['options'][option_name] - == msg['options'][option_name] - == option_value - ) - - -# TODO: test invalid client id diff --git a/tests/test_linux/test_dhcp/test_hooks.py b/tests/test_linux/test_dhcp/test_hooks.py deleted file mode 100644 index ded45998d..000000000 --- a/tests/test_linux/test_dhcp/test_hooks.py +++ /dev/null @@ -1,199 +0,0 @@ -import asyncio -import errno -import json -import logging - -import pytest -from pr2test.marks import require_root -from test_dhcp.conftest import GetIPv4Addrs - -from pyroute2.dhcp import hooks -from pyroute2.dhcp.leases import JSONFileLease -from pyroute2.fixtures.iproute import TestContext -from pyroute2.iproute.linux import AsyncIPRoute -from pyroute2.netlink.exceptions import NetlinkError - -pytestmark = [ - require_root(), - pytest.mark.asyncio, - pytest.mark.usefixtures('setns_context'), -] - -FAKE_LEASE = { - 'ack': { - 'op': 2, - 'htype': 1, - 'hlen': 6, - 'hops': 0, - 'xid': 1323206580, - 'secs': 0, - 'flags': 32768, - 'ciaddr': '0.0.0.0', - 'yiaddr': '192.168.112.73', - 'siaddr': '192.168.112.1', - 'giaddr': '0.0.0.0', - 'chaddr': '72:c1:55:6f:76:83', - 'sname': '', - 'file': '', - 'cookie': '63:82:53', - 'options': { - 'message_type': 5, - 'server_id': '192.168.112.1', - 'lease_time': 120, - 'renewal_time': 60, - 'rebinding_time': 105, - 'subnet_mask': '255.255.255.0', - 'broadcast_address': '192.168.112.255', - 'router': ['192.168.112.1'], - 'name_server': ['192.168.112.1'], - }, - }, - 'interface': '', - 'server_mac': '2e:7e:7d:8e:5f:5f', - 'obtained': 1738249608.073041, -} - - -@pytest.fixture -def fake_lease(test_link_ifname: str) -> JSONFileLease: - '''Fixture that returns a fake lease loaded from disk.''' - raw_lease = FAKE_LEASE.copy() - raw_lease['interface'] = test_link_ifname - JSONFileLease._get_path(test_link_ifname).write_text(json.dumps(raw_lease)) - lease = JSONFileLease.load(test_link_ifname) - assert lease - return lease - - -async def test_add_and_remove_ip_hooks( - fake_lease: JSONFileLease, - async_context: TestContext[AsyncIPRoute], - get_ipv4_addrs: GetIPv4Addrs, - caplog: pytest.LogCaptureFixture, -): - '''Test the hooks that add & remove an address from an interface.''' - caplog.set_level(logging.INFO, logger='pyroute2.dhcp.hooks') - - # call the hook that adds the IP address to the dummy interface - await hooks.configure_ip(lease=fake_lease) - # check the ip addr & broadcast addr have ben set - assert len(addrs := await get_ipv4_addrs()) == 1 - addr = addrs[0] - assert addr.get('IFA_ADDRESS') == fake_lease.ip - assert addr.get('IFA_BROADCAST') == fake_lease.broadcast_address - - # call the hooks that removes the IP address - await hooks.remove_ip(lease=fake_lease) - # check the interface has no address anymore - assert len(addrs := await get_ipv4_addrs()) == 0 - - assert caplog.messages == [ - f'Adding {fake_lease.ip}/{fake_lease.subnet_mask}' - f' to {fake_lease.interface}', - f'Removing {fake_lease.ip}/{fake_lease.subnet_mask}' - f' from {fake_lease.interface}', - ] - - -async def test_configure_ip_missing_broadcast_addr( - fake_lease: JSONFileLease, - async_context: TestContext[AsyncIPRoute], - get_ipv4_addrs: GetIPv4Addrs, - caplog: pytest.LogCaptureFixture, -): - '''The configure_ip hook mustn't crash when broadcast addr is missing.''' - caplog.set_level(logging.DEBUG, logger='pyroute2.dhcp.hooks') - - del fake_lease.ack['options']['broadcast_address'] - await hooks.configure_ip(fake_lease) - assert caplog.messages == [ - f'Adding {fake_lease.ip}/{fake_lease.subnet_mask}' - f' to {fake_lease.interface}', - 'Lease does not set ', - ] - # check the ip addr has been set, but no broadcast addr - assert len(addrs := await get_ipv4_addrs()) == 1 - addr = addrs[0] - assert addr.get('IFA_ADDRESS') == fake_lease.ip - assert addr.get('IFA_BROADCAST') is None - - -@pytest.mark.asyncio -async def test_add_and_remove_gw_hooks( - fake_lease: JSONFileLease, - async_context: TestContext[AsyncIPRoute], - caplog: pytest.LogCaptureFixture, -): - '''Test the hooks that add & remove the default gw for a lease.''' - caplog.set_level(logging.INFO, logger='pyroute2.dhcp.hooks') - await async_context.ipr.addr( - 'add', - index=async_context.test_link.index, - address=fake_lease.ip, - prefixlen=fake_lease.prefixlen, - ) - await hooks.add_default_gw(lease=fake_lease) - routes = await async_context.ipr.route('get', dst='1.2.3.4') - assert len(routes) == 1 - assert routes[0].get('RTA_DST') == '1.2.3.4' - assert routes[0].get('RTA_OIF') == async_context.test_link.index - assert routes[0].get('RTA_PREFSRC') == fake_lease.ip - await hooks.remove_default_gw(lease=fake_lease) - with pytest.raises(NetlinkError) as err_ctx: - await async_context.ipr.route('get', dst='1.2.3.4') - assert err_ctx.value.code == errno.ENETUNREACH - - assert caplog.messages == [ - f'Adding {fake_lease.default_gateway} ' - f'as default route through {fake_lease.interface}', - f'Removing {fake_lease.default_gateway} as default route', - ] - - -async def test_remove_gw_already_removed( - fake_lease: JSONFileLease, caplog: pytest.LogCaptureFixture -): - '''Removing the default gw must not crash when it doesn't exist.''' - caplog.set_level(logging.INFO, logger='pyroute2.dhcp.hooks') - await hooks.remove_default_gw(lease=fake_lease) - assert caplog.messages == [ - f'Removing {fake_lease.default_gateway} as default route', - 'Default route was already removed by another process', - ] - - -async def test_hook_timeout( - fake_lease: JSONFileLease, caplog: pytest.LogCaptureFixture -): - '''Hooks that exceed the timeout cause an error log but no crash.''' - caplog.set_level(logging.ERROR, logger='pyroute2.dhcp.hooks') - - @hooks.hook(hooks.Trigger.BOUND) - async def sleepy_hook(**_): - await asyncio.sleep(10) - - await hooks.run_hooks( - hooks=[sleepy_hook], - lease=fake_lease, - trigger=hooks.Trigger.BOUND, - timeout=0.1, - ) - assert caplog.messages == ["Hook 'sleepy_hook' timed out"] - - -async def test_failing_hook( - fake_lease: JSONFileLease, caplog: pytest.LogCaptureFixture -): - '''Hooks that raise an exception cause an error log but no crash.''' - caplog.set_level(logging.ERROR, logger='pyroute2.dhcp.hooks') - - @hooks.hook(hooks.Trigger.BOUND) - async def failing_hook(**_): - raise RuntimeError('boom') - - await hooks.run_hooks( - hooks=[failing_hook], lease=fake_lease, trigger=hooks.Trigger.BOUND - ) - assert caplog.messages == [ - "Hook failing_hook failed: RuntimeError('boom')" - ] diff --git a/tests/test_linux/test_dhcp/test_integration.py b/tests/test_linux/test_dhcp/test_integration.py deleted file mode 100644 index 294a51cc3..000000000 --- a/tests/test_linux/test_dhcp/test_integration.py +++ /dev/null @@ -1,391 +0,0 @@ -import json -import os -from ipaddress import IPv4Address -from pathlib import Path - -import pytest -from fixtures.dhcp_servers.dnsmasq import DnsmasqFixture -from fixtures.dhcp_servers.udhcpd import UdhcpdFixture -from fixtures.interfaces import VethPair -from pr2test.marks import require_root - -from pyroute2.dhcp import fsm, hooks -from pyroute2.dhcp.client import AsyncDHCPClient, ClientConfig -from pyroute2.dhcp.enums import bootp, dhcp -from pyroute2.dhcp.leases import JSONFileLease, JSONStdoutLease -from pyroute2.fixtures.iproute import TestContext -from pyroute2.iproute.linux import AsyncIPRoute - -pytestmark = [require_root(), pytest.mark.asyncio] - - -async def test_get_lease_from_dnsmasq( - dnsmasq: DnsmasqFixture, - veth_pair: VethPair, - client_config: ClientConfig, - tmpdir: str, - monkeypatch: pytest.MonkeyPatch, -): - '''The client can get a lease from dnsmasq and write it to a file.''' - work_dir = Path(tmpdir) - # Patch JSONFileLease so leases get written to the temp dir - # instead of whatever the working directory is - monkeypatch.setattr(JSONFileLease, '_get_lease_dir', lambda: work_dir) - # boot up the dhcp client and wait for a lease - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(fsm.State.BOUND, timeout=10) - assert cli.state == fsm.State.BOUND - lease = cli.lease - xid = cli.xid - assert lease - assert xid - - assert lease.ack['xid'] == xid.for_state(fsm.State.REQUESTING) - - # check the obtained lease - assert lease.interface == veth_pair.client - assert lease.ack['op'] == bootp.MessageType.BOOTREPLY - assert lease.ack['options']['message_type'] == dhcp.MessageType.ACK - assert lease.ack['options']['lease_time'] == dnsmasq.config.lease_time - assert ( - lease.ack['options']['renewal_time'] == dnsmasq.config.lease_time / 2 - ) - assert lease.expiration_in > lease.rebinding_in > lease.renewal_in > 0 - assert lease.expired is False - assert lease.server_id == str(dnsmasq.config.range.router) - assert lease.routers == [str(dnsmasq.config.range.router)] - assert ( - dnsmasq.config.range.start - <= IPv4Address(lease.ip) - <= dnsmasq.config.range.end - ) - assert lease.ack['chaddr'] - # TODO: check chaddr matches veth_pair.client's MAC - - # check the lease was written to disk and can be loaded - expected_lease_file = JSONFileLease._get_path(lease.interface) - assert expected_lease_file.is_file() - json_lease = json.loads(expected_lease_file.read_bytes()) - assert isinstance(json_lease, dict) - assert JSONFileLease(**json_lease) == lease - - -@pytest.mark.parametrize('lease_time', [5]) -async def test_short_udhcpd_lease(udhcpd: UdhcpdFixture, veth_pair: VethPair): - '''Test getting a lease from udhcpd and renewing it.''' - cfg = ClientConfig(interface=veth_pair.client, lease_type=JSONStdoutLease) - async with AsyncDHCPClient(cfg) as cli: - # No lease, we're in the INIT state - assert cli.state == fsm.State.INIT - # Start requesting an IP - await cli.bootstrap() - xid = cli.xid - # Then, the client in the SELECTING state while sending DISCOVERs - await cli.wait_for_state(fsm.State.SELECTING, timeout=1) - # Once we get an OFFER the client switches to REQUESTING - await cli.wait_for_state(fsm.State.REQUESTING, timeout=1) - # After getting an ACK, we're BOUND ! - await cli.wait_for_state(fsm.State.BOUND, timeout=1) - # ACK corresponds to a request that was made in the REQUESTING state - assert cli.lease.ack['xid'] == xid.for_state(fsm.State.REQUESTING) - - # a few seconds later, the renewal timer expires - await cli.wait_for_state(fsm.State.RENEWING, timeout=3) - # and we're bound again - await cli.wait_for_state(fsm.State.BOUND, timeout=1) - # ACK corresponds to a request that was made in the RENEWING state - assert cli.lease.ack['xid'] == xid.for_state(fsm.State.RENEWING) - - # Stop here, that's enough - lease = cli.lease - assert lease - assert xid - - # The obtained IP must be in the range - assert ( - udhcpd.config.range.start - <= IPv4Address(lease.ip) - <= udhcpd.config.range.end - ) - # check the lease - assert lease.server_id == str(udhcpd.config.range.router) - assert lease.routers == [str(udhcpd.config.range.router)] - assert lease.interface == veth_pair.client - assert lease.ack["options"]["lease_time"] == udhcpd.config.lease_time - # Check udhcpd output matches our expectations - assert udhcpd.stderr[-3:] == [ - f'udhcpd: sending OFFER to {lease.ip}', - f'udhcpd: sending ACK to {lease.ip}', - f'udhcpd: sending ACK to {lease.ip}', - ] - - -async def test_lease_file(client_config: ClientConfig): - '''The client must write a pidfile when configured to.''' - client_config.write_pidfile = True - async with AsyncDHCPClient(client_config): - assert client_config.pidfile_path.exists() - assert int(client_config.pidfile_path.read_text()) == os.getpid() - assert not client_config.pidfile_path.exists() - - -@pytest.mark.parametrize('lease_time', [5]) -async def test_lease_expiration( - udhcpd: UdhcpdFixture, - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, - lease_time: int, -): - '''The client must go back to INIT when the lease expires.''' - caplog.set_level('INFO', logger='pyroute2.dhcp') - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - # wait for the client to get a lease - await cli.wait_for_state(fsm.State.RENEWING, timeout=5.0) - # stop udhcpd, so nobody is answering anymore - await udhcpd.__aexit__(None, None, None) - # renewing timer expires - await cli.wait_for_state(fsm.State.RENEWING, timeout=5.0) - # rebinding timer expires - await cli.wait_for_state(fsm.State.REBINDING, timeout=5.0) - # lease expire, we look for a new one - await cli.wait_for_state(fsm.State.INIT, timeout=5.0) - await cli.wait_for_state(fsm.State.SELECTING, timeout=5.0) - # Check the lease scheduling logs are emitted in the right order - timer_logs_args = [ - i.args for i in caplog.records if i.msg.startswith('Scheduling') - ] - assert timer_logs_args[0][0] == 'renewal' - assert timer_logs_args[1][0] == 'rebinding' - assert timer_logs_args[2][0] == 'expiration' - # check the computed rebinding times are consistent - assert ( - 0 - < timer_logs_args[0][1] - < timer_logs_args[1][1] - < timer_logs_args[2][1] - <= lease_time - ) - assert ( - caplog.messages.index('T1 expired, renewing lease') - < caplog.messages.index('T2 expired, rebinding lease') - < caplog.messages.index('Lease expired') - ) - - -async def test_mac_addr_change_same_client_id( - dnsmasq: DnsmasqFixture, - client_config: ClientConfig, - veth_pair: VethPair, - caplog: pytest.LogCaptureFixture, - async_context: TestContext[AsyncIPRoute], - monkeypatch: pytest.MonkeyPatch, -): - '''The mac addr can change during the lease lifetime, renewing works''' - new_mac = '06:06:06:06:06:06' - caplog.set_level('INFO', logger='pyroute2.ext.rawsocket') - caplog.set_level('DEBUG', logger='pyroute2.dhcp') - client_config.client_id = b'Coucou' - # make sure the ip is configured, which is assumed by the client when bound - client_config.hooks = [hooks.configure_ip] - # renew & rebind earlier than dnsmasq tells the client (min. 60s) - monkeypatch.setattr('pyroute2.dhcp.leases.Lease.renewal_in', 1.0) - monkeypatch.setattr('pyroute2.dhcp.leases.Lease.rebinding_in', 3.0) - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - # wait for the client to get a lease - await cli.wait_for_state(fsm.State.BOUND, timeout=5.0) - first_lease = cli.lease - # change the mac addr - await async_context.ipr.link( - 'set', index=veth_pair.client_idx, address=new_mac - ) - # wait for the client to renew - await cli.wait_for_state(fsm.State.RENEWING, timeout=5.0) - # dnsmasq should have renewed the lease even though the mac changed - await cli.wait_for_state(fsm.State.BOUND, timeout=5.0) - second_lease = cli.lease - # the client's mac addr was indeed changed - assert cli._sock.l2addr == new_mac - assert second_lease.ack['chaddr'] == new_mac - assert first_lease.ack['chaddr'] != new_mac - assert any( - ( - f'l2addr for {veth_pair.client} changed' in i - for i in caplog.messages - ) - ) - # but not the ip - assert first_lease.ip == second_lease.ip - assert first_lease.obtained < second_lease.obtained - assert first_lease.ack['xid'] != second_lease.ack['xid'] - - -@pytest.mark.parametrize('lease_time', [5]) -async def test_mac_addr_change( - udhcpd: UdhcpdFixture, - client_config: ClientConfig, - veth_pair: VethPair, - caplog: pytest.LogCaptureFixture, - async_context: TestContext[AsyncIPRoute], -): - caplog.set_level('INFO', logger='pyroute2.ext.rawsocket') - caplog.set_level('INFO', logger='pyroute2.dhcp') - async with AsyncDHCPClient(client_config) as cli: - '''The mac (& client id) changes, we have to re-discover.''' - await cli.bootstrap() - # wait for the client to get a lease - await cli.wait_for_state(fsm.State.BOUND, timeout=5.0) - first_ip = cli.lease.ip - # change the mac addr - await async_context.ipr.link( - 'set', index=veth_pair.client_idx, address='06:06:06:06:06:06' - ) - # the client renews after a very short while - await cli.wait_for_state(fsm.State.RENEWING, timeout=5.0) - # since the mac (and client id) changed, the server does not answer - await cli.wait_for_state(fsm.State.REBINDING, timeout=5.0) - # we go back to discover - await cli.wait_for_state(fsm.State.SELECTING, timeout=5.0) - await cli.wait_for_state(fsm.State.REQUESTING, timeout=5.0) - # dnsmasq should have renewed the lease with a different ip - await cli.wait_for_state(fsm.State.BOUND, timeout=5.0) - second_ip = cli.lease.ip - assert second_ip != first_ip - - # the client's mac addr was indeed changed - assert any( - [ - f'l2addr for {veth_pair.client} changed' in i - for i in caplog.messages - ] - ) - assert cli._sock.l2addr == '06:06:06:06:06:06' - # udhcpd sent 2 offers and 2 acks - assert udhcpd.stderr[-4:] == [ - f'udhcpd: sending OFFER to {first_ip}', - f'udhcpd: sending ACK to {first_ip}', - f'udhcpd: sending OFFER to {second_ip}', - f'udhcpd: sending ACK to {second_ip}', - ] - - -@pytest.mark.parametrize('delete_lease', (True, False)) -@pytest.mark.parametrize('release', (True, False)) -async def test_fixed_client_id_changing_mac( - dnsmasq: DnsmasqFixture, - client_config: ClientConfig, - veth_pair: VethPair, - caplog: pytest.LogCaptureFixture, - async_context: TestContext[AsyncIPRoute], - release: bool, - delete_lease: bool, -): - '''When using a fixed client id, the client keeps its IP, - even when the mac changes. - - But if the lease is released and deleted locally, the client - will send a DISCOVER with a different mac & will be granted - a new IP. - ''' - caplog.set_level('INFO', logger='pyroute2.ext.rawsocket') - caplog.set_level('INFO', logger='pyroute2.dhcp') - client_config.client_id = b'test' - client_config.release = release - - # get an initial lease - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(fsm.State.BOUND, timeout=3.0) - first_lease = cli.lease - - # change the mac addr - await async_context.ipr.link( - 'set', index=veth_pair.client_idx, address='06:06:06:06:06:06' - ) - if delete_lease: - JSONFileLease._get_path(veth_pair.client).unlink() - - # get a second lease - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(fsm.State.BOUND, timeout=3.0) - second_lease = cli.lease - - if release: - await dnsmasq.wait_for_log('DHCPRELEASE') - - if release and delete_lease: - # the lease was released from the server and deleted locally, - # and the mac changed: we might get another IP but withot guarantee - # assert first_lease.ip != second_lease.ip - pass - else: - # the mac changed, but the client id stays the same so we're - # supposed to get the same ip twice - assert first_lease.ip == second_lease.ip - assert first_lease.ack['chaddr'] != second_lease.ack['chaddr'] - assert second_lease.ack['chaddr'] == '06:06:06:06:06:06' - assert first_lease.obtained < second_lease.obtained - - # check logs - logged_releases = [i for i in dnsmasq.stderr if 'DHCPRELEASE' in i] - if release: - assert len(logged_releases) == 2 - else: - assert not logged_releases - - init_count = caplog.messages.count('OFF -> INIT') - init_reboot_count = caplog.messages.count('OFF -> INIT_REBOOT') - if delete_lease: - # since the lease was deleted the client started from scratch twice - assert init_count == 2 - assert init_reboot_count == 0 - else: - # the second run read the existing lease - assert init_reboot_count == 1 - assert init_count == 1 - - -@pytest.mark.parametrize('lease_time', [-1]) -async def test_infinite_lease( - dnsmasq: DnsmasqFixture, - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, -): - '''Infinite leases are supported.''' - caplog.set_level('DEBUG', logger='pyroute2.dhcp') - async with AsyncDHCPClient(config=client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(fsm.State.BOUND, timeout=3.0) - lease = cli.lease - assert 'renewal time is infinite' in caplog.messages - assert lease.lease_time == -1 - assert lease.expiration_in is None - assert lease.rebinding_in is None - assert lease.renewal_in is None - assert lease.expired is False - - -async def test_lease_write_failure( - dnsmasq: DnsmasqFixture, - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -): - '''Failure to write a lease should not crash the client.''' - caplog.set_level("ERROR") - # Create a temp workdir for the test and delete it - # while the client is running - wd = tmp_path / "will_be_deleted" - wd.mkdir() - monkeypatch.chdir(wd) - async with AsyncDHCPClient(config=client_config) as cli: - await cli.bootstrap() - wd.rmdir() - await cli.wait_for_state(fsm.State.BOUND, timeout=3.0) - assert caplog.messages == [ - 'Could not dump lease: [Errno 2] No such file or directory' - ] diff --git a/tests/test_linux/test_dhcp/test_parser.py b/tests/test_linux/test_dhcp/test_parser.py deleted file mode 100644 index 862f36fea..000000000 --- a/tests/test_linux/test_dhcp/test_parser.py +++ /dev/null @@ -1,438 +0,0 @@ -from typing import Iterable - -import pytest -from fixtures.pcap_files import PcapFile - -from pyroute2.dhcp.dhcp4socket import AsyncDHCP4Socket -from pyroute2.dhcp.enums import bootp, dhcp -from pyroute2.dhcp.messages import ReceivedDHCPMessage - - -def parse_pcap( - pcap: Iterable[bytes], expected_packets: int -) -> list[ReceivedDHCPMessage]: - decoded_dhcp_messages = [AsyncDHCP4Socket._decode_msg(i) for i in pcap] - assert len(decoded_dhcp_messages) == expected_packets - return decoded_dhcp_messages - - -def test_decode_simple_lease_process(pcap: PcapFile): - '''Decode a simple DHCP handshake using AsyncDHCP4Socket.''' - discover, offer, request, ack = parse_pcap(pcap, expected_packets=4) - assert discover.message_type == dhcp.MessageType.DISCOVER - assert discover.dhcp['flags'] == bootp.Flag.BROADCAST - assert discover.sport, discover.dport == (68, 67) - assert discover.dhcp['options']['lease_time'] == -1 # infinity - assert discover.dhcp['options']['parameter_list'] == [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.DOMAIN_NAME, - ] - assert discover.dhcp['xid'] == 0x22334455 - assert offer.message_type == dhcp.MessageType.OFFER - assert offer.dhcp['flags'] == bootp.Flag.BROADCAST - assert offer.sport, offer.dport == (67, 68) - assert offer.dhcp['xid'] == 0x22334455 - assert offer.dhcp['options']['lease_time'] == 43200 - assert offer.dhcp['options']['name_server'] == ['192.168.94.254'] - assert offer.dhcp['options']['router'] == ['192.168.94.254'] - assert offer.dhcp['options']['server_id'] == '192.168.94.254' - assert offer.dhcp['options']['subnet_mask'] == '255.255.255.0' - - assert request.message_type == dhcp.MessageType.REQUEST - assert request.dhcp['flags'] == bootp.Flag.BROADCAST - assert request.sport, request.dport == (68, 67) - assert request.dhcp['xid'] == 0x22334455 - - assert ack.message_type == dhcp.MessageType.ACK - assert ack.dhcp['flags'] == bootp.Flag.BROADCAST - assert ack.sport, ack.dport == (67, 68) - assert ack.dhcp['xid'] == 0x22334455 - assert ack.dhcp['options']['lease_time'] == 43200 - assert ack.dhcp['options']['name_server'] == ['192.168.94.254'] - assert ack.dhcp['options']['router'] == ['192.168.94.254'] - assert ack.dhcp['options']['server_id'] == '192.168.94.254' - assert ack.dhcp['options']['subnet_mask'] == '255.255.255.0' - - -def test_android_reboot_request(pcap: PcapFile): - '''Decode the request sent by an Android phone in init-reboot state.''' - client_mac = '32:7a:80:aa:a7:c7' - request = parse_pcap(pcap, expected_packets=1)[0] - assert request.message_type == dhcp.MessageType.REQUEST - assert request.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert request.dhcp['chaddr'] == client_mac - assert request.dhcp['flags'] == bootp.Flag.UNICAST - assert request.dhcp['options'] == { - 'client_id': { - 'key': client_mac, - 'type': 1, # TODO use constant/enum ? - }, - 'host_name': 'TFY-LX1', - 'max_msg_size': 1500, - 'message_type': dhcp.MessageType.REQUEST, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.DOMAIN_NAME, - dhcp.Option.INTERFACE_MTU, - dhcp.Option.BROADCAST_ADDRESS, - # TODO: we should ask for these three too - dhcp.Option.LEASE_TIME, - dhcp.Option.RENEWAL_TIME, - dhcp.Option.REBINDING_TIME, - dhcp.Option.VENDOR_SPECIFIC_INFORMATION, - dhcp.Option.DHCP_CAPTIVE_PORTAL, - dhcp.Option.IPV6_ONLY_PREFERRED, - ], - 'requested_ip': '192.168.94.191', - 'vendor_id': 'android-dhcp-13', - } - assert request.eth_src == client_mac - assert request.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert request.ip_src == '0.0.0.0' - assert (request.sport, request.dport) == (68, 67) - - -def test_wii_discover(pcap: PcapFile): - '''Decode the request sent by a Wii trying to get an address.''' - client_mac = '00:1e:a9:87:91:a7' - discover = parse_pcap(pcap, expected_packets=1)[0] - assert discover.message_type == dhcp.MessageType.DISCOVER - assert discover.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert discover.dhcp['chaddr'] == client_mac - assert discover.dhcp['flags'] == bootp.Flag.UNICAST - assert discover.dhcp['options'] == { - 'client_id': {'key': client_mac, 'type': 1}, - 'host_name': 'Wii', - 'message_type': dhcp.MessageType.DISCOVER, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.DOMAIN_NAME, - dhcp.Option.BROADCAST_ADDRESS, - dhcp.Option.STATIC_ROUTE, - ], - 'requested_ip': '192.168.94.147', - } - assert discover.eth_src == client_mac - assert discover.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert discover.ip_src == '0.0.0.0' - assert (discover.sport, discover.dport) == (68, 67) - - -def test_washing_machine_request(pcap: PcapFile): - '''Decode a request sent by Quentin's "smart" (sic) washing machine.''' - washing_mac = '14:7f:67:8a:7b:4a' - request = parse_pcap(pcap, expected_packets=1)[0] - assert request.message_type == dhcp.MessageType.REQUEST - assert request.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert request.dhcp['chaddr'] == washing_mac - assert request.dhcp['flags'] == bootp.Flag.UNICAST - assert request.dhcp['options'] == { - 'host_name': 'LG_Smart_Laundry2_open', - 'max_msg_size': 1500, - 'message_type': dhcp.MessageType.REQUEST, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.BROADCAST_ADDRESS, - dhcp.Option.NAME_SERVER, - ], - 'requested_ip': '192.168.0.33', - 'server_id': '192.168.0.254', - } - # despite being smart, this washing machine seems to invert the endianness - # for the 'secs' field, so `00 01` (1s) becomes `01 00` (256s) - assert request.dhcp['secs'] == 256 - assert request.eth_src == washing_mac - assert request.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert request.ip_src == '0.0.0.0' - assert (request.sport, request.dport) == (68, 67) - - -def test_netatmo_discover_request(pcap: PcapFile): - disco1, disco2, request = parse_pcap(pcap, expected_packets=3) - '''Packets from a netatmo weather station. - - it uses a embedded stack (lwip), so packets are minimalistic. - ''' - # their source says: - # we don't need the broadcast flag since we can receive unicast traffic - # before being fully configured! - assert ( - disco1.dhcp['flags'] - == disco2.dhcp['flags'] - == request.dhcp['flags'] - == bootp.Flag.UNICAST - ) - - # they do not seem to increment the `secs` field with time - assert ( - disco1.dhcp['secs'] == disco2.dhcp['secs'] == request.dhcp['secs'] == 0 - ) - - # they do not pass a client_id in options - assert ( - disco1.dhcp['options'] - == disco2.dhcp['options'] - == { - 'message_type': dhcp.MessageType.DISCOVER, - 'max_msg_size': 1500, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.BROADCAST_ADDRESS, - dhcp.Option.NAME_SERVER, - ], - } - ) - assert request.dhcp['options'] == { - 'message_type': dhcp.MessageType.REQUEST, - 'max_msg_size': 1500, - 'requested_ip': '192.168.0.8', - 'server_id': '192.168.0.254', - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.BROADCAST_ADDRESS, - dhcp.Option.NAME_SERVER, - ], - 'host_name': 'Netatmo-Personal-Weather-Station', - } - - -def test_invalid_router_option( - pcap: PcapFile, caplog: pytest.LogCaptureFixture -): - '''Last opt. has an invalid length; the rest of the packet is decoded.''' - caplog.set_level('ERROR', logger='pyroute2.dhcp') - ack = parse_pcap(pcap, 1)[0] - assert caplog.messages == [ - 'Cannot decode option 3 as string: ' - 'unpack requires a buffer of 255 bytes' - ] - assert ack.dhcp['options'] == { - 'broadcast_address': '192.168.42.255', - 'domain_name': 'toulouse.fourcot.fr', - 'lease_time': 604800, - 'message_type': dhcp.MessageType.ACK, - 'name_server': ['192.168.42.10'], - 'rebinding_time': 529200, - 'renewal_time': 302400, - 'server_id': '192.168.42.10', - 'subnet_mask': '255.255.255.0', - } - - -def test_invalid_client_id_option( - pcap: PcapFile, caplog: pytest.LogCaptureFixture -): - '''The client id is declared as 8 bytes long instead of 7.''' - caplog.set_level('ERROR', logger='pyroute2.dhcp') - req = parse_pcap(pcap, 1)[0] - # Options before the failed one are still decoded - assert req.dhcp['options'] == { - 'max_msg_size': 1500, - 'message_type': dhcp.MessageType.REQUEST, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.CLASSLESS_STATIC_ROUTE, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.DOMAIN_NAME, - dhcp.Option.IPV6_ONLY_PREFERRED, - dhcp.Option.DHCP_CAPTIVE_PORTAL, - dhcp.Option.DOMAIN_SEARCH, - dhcp.Option.PRIVATE_PROXY_AUTODISCOVERY, - dhcp.Option.LDAP_SERVERS, - dhcp.Option.NETBIOS_NAME_SERVER, - dhcp.Option.NETBIOS_NODE_TYPE, - ], - } - - -@pytest.mark.parametrize( - ('offset', 'err_msg'), - ( - ( - 0, - 'Cannot decode ethmsg dst: unpack_from requires a buffer ' - 'of at least 6 bytes for unpacking 6 bytes at offset 0', - ), - ( - 1, - 'Cannot decode ethmsg dst: unpack_from requires a buffer ' - 'of at least 6 bytes for unpacking 6 bytes at offset 0', - ), - ( - 8, - 'Cannot decode ethmsg src: unpack_from requires a buffer ' - 'of at least 12 bytes for unpacking 6 bytes at offset 6', - ), - ( - 20, - 'Cannot decode ip4msg flags: unpack_from requires a buffer ' - 'of at least 22 bytes for unpacking 2 bytes at offset 20', - ), - ( - 30, - 'Cannot decode ip4msg dst: unpack_from requires a buffer ' - 'of at least 34 bytes for unpacking 4 bytes at offset 30', - ), - ( - 40, - 'Cannot decode udpmsg csum: unpack_from requires a buffer ' - 'of at least 42 bytes for unpacking 2 bytes at offset 40', - ), - ( - 50, - 'Cannot decode dhcp4msg secs: unpack_from requires a buffer ' - 'of at least 52 bytes for unpacking 2 bytes at offset 50', - ), - ( - 60, - 'Cannot decode dhcp4msg yiaddr: unpack_from requires a buffer ' - 'of at least 62 bytes for unpacking 4 bytes at offset 58', - ), - ( - 70, - 'Cannot decode dhcp4msg chaddr: unpack_from requires a buffer ' - 'of at least 86 bytes for unpacking 16 bytes at offset 70', - ), - ( - 100, - 'Cannot decode dhcp4msg sname: unpack_from requires a buffer ' - 'of at least 150 bytes for unpacking 64 bytes at offset 86', - ), - ( - 200, - 'Cannot decode dhcp4msg file: unpack_from requires a buffer ' - 'of at least 278 bytes for unpacking 128 bytes at offset 150', - ), - ( - 280, - 'Cannot decode dhcp4msg cookie: unpack_from requires a buffer ' - 'of at least 282 bytes for unpacking 4 bytes at offset 278', - ), - ), -) -def test_truncated_packet(pcap: PcapFile, offset: int, err_msg: str): - '''Truncated packets raise ValueError when decoded.''' - with pytest.raises(ValueError) as err_ctx: - parse_pcap([i[:offset] for i in pcap], 1)[0] - assert str(err_ctx.value) == err_msg + f' (actual buffer size is {offset})' - - -def test_android_tethering_renew(pcap: PcapFile): - '''Renew process for a Debian pc connected to an Android phone.''' - client_mac = 'a0:a4:c5:93:ac:60' - client_ip = '192.168.72.168' - request, ack = parse_pcap(pcap, expected_packets=2) - assert request.message_type == dhcp.MessageType.REQUEST - assert request.eth_src == request.dhcp['chaddr'] == client_mac - assert request.ip_src == request.dhcp['ciaddr'] == client_ip - assert request.dhcp['secs'] == 1 - assert request.dhcp['options'] == { - 'client_id': {'key': 'a0:a4:c5:93:ac:60', 'type': 1}, - 'host_name': 'thinkpad-eno', - 'max_msg_size': 0xFFFF, - 'message_type': dhcp.MessageType.REQUEST, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.TIME_OFFSET, - dhcp.Option.NAME_SERVER, - dhcp.Option.HOST_NAME, - dhcp.Option.DOMAIN_NAME, - dhcp.Option.INTERFACE_MTU, - dhcp.Option.BROADCAST_ADDRESS, - dhcp.Option.CLASSLESS_STATIC_ROUTE, - dhcp.Option.ROUTER, - dhcp.Option.STATIC_ROUTE, - dhcp.Option.NIS_DOMAIN, - dhcp.Option.NIS_SERVERS, - dhcp.Option.NTP_SERVERS, - dhcp.Option.DOMAIN_SEARCH, - dhcp.Option.PRIVATE_CLASSIC_ROUTE_MS, - dhcp.Option.PRIVATE_PROXY_AUTODISCOVERY, - dhcp.Option.ROOT_PATH, - ], - } - - assert ack.message_type == dhcp.MessageType.ACK - assert ack.eth_dst == ack.dhcp['chaddr'] == client_mac - assert ack.ip_dst == ack.dhcp['ciaddr'] == client_ip - assert ack.dhcp['options'] == { - 'broadcast_address': '192.168.72.255', - 'host_name': 'thinkpad-eno', - 'lease_time': 3599, - 'message_type': dhcp.MessageType.ACK, - 'name_server': ['192.168.72.238'], - 'rebinding_time': 3149, - 'renewal_time': 1799, - 'router': ['192.168.72.238'], - 'server_id': '192.168.72.238', - 'subnet_mask': '255.255.255.0', - 'vendor_specific_information': 'ANDROID_METERED', - } - - -def test_huawei_discover_option_148(pcap: PcapFile): - '''A DHCP offer with a non-standard option 148.''' - discover, offer = parse_pcap(pcap, expected_packets=2) - assert discover.message_type == dhcp.MessageType.DISCOVER - assert discover.eth_src == 'a4:7c:c9:aa:20:20' - assert discover.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert discover.ip_src == '0.0.0.0' - assert discover.ip_dst == '255.255.255.255' - assert discover.dhcp['flags'] == bootp.Flag.UNICAST - assert discover.dhcp['options'] == { - 'client_id': {'key': 'a4:7c:c9:aa:20:20', 'type': 1}, - 'max_msg_size': 1464, - 'message_type': dhcp.MessageType.DISCOVER, - 'parameter_list': [ - dhcp.Option.SUBNET_MASK, - dhcp.Option.ROUTER, - dhcp.Option.NAME_SERVER, - dhcp.Option.DOMAIN_NAME, - dhcp.Option.BROADCAST_ADDRESS, - dhcp.Option.STATIC_ROUTE, - dhcp.Option.VENDOR_SPECIFIC_INFORMATION, - dhcp.Option.NETBIOS_NAME_SERVER, - dhcp.Option.CLASSLESS_STATIC_ROUTE, - dhcp.Option.DOTS_ADDR, - 184, # this option is not assigned, Huawei calls it "option184"... - ], - 'vendor_id': 'huawei AirEngine5761-11', - } - - assert offer.message_type == dhcp.MessageType.OFFER - assert offer.eth_src == 'b0:41:6f:06:26:14' - assert offer.eth_dst == 'a4:7c:c9:aa:20:20' - assert offer.ip_src == '10.184.16.1' - # since the ap made an unicast DISCOVER, the offer is unicast - assert offer.ip_dst == '10.184.27.103' - assert offer.dhcp['options'] == { - 'broadcast_address': '10.184.31.255', - # FIXME: this is not the correct format for the DOTS_ADDR option, - # which is defined in RFC 8973. - # If we implemented a parser for this option, it would crash and skip - # the huawei value, which we don't want either... - 'dots_addr': list( - b'agilemode=agile-cloud;' - b'agilemanage-mode=ip;' - b'agilemanage-domain=48.194.254.137;' - b'agilemanage-port=10021;' - ), - 'lease_time': 604800, - 'message_type': dhcp.MessageType.OFFER, - 'name_server': ['10.184.16.1'], - 'rebinding_time': 529200, - 'renewal_time': 302400, - 'router': ['10.184.16.1'], - 'server_id': '10.184.16.1', - 'subnet_mask': '255.255.240.0', - } diff --git a/tests/test_linux/test_dhcp/test_server_detector.py b/tests/test_linux/test_dhcp/test_server_detector.py deleted file mode 100644 index 5118d0172..000000000 --- a/tests/test_linux/test_dhcp/test_server_detector.py +++ /dev/null @@ -1,269 +0,0 @@ -import asyncio -import json -from operator import itemgetter - -import pytest -from fixtures.dhcp_servers.dnsmasq import DnsmasqConfig, DnsmasqFixture -from fixtures.dhcp_servers.udhcpd import UdhcpdFixture -from fixtures.interfaces import DHCPRangeConfig, VethPair -from pr2test.marks import require_root -from test_dhcp.conftest import parse_stdout_leases - -from pyroute2.dhcp.enums import dhcp -from pyroute2.iproute.linux import AsyncIPRoute - -pytestmark = [pytest.mark.asyncio, require_root()] - - -async def test_detect_dnsmasq_once( - dnsmasq: DnsmasqFixture, veth_pair: VethPair -): - process = await asyncio.create_subprocess_exec( - 'dhcp-server-detector', - veth_pair.client, - '--exit-on-first-offer', - '--log-level=INFO', - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=5) - assert process.returncode == 0 - - # check the offer was properly encoded - offer = json.loads(stdout) - assert offer['interface'] == veth_pair.client - msg = offer['message'] - assert msg['dhcp']['options']['message_type'] == dhcp.MessageType.OFFER - assert msg['dhcp']['options']['server_id'] == str( - dnsmasq.config.range.router - ) - assert msg['dport'] == 68 - assert msg['sport'] == 67 - assert msg['ip_dst'] == '255.255.255.255' - assert msg['eth_dst'] == 'ff:ff:ff:ff:ff:ff' - - # check we have the expected logs - logs = [i.decode() for i in stderr.splitlines()] - assert len(logs) == 2 - assert f'[{veth_pair.client}] -> DISCOVER' in logs[0] - assert f'[{veth_pair.client}] <- OFFER from ' in logs[1] - - -async def test_detect_udhcpd_multiple( - udhcpd: UdhcpdFixture, veth_pair: VethPair -): - process = await asyncio.create_subprocess_exec( - 'dhcp-server-detector', - veth_pair.client, - # this should send 2 messages and stop - '--duration=0.9', - '--interval=0.5', - '--log-level=INFO', - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=5) - assert process.returncode == 0 - - # dont bother decoding the json here, we just check we have 2 messages - assert stdout.decode().count(f'"interface": "{veth_pair.client}"') == 2 - - # check we have the expected logs - logs = [i.decode() for i in stderr.splitlines()] - assert len(logs) == 4 - assert '-> DISCOVER' in logs[0] - assert '<- OFFER' in logs[1] - assert '-> DISCOVER' in logs[2] - assert '<- OFFER' in logs[3] - - -async def test_detect_no_response(veth_pair: VethPair): - '''The detector exits on error when there is no response.''' - process = await asyncio.create_subprocess_exec( - 'dhcp-server-detector', - veth_pair.client, - '--duration=1', - '--interval=0.9', - '--log-level=INFO', - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=5) - assert process.returncode == 1 # no response, exited on error - assert not stdout # no lease was written - assert stderr.count(b"DISCOVER") == 2 # 2 requests sent - - -async def test_detect_wrong_interface(): - '''The only passed interface does not exist.''' - process = await asyncio.create_subprocess_exec( - 'dhcp-server-detector', - 'definitely_does_not_exist', - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=5) - assert process.returncode == 1 - assert not stdout - assert ( - "'definitely_does_not_exist': [Errno 2] Link not found" - ) in stderr.decode() - - -async def test_interface_goes_down_during_detection( - udhcpd: UdhcpdFixture, veth_pair: VethPair -): - '''The interface goes down after a response has been received.''' - process = await asyncio.create_subprocess_exec( - 'dhcp-server-detector', - veth_pair.client, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - - # TODO put in a fixture - async def shutdown_iface(ifindex: int): - '''Shuts down the interface after udhcpd sent an offer.''' - await asyncio.wait_for(udhcpd.wait_for_log('OFFER'), timeout=2) - async with AsyncIPRoute() as ipr: - await ipr.link("set", index=ifindex, state='down') - - shutdown_task = asyncio.create_task( - shutdown_iface(ifindex=veth_pair.client_idx) - ) - - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=3) - await shutdown_task - # we still got a response, so exit is successful - assert process.returncode == 0 - offer = json.loads(stdout) - assert offer['interface'] == veth_pair.client - msg = offer['message'] - assert msg['dhcp']['options']['message_type'] == dhcp.MessageType.OFFER - # there is 1 error log - assert stderr.count(b'ERROR') == 1 - assert ( - f'{veth_pair.client!r}: [Errno 100] Network is down' - ) in stderr.decode() - - -@pytest.mark.parametrize( - ('run_dhcp_server_outside_vlan',), ((True,), (False,)) -) -async def test_detect_with_vlan( - udhcpd: UdhcpdFixture, - veth_pair: VethPair, - async_ipr: AsyncIPRoute, - caplog: pytest.LogCaptureFixture, - run_dhcp_server_outside_vlan: bool, - async_context, -): - '''Get an offer from dnsmasq over a vlan, - and maybe another offer from udhcpd outside of it. - - The vlan on which dnsmasq listens is over the veth pair. - The socket listening outside of the vlan should *not* receive - the response sent by dnsmasq. - ''' - - if run_dhcp_server_outside_vlan is False: - # stop udhcpd, which runs on the veth directly - await udhcpd.__aexit__(None, None, None) - # now only dnsmasq will remain, listening on the vlan - - # configure a dhcp range for the vlan - dnsmasq_range = DHCPRangeConfig( - start='192.168.11.10', - end='192.168.11.20', - router='192.168.11.1', - broadcast='192.168.11.255', - netmask='255.255.255.0', - ) - # create a pair of vlan interfaces over the veth pair - vlan_id = 151 - srv_vlan_name = f'srv.{vlan_id}' - cli_vlan_name = f'cli.{vlan_id}' - await async_ipr.link( - 'add', - ifname=srv_vlan_name, - kind='vlan', - link=veth_pair.server_idx, - vlan_id=vlan_id, - ) - await async_ipr.link( - 'add', - ifname=cli_vlan_name, - kind='vlan', - link=veth_pair.client_idx, - vlan_id=vlan_id, - ) - srv_vlan_idx = (await async_ipr.link_lookup(ifname=srv_vlan_name))[0] - cli_vlan_idx = (await async_ipr.link_lookup(ifname=cli_vlan_name))[0] - - # add an ip address only on the server end - await async_ipr.addr( - 'add', - index=srv_vlan_idx, - address=str(dnsmasq_range.router), - prefixlen=24, - ) - - # bring up both interfaces - await async_ipr.link('set', index=srv_vlan_idx, state='up') - await async_ipr.link('set', index=cli_vlan_idx, state='up') - - # run dnsmasq over the vlan - dnsmasq_cfg = DnsmasqConfig(range=dnsmasq_range, interface=srv_vlan_name) - async with DnsmasqFixture(dnsmasq_cfg): - # start the server detector - process = await asyncio.create_subprocess_exec( - 'dhcp-server-detector', - veth_pair.client, - cli_vlan_name, - # this should send 1 discover and stop - '--duration=0.9', - '--interval=1.0', - '--log-level=DEBUG', - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await asyncio.wait_for( - process.communicate(), timeout=3 - ) - # we got at least one offer, so the detector returns 0 - assert process.returncode == 0 - # the vlan interface starts with "c" so it's first when sorted this way - offers = sorted(parse_stdout_leases(stdout), key=itemgetter('interface')) - - assert offers - # we always get an offer from dnsmasq on the vlan - vlan_offer = offers[0] - assert vlan_offer['interface'] == cli_vlan_name - assert vlan_offer['message']['dhcp']['options']['server_id'] == str( - dnsmasq_range.router - ) - - if run_dhcp_server_outside_vlan: - # 2 servers, 2 leases - assert len(offers) == 2 - # check the offer sent by udhcpd - non_vlan_offer = offers[1] - assert non_vlan_offer['interface'] == veth_pair.client - assert non_vlan_offer['message']['dhcp']['options'][ - 'server_id' - ] == str(udhcpd.config.range.router) - assert ( - vlan_offer['message']['dhcp']['xid'] - != non_vlan_offer['message']['dhcp']['xid'] - ) - else: - # we stopped udhcpd so only got an offer from dnsmasq - assert len(offers) == 1 - - # The bpf filter drops packets that are intended for vlans; failing that, - # we would receive a copy of all packets meant for "upper" vlans when - # on a non-vlan interface. - # since we have a different xid per interface, they're discarded, - # but should not happen anyway. - # So if this assert fails it means the BPF filter does not work - assert b'Got OFFER with xid mismatch, ignoring' not in stderr diff --git a/tests/test_linux/test_dhcp/test_unit.py b/tests/test_linux/test_dhcp/test_unit.py deleted file mode 100644 index cdfe6d503..000000000 --- a/tests/test_linux/test_dhcp/test_unit.py +++ /dev/null @@ -1,437 +0,0 @@ -import asyncio -from typing import Callable - -import pytest -from fixtures.dhcp_servers.mock import MockDHCPServerFixture -from fixtures.interfaces import VethPair - -from pyroute2.dhcp.client import ( - AsyncDHCPClient, - ClientConfig, - randomized_increasing_backoff, -) -from pyroute2.dhcp.dhcp4msg import dhcp4msg -from pyroute2.dhcp.enums import bootp, dhcp -from pyroute2.dhcp.fsm import State -from pyroute2.dhcp.leases import JSONFileLease - -pytestmark = pytest.mark.asyncio - - -@pytest.mark.parametrize( - ('dhcp_client_host_name',), (('nebuchadnezzar',), (None,)) -) -@pytest.mark.parametrize( - ('dhcp_client_vendor_id',), (('fake-vendor-id',), (None,)) -) -async def test_get_and_renew_lease( - mock_dhcp_server: MockDHCPServerFixture, - set_fixed_xid: Callable[[int], None], - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, -): - '''A lease is obtained with a 1s renewing time, the client renews it. - - The test pcap file contains the OFFER & the 2 ACKs. - ''' - caplog.set_level('INFO') - - expected_vid_and_hostname = {} - if vid := client_config.vendor_id: - # vendor id must be present only if configured - expected_vid_and_hostname['vendor_id'] = vid - if hn := client_config.host_name: - # same for host_name - expected_vid_and_hostname['host_name'] = hn - - # Make xids non random so they match the ones in the pcap - set_fixed_xid(0x12345670) - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(State.SELECTING, timeout=1) - # server sends an OFFER - await cli.wait_for_state(State.REQUESTING, timeout=1) - # server sends an ACK - await cli.wait_for_state(State.BOUND, timeout=1) - # the ACK in the pcap was modified to set a renewing time of 1s - await cli.wait_for_state(State.RENEWING, timeout=2) - await cli.wait_for_state(State.BOUND, timeout=1) - - assert len(mock_dhcp_server.decoded_requests) == 4 - discover, request, renew_request, release = ( - mock_dhcp_server.decoded_requests - ) - - # First, the client sends a discover: - assert discover.message_type == dhcp.MessageType.DISCOVER - # This is a broadcast message - assert discover.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert discover.ip_dst == '255.255.255.255' - assert discover.ip_src == '0.0.0.0' - - assert discover.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert discover.dhcp['flags'] == bootp.Flag.BROADCAST - # all bootp ip addr fields are left blank - assert all([discover.dhcp[f'{x}iaddr'] == '0.0.0.0' for x in 'cysg']) - # the requested parameters match those in the client config - assert discover.dhcp['options']['parameter_list'] == list( - client_config.requested_parameters - ) - assert discover.sport, release.dport == (68, 67) - - # The pcap contains an offer in response to the discover. - # The client sends a request for that offer: - assert request.message_type == dhcp.MessageType.REQUEST - # This is a broadcast message - assert request.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert request.ip_dst == '255.255.255.255' - assert request.ip_src == '0.0.0.0' - assert request.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert request.dhcp['flags'] == bootp.Flag.BROADCAST - assert request.dhcp['options'] == { - 'client_id': {'key': request.eth_src, 'type': 1}, - 'message_type': dhcp.MessageType.REQUEST, - 'parameter_list': list(client_config.requested_parameters), - 'requested_ip': '192.168.186.73', - 'server_id': '192.168.186.1', - **expected_vid_and_hostname, - } - assert request.sport, release.dport == (68, 67) - - # the server sends an ACK and the client is bound. - - # a while later (actually 1 sec.), the client sends - # a new REQUEST to renew its lease and switches to RENEWING - assert renew_request.message_type == dhcp.MessageType.REQUEST - # it's an unicast request - assert renew_request.dhcp['flags'] == bootp.Flag.UNICAST - assert renew_request.eth_dst == '2e:7e:7d:8e:5f:5f' - assert renew_request.ip_dst == '192.168.186.1' - assert renew_request.ip_src == '192.168.186.73' - # no server id nor requested ip in this case - assert renew_request.dhcp['options'] == { - 'client_id': {'key': renew_request.eth_src, 'type': 1}, - 'message_type': dhcp.MessageType.REQUEST, - 'parameter_list': list(client_config.requested_parameters), - **expected_vid_and_hostname, - } - assert renew_request.sport, release.dport == (68, 67) - - # since we stopped the client, it sends a RELEASE (unicast too) - assert release.message_type == dhcp.MessageType.RELEASE - assert release.dhcp['flags'] == bootp.Flag.UNICAST - assert release.eth_dst == '2e:7e:7d:8e:5f:5f' - assert renew_request.ip_dst == '192.168.186.1' - assert renew_request.ip_src == '192.168.186.73' - assert release.dhcp['options'] == { - 'client_id': {'key': release.eth_src, 'type': 1}, - 'message_type': dhcp.MessageType.RELEASE, - 'server_id': '192.168.186.1', - **expected_vid_and_hostname, - } - assert release.sport, release.dport == (68, 67) - - -async def test_ack_invalid_request_state( - mock_dhcp_server: MockDHCPServerFixture, - set_fixed_xid: Callable[[int], None], - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, -): - '''An ack received with an invalid request state in its xid - must cause a warning to be logged and the lease to be ignored. - ''' - caplog.set_level('WARNING') - # Make xids non random so they match the ones in the pcap - set_fixed_xid(0x12345670) - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(State.SELECTING, timeout=1) - # server sends an OFFER - await cli.wait_for_state(State.REQUESTING, timeout=1) - # server sends an ACK, which is ignored by the client - await asyncio.sleep(0.2) - assert cli.lease is None - - assert caplog.messages == ['Invalid request state for xid 0x1234567f'] - assert len(mock_dhcp_server.decoded_requests) == 2 - discover, request = mock_dhcp_server.decoded_requests - assert discover.message_type == dhcp.MessageType.DISCOVER - assert request.message_type == dhcp.MessageType.REQUEST - - -async def test_init_reboot_nak( - mock_dhcp_server: MockDHCPServerFixture, - client_config: ClientConfig, - veth_pair: VethPair, - caplog: pytest.LogCaptureFixture, - set_fixed_xid: Callable[[int], None], -): - '''The server doesn't like the requested IP in INIT-REBOOT. - - It sends a NAK and the client goes back to INIT and gets a new lease. - ''' - set_fixed_xid(0xDD435A20) - caplog.set_level('INFO') - # Create a fake lease to start the client in INIT-REBOOT - old_lease = JSONFileLease( - ack=dhcp4msg( - { - 'op': bootp.MessageType.BOOTREPLY, - 'flags': bootp.Flag.BROADCAST, - 'yiaddr': '192.168.186.73', - 'chaddr': '72:c1:55:6f:76:83', - 'options': { - 'message_type': 5, - 'server_id': '192.168.186.1', - 'lease_time': 1, - }, - } - ), - interface=veth_pair.client, - server_mac='2e:7e:7d:8e:5f:5f', - ) - old_lease.dump() - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - # The client loaded the lease we just wrote and sents a REQUEST - await cli.wait_for_state(State.REBOOTING, timeout=1) - # The server sends a NAK, so the client goes back to INIT - await cli.wait_for_state(State.INIT, timeout=1) - await cli.wait_for_state(State.SELECTING, timeout=1) - # The server sends an OFFER an the client requests it - await cli.wait_for_state(State.REQUESTING, timeout=1) - # The servers ACKs the request and we're bound ! - await cli.wait_for_state(State.BOUND, timeout=1) - - assert len(mock_dhcp_server.decoded_requests) == 4 - request1, discover, request2, release = mock_dhcp_server.decoded_requests - assert request1.message_type == dhcp.MessageType.REQUEST - assert request1.dhcp['flags'] == bootp.Flag.BROADCAST - assert request1.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert request1.dhcp['options']['requested_ip'] == old_lease.ip - assert request1.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert request1.ip_dst == '255.255.255.255' - assert request1.ip_src == '0.0.0.0' - - assert discover.message_type == dhcp.MessageType.DISCOVER - assert discover.dhcp['flags'] == bootp.Flag.BROADCAST - assert discover.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert discover.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert discover.ip_dst == '255.255.255.255' - assert discover.ip_src == '0.0.0.0' - - assert request2.message_type == dhcp.MessageType.REQUEST - assert request2.dhcp['flags'] == bootp.Flag.BROADCAST - assert request2.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert request2.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert request2.ip_dst == '255.255.255.255' - assert request2.ip_src == '0.0.0.0' - - assert release.message_type == dhcp.MessageType.RELEASE - assert release.dhcp['flags'] == bootp.Flag.UNICAST - assert release.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert release.eth_dst == '2e:7e:7d:8e:5f:5f' - assert release.ip_dst == '192.168.186.1' - assert release.ip_src == '192.168.186.85' - - -async def test_requesting_timeout( - mock_dhcp_server: MockDHCPServerFixture, - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, - set_fixed_xid: Callable[[int], None], -): - '''The client resets itself after a timeout in the REQUESTING state.''' - set_fixed_xid(0xDD435A20) - caplog.set_level('INFO') - # Timeout after 1s when requesting an offer and no answer - client_config.timeouts[State.REQUESTING] = 1 - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - # The client sends a DISCOVER, the servers sends an OFFER - await cli.wait_for_state(State.SELECTING, timeout=1) - await cli.wait_for_state(State.REQUESTING, timeout=1) - # and then the server nevers send an ack - # the client goes back to SELECTING after the timeout - await cli.wait_for_state(State.SELECTING, timeout=3) - - # the client has reset - assert 'Resetting after 1.0 seconds' in caplog.messages - - assert len(mock_dhcp_server.decoded_requests) == 2 - discover, request = mock_dhcp_server.decoded_requests - - assert discover.message_type == dhcp.MessageType.DISCOVER - assert discover.dhcp['flags'] == bootp.Flag.BROADCAST - assert discover.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert discover.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert discover.ip_dst == '255.255.255.255' - assert discover.ip_src == '0.0.0.0' - assert discover.dhcp['options']['parameter_list'] == list( - client_config.requested_parameters - ) - - assert request.message_type == dhcp.MessageType.REQUEST - assert request.dhcp['flags'] == bootp.Flag.BROADCAST - assert request.dhcp['op'] == bootp.MessageType.BOOTREQUEST - assert request.eth_dst == 'ff:ff:ff:ff:ff:ff' - assert request.ip_dst == '255.255.255.255' - assert request.ip_src == '0.0.0.0' - assert request.dhcp['options']['parameter_list'] == list( - client_config.requested_parameters - ) - - -async def test_wait_for_state_timeout(client_config: ClientConfig): - '''wait_for_state() can timeout after a given delay''' - async with AsyncDHCPClient(client_config) as cli: - with pytest.raises(asyncio.exceptions.TimeoutError) as err_ctx: - await cli.wait_for_state(State.BOUND, timeout=0.2) - assert ( - str(err_ctx.value) - == 'Timed out waiting for the BOUND state. Current state: INIT' - ) - - -async def test_offer_wrong_xid( - client_config: ClientConfig, - mock_dhcp_server: MockDHCPServerFixture, - set_fixed_xid: Callable[[int], None], - caplog: pytest.LogCaptureFixture, -): - '''The client discards & logs packets with an unknown xid. - - Since we just need a dhcp offer, the pcap for this test - is a symlink to the one for test_requesting_timeout - ''' - set_fixed_xid(0x98765432) - caplog.set_level('ERROR') - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(State.SELECTING, timeout=1) - # wait a tiny bit for the offer to arrive - await asyncio.sleep(0.5) - assert caplog.messages == [ - 'Incorrect xid 0xdd435a25 (expected 0x9876543X), discarding' - ] - - assert len(mock_dhcp_server.decoded_requests) == 1 - discover = mock_dhcp_server.decoded_requests[0] - assert discover.message_type == dhcp.MessageType.DISCOVER - - -async def test_unknown_message( - client_config: ClientConfig, - mock_dhcp_server: MockDHCPServerFixture, - caplog: pytest.LogCaptureFixture, - set_fixed_xid: Callable[[int], None], -): - '''Unknown messages must be discarded.''' - set_fixed_xid(0x8F506FD0) - caplog.set_level('DEBUG', logger='pyroute2.dhcp.client') - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - # wait a tiny bit for the packet to arrive - await asyncio.sleep(0.5) - # the capture contains a request, which the client does not handle - assert 'DHCP REQUEST messages are not handled' - assert len(mock_dhcp_server.decoded_requests) == 1 - - -async def test_wrong_state_change(client_config: ClientConfig): - '''One cannot trigger a state change like that.''' - async with AsyncDHCPClient(client_config) as cli: - with pytest.raises(ValueError) as err_ctx: - await cli.transition(State.BOUND) - assert str(err_ctx.value) == 'Cannot transition from INIT to BOUND' - - -async def test_unexpected_dhcp_message( - client_config: ClientConfig, - mock_dhcp_server: MockDHCPServerFixture, - set_fixed_xid: Callable[[int], None], - caplog: pytest.LogCaptureFixture, -): - '''Client sends a DISCOVER, the server sends an ACK, it is ignored.''' - caplog.set_level('DEBUG', logger='pyroute2.dhcp') - set_fixed_xid(0x12345670) - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await cli.wait_for_state(State.SELECTING, timeout=1) - # TODO: if we want to avoid a sleep here, we should include an - # OFFER and another ACK in the pcap, so we can simply wait for - # the client to be bound. - await asyncio.sleep(0.2) - # The client is still SELECTING: it didn't receive an OFFER - assert cli.state == State.SELECTING - assert ( - 'Ignoring call to \'ack_received\' in SELECTING state' - in caplog.messages - ) - assert len(mock_dhcp_server.decoded_requests) == 1 - discover = mock_dhcp_server.decoded_requests[0] - assert discover.message_type == dhcp.MessageType.DISCOVER - - -async def test_backoff(): - '''Test that the function generating wait times works as expected.''' - backoff = randomized_increasing_backoff() - wait_times = [next(backoff) for _ in range(100)] - assert min(wait_times) == 4.0 - assert max(wait_times) == 32.0 - assert len([i for i in wait_times if 4.0 < i < 32.0]) > 2 - assert sorted(wait_times) == wait_times - - -@pytest.mark.parametrize( - 'bad_lease_data', - ( - 'not json', - '{"unexpected": "json"}', - '"valid json (?) but still unexpected"', - ), -) -async def test_corrupted_lease_file( - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, - bad_lease_data: str, -): - caplog.set_level('WARNING', logger='pyroute2.dhcp.client') - JSONFileLease._get_path(client_config.interface).write_text(bad_lease_data) - async with AsyncDHCPClient(client_config) as cli: - assert cli.lease is None - assert len(caplog.messages) == 1 - assert caplog.messages[0].startswith('Error loading lease: ') - - -@pytest.mark.parametrize( - ('truncate_offset', 'err_prefix'), - ( - (30, 'Cannot decode ip4msg dst'), - (150, 'Cannot decode dhcp4msg file'), - (300, 'Cannot decode option 58 as >i'), - (330, 'Cannot decode option 6 as string'), - ), -) -async def test_truncated_packet( - client_config: ClientConfig, - caplog: pytest.LogCaptureFixture, - mock_dhcp_server: MockDHCPServerFixture, - set_fixed_xid: Callable[[int], None], - truncate_offset: int, - err_prefix: str, -): - '''Test the client does not crash on invalid/truncated packets.''' - mock_dhcp_server.truncate_at = truncate_offset - caplog.set_level('ERROR', logger='pyroute2.dhcp.client') - set_fixed_xid(0x8057FB00) - async with AsyncDHCPClient(client_config) as cli: - await cli.bootstrap() - await asyncio.sleep(0.2) - # check nothing crashed - assert cli.state == State.SELECTING - assert not cli._receiver_task.done() - assert not cli._sender_task.done() - assert len(caplog.messages) == 1 - assert caplog.messages[0].startswith(err_prefix) diff --git a/tests/test_linux/test_diag.py b/tests/test_linux/test_diag.py index c2499c2d3..d937ddad9 100644 --- a/tests/test_linux/test_diag.py +++ b/tests/test_linux/test_diag.py @@ -1,4 +1,4 @@ -from socket import AF_INET +from socket import AF_UNIX from pr2test.marks import require_root @@ -15,17 +15,16 @@ def test_basic(): with DiagSocket() as ds: ds.bind() - sstats = ds.get_sock_stats(family=AF_INET) + sstats = ds.get_sock_stats(family=AF_UNIX) for s in sstats: - sstats_set.add(s['idiag_inode']) + sstats_set.add(s['udiag_ino']) - with open('/proc/net/tcp') as fd: + with open('/proc/net/unix') as fd: for line in fd.readlines(): line = line.split() try: - pstats_set.add(int(line[9])) + pstats_set.add(int(line[6])) except ValueError: pass - assert len(sstats_set - pstats_set) < 10 - assert len(pstats_set - sstats_set) < 10 + assert sstats_set == pstats_set diff --git a/tests/test_linux/test_dquot/test_dquot.py b/tests/test_linux/test_dquot/test_dquot.py index f88aec7b1..fd9749b89 100644 --- a/tests/test_linux/test_dquot/test_dquot.py +++ b/tests/test_linux/test_dquot/test_dquot.py @@ -1,10 +1,20 @@ import os import subprocess +import sys import pytest +from pr2test.marks import require_root from pyroute2 import DQuotSocket +pytestmark = [ + pytest.mark.skipif( + sys.version_info < (3, 7), + reason='the test module requires Python > 3.6', + ), + require_root(), +] + class DQuotContextManager: def __init__(self): diff --git a/tests/test_linux/test_ethtool.py b/tests/test_linux/test_ethtool.py index 3b6bded94..5b1071f4b 100644 --- a/tests/test_linux/test_ethtool.py +++ b/tests/test_linux/test_ethtool.py @@ -12,7 +12,7 @@ def get_fds(): os.close(fd) -def _test_pipe_leak(): +def test_pipe_leak(): fds = get_fds() etht = Ethtool() etht.close() @@ -20,7 +20,7 @@ def _test_pipe_leak(): assert get_fds() == fds -def _test_context_manager(): +def test_context_manager(): fds = get_fds() with Ethtool(): pass diff --git a/tests/test_linux/test_generic/test_l2tp.py b/tests/test_linux/test_generic/test_l2tp.py index b5d0cd7d1..b316e1aee 100644 --- a/tests/test_linux/test_generic/test_l2tp.py +++ b/tests/test_linux/test_generic/test_l2tp.py @@ -39,6 +39,7 @@ def l2ctx(context): context.l2tp.close() +@pytest.mark.xfail(reason='flaky test, only to collect failure logs') def test_complete(l2ctx): # 1. create tunnel l2ctx.l2tp.create_tunnel( @@ -90,8 +91,9 @@ def test_complete(l2ctx): try: l2ctx.l2tp.get_session(tunnel_id=2324, session_id=3435) except NetlinkError: - break - time.sleep(0.1) + time.sleep(0.1) + continue + break else: raise Exception('could not remove L2TP session') @@ -101,7 +103,8 @@ def test_complete(l2ctx): try: l2ctx.l2tp.get_tunnel(tunnel_id=2324) except NetlinkError: - break - time.sleep(0.1) + time.sleep(0.1) + continue + break else: raise Exception('could not remove L2TP tunnel') diff --git a/tests/test_linux/test_ipdb.py b/tests/test_linux/test_ipdb.py new file mode 100644 index 000000000..c5cdac227 --- /dev/null +++ b/tests/test_linux/test_ipdb.py @@ -0,0 +1,88 @@ +import pytest +from pr2test.marks import require_root + +from pyroute2 import IPDB + +pytestmark = [require_root()] + + +@pytest.fixture +def ictx(context): + context.ipdb = IPDB(deprecation_warning=False) + yield context + context.ipdb.release() + + +def test_interface_dummy(ictx): + ifname = ictx.new_ifname + ipaddr = ictx.new_ipaddr + interface = ictx.ipdb.create(ifname=ifname, kind='dummy') + interface.up() + interface.add_ip(f'{ipaddr}/24') + interface.commit() + + ictx.ndb.interfaces.wait(action='add', ifname=ifname, timeout=3) + ictx.ndb.addresses.wait(action='add', address=ipaddr, timeout=3) + assert ictx.ndb.interfaces[ifname]['state'] == 'up' + assert ( + ictx.ndb.addresses.wait(action='add', address=ipaddr, prefixlen=24)[ + 'index' + ] + == interface['index'] + ) + + interface.del_ip(f'{ipaddr}/24') + interface.commit() + + ictx.ndb.addresses.wait( + action='remove', address=ipaddr, prefixlen=24, timeout=3 + ) + + +def test_interface_veth(ictx): + netns = ictx.new_nsname + ictx.ndb.sources.add(netns=netns) + v0 = ictx.new_ifname + v1 = ictx.new_ifname + + veth0 = ictx.ipdb.create(ifname=v0, kind='veth', peer=v1) + veth0.up() + veth0.commit() + + veth1 = ictx.ipdb.interfaces[v1] + veth1['net_ns_fd'] = netns + veth1.commit() + + ictx.ndb.interfaces.wait(ifname=v0, target='localhost', timeout=3) + ictx.ndb.interfaces.wait(ifname=v1, target=netns, timeout=3) + + +def test_interface_bridge(ictx): + ifname = ictx.new_ifname + + with ictx.ipdb.create(ifname=ifname, kind='bridge') as i: + i.up() + i['address'] = '00:11:22:33:44:55' + i['br_stp_state'] = 1 + i['br_forward_delay'] = 1000 + + i = ictx.ndb.interfaces.wait(ifname=ifname, timeout=3) + assert i['state'] == 'up' + assert i['address'] == '00:11:22:33:44:55' + assert i['br_stp_state'] == 1 + assert i['br_forward_delay'] == 1000 + + +def test_route_basic(ictx): + ipaddr = ictx.new_ipaddr + gateway = ictx.new_ipaddr + net = ictx.new_ip4net + ifname = ictx.default_interface.ifname + + with ictx.ipdb.interfaces[ifname] as i: + i.up() + i.add_ip(f'{ipaddr}/24') + + ictx.ipdb.routes.add( + gateway=gateway, dst=f'{net.network}/{net.netmask}' + ).commit() diff --git a/tests/test_linux/test_ipr/test_basic.py b/tests/test_linux/test_ipr/test_basic.py index 95e5b6eab..d38451c71 100644 --- a/tests/test_linux/test_ipr/test_basic.py +++ b/tests/test_linux/test_ipr/test_basic.py @@ -24,14 +24,22 @@ def test_close(): ipr.close() # Shouldn't be able to use the socket after closing with pytest.raises(socket.error): - ipr.link('get', index=1) + ipr.get_links() def test_fileno(): + ipr1 = IPRoute() + ipr2 = IPRoute(fileno=ipr1.fileno()) + + ipr1.close() with pytest.raises(OSError) as e: - IPRoute(fileno=2048) + ipr2.get_links() assert e.value.errno == 9 # sendto -> Bad file descriptor + with pytest.raises(OSError) as e: + ipr2.close() + assert e.value.errno == 9 # close -> Bad file descriptor + def test_get_policy_map(context): assert isinstance(context.ipr.get_policy_map(), dict) diff --git a/tests/test_linux/test_ipr/test_link.py b/tests/test_linux/test_ipr/test_link.py index c6b70aab7..25e734932 100644 --- a/tests/test_linux/test_ipr/test_link.py +++ b/tests/test_linux/test_ipr/test_link.py @@ -15,9 +15,9 @@ def test_updown_link(context): index, ifname = context.default_interface context.ipr.link('set', index=index, state='up') - assert tuple(context.ipr.get_links(ifname=ifname))[0]['flags'] & 1 + assert context.ipr.get_links(ifname=ifname)[0]['flags'] & 1 context.ipr.link('set', index=index, state='down') - assert not (tuple(context.ipr.get_links(ifname=ifname))[0]['flags'] & 1) + assert not (context.ipr.get_links(ifname=ifname)[0]['flags'] & 1) @skip_if_not_supported @@ -99,19 +99,19 @@ def test_link_arp_flag(context): index, _ = context.default_interface # by default dummy interface have NOARP set - assert tuple(context.ipr.get_links(index))[0]['flags'] & IFF_NOARP + assert context.ipr.get_links(index)[0]['flags'] & IFF_NOARP context.ipr.link('set', index=index, arp=True) - assert not tuple(context.ipr.get_links(index))[0]['flags'] & IFF_NOARP + assert not context.ipr.get_links(index)[0]['flags'] & IFF_NOARP context.ipr.link('set', index=index, arp=False) - assert tuple(context.ipr.get_links(index))[0]['flags'] & IFF_NOARP + assert context.ipr.get_links(index)[0]['flags'] & IFF_NOARP context.ipr.link('set', index=index, noarp=False) - assert not tuple(context.ipr.get_links(index))[0]['flags'] & IFF_NOARP + assert not context.ipr.get_links(index)[0]['flags'] & IFF_NOARP context.ipr.link('set', index=index, noarp=True) - assert tuple(context.ipr.get_links(index))[0]['flags'] & IFF_NOARP + assert context.ipr.get_links(index)[0]['flags'] & IFF_NOARP @pytest.mark.parametrize('context', test_matrix, indirect=True) @@ -119,11 +119,11 @@ def test_symbolic_flags_ifinfmsg(context): index, _ = context.default_interface context.ipr.link('set', index=index, flags=['IFF_UP']) - iface = tuple(context.ipr.get_links(index))[0] + iface = context.ipr.get_links(index)[0] assert iface['flags'] & 1 assert 'IFF_UP' in iface.flags2names(iface['flags']) context.ipr.link('set', index=index, flags=['!IFF_UP']) - assert not (tuple(context.ipr.get_links(index))[0]['flags'] & 1) + assert not (context.ipr.get_links(index)[0]['flags'] & 1) @pytest.mark.parametrize('context', test_matrix, indirect=True) diff --git a/tests/test_linux/test_ipr/test_match.py b/tests/test_linux/test_ipr/test_match.py index 5ca9d9e80..7e703fe67 100644 --- a/tests/test_linux/test_ipr/test_match.py +++ b/tests/test_linux/test_ipr/test_match.py @@ -2,4 +2,4 @@ def test_match_callable(context): - assert len(tuple(context.ipr.get_links(match=partial(lambda x: x)))) > 0 + assert len(context.ipr.get_links(match=partial(lambda x: x))) > 0 diff --git a/tests/test_linux/test_ipr/test_netns.py b/tests/test_linux/test_ipr/test_netns.py index eca5e8dbd..20f13473a 100644 --- a/tests/test_linux/test_ipr/test_netns.py +++ b/tests/test_linux/test_ipr/test_netns.py @@ -1,8 +1,3 @@ -import ctypes -import errno -import os - -import pytest from pr2test.marks import require_root from pyroute2 import NetNS @@ -10,37 +5,6 @@ pytestmark = [require_root()] -def test_flags(context): - nsname = context.new_nsname - with pytest.raises(FileNotFoundError) as e: - NetNS(nsname, flags=0) - assert e.value.args[0] == errno.ENOENT - # 8<----------------------------------------------------- - ns = NetNS(nsname, flags=os.O_CREAT) - assert len([x.get('index') for x in ns.link('dump')]) > 0 - ns.close() - # 8<----------------------------------------------------- - ns = NetNS(nsname, flags=0) - assert len([x.get('index') for x in ns.link('dump')]) > 0 - ns.close() - - -def test_libc_id(context): - libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) - nsname = context.new_nsname - ns = NetNS(nsname, libc=libc) - assert id(ns.asyncore.libc) == id(libc) - ns.close() - - -def test_libc_fail_string(context): - nsname = context.new_nsname - with pytest.raises(AttributeError): - # if we pass a string instead of a libc object, the - # libc.mount() must fail with AttributeError - NetNS(nsname, libc='nonsense') - - def test_get_netns_info(context): nsname = context.new_nsname peer_name = context.new_ifname diff --git a/tests/test_linux/test_ipr/test_probe.py b/tests/test_linux/test_ipr/test_probe.py deleted file mode 100644 index df8b7a7cf..000000000 --- a/tests/test_linux/test_ipr/test_probe.py +++ /dev/null @@ -1,77 +0,0 @@ -import errno -from socket import AF_INET - -import pytest -from pr2test.context_manager import make_test_matrix, skip_if_not_supported -from pr2test.marks import require_root - -from pyroute2 import NetlinkError - -pytestmark = [require_root()] -test_matrix = make_test_matrix(targets=['local', 'netns']) - - -@pytest.mark.parametrize('context', test_matrix, indirect=True) -@skip_if_not_supported -def test_ping_ok(context): - index, ifname = context.default_interface - ipaddr = context.new_ipaddr - - context.ipr.addr('add', index=index, address=ipaddr, prefixlen=24) - context.ipr.link('set', index=index, state='up') - context.ipr.link( - 'set', index=context.ipr.link_lookup(ifname='lo'), state='up' - ) - - context.ndb.interfaces.wait(ifname=ifname, state='up') - context.ndb.interfaces.wait(ifname='lo', state='up') - - probes = [x for x in context.ipr.probe('add', kind='ping', dst=ipaddr)] - probe = probes[0] - - assert len(probes) == 1 - assert probe['family'] == AF_INET - assert probe['proto'] == 1 - assert probe['port'] == 0 - assert probe['dst_len'] == 32 - assert probe.get('dst') == ipaddr - assert probe.get('kind') == 'ping' - - -@pytest.mark.parametrize( - 'context', make_test_matrix(targets=['netns']), indirect=True -) -@skip_if_not_supported -def test_ping_fail_ehostunreach(context): - context.ipr.link( - 'set', index=context.ipr.link_lookup(ifname='lo'), state='down' - ) - with pytest.raises(NetlinkError) as e: - context.ipr.probe('add', kind='ping', dst='127.0.0.1') - assert e.value.code == errno.EHOSTUNREACH - - -@pytest.mark.parametrize( - 'context', make_test_matrix(targets=['netns']), indirect=True -) -@skip_if_not_supported -def test_ping_fail_etimedout(context): - index, ifname = context.default_interface - ipaddr = context.new_ipaddr - target = context.new_ipaddr - - context.ipr.addr('add', index=index, address=ipaddr, prefixlen=24) - context.ipr.link('set', index=index, state='up') - context.ipr.link( - 'set', index=context.ipr.link_lookup(ifname='lo'), state='up' - ) - - context.ndb.interfaces.wait(ifname=ifname, state='up') - context.ndb.interfaces.wait(ifname='lo', state='up') - - context.ipr.link( - 'set', index=context.ipr.link_lookup(ifname='lo'), state='up' - ) - with pytest.raises(NetlinkError) as e: - context.ipr.probe('add', kind='ping', dst=target) - assert e.value.code == errno.ETIMEDOUT diff --git a/tests/test_linux/test_ipr/test_route.py b/tests/test_linux/test_ipr/test_route.py index a1aa0bd8c..84485419a 100644 --- a/tests/test_linux/test_ipr/test_route.py +++ b/tests/test_linux/test_ipr/test_route.py @@ -9,7 +9,6 @@ from pyroute2 import IPRoute, NetlinkError from pyroute2.common import AF_MPLS -from pyroute2.netlink.rt_files import RtTablesFile from pyroute2.netlink.rtnl.rtmsg import RTNH_F_ONLINK pytestmark = [require_root()] @@ -454,22 +453,3 @@ def test_flush_routes(context): time.sleep(0.1) else: raise Exception('route table not flushed') - - -def test_route_with_rt_file(context, fake_rt_file): - fake_rt_file.create(RtTablesFile, {"254": "my_table"}) - - if not context.ipr.get_default_routes(table="my_table"): - pytest.skip('no default IPv4 routes') - require_kernel(4, 20) - - with IPRoute(strict_check=True) as ip: - rts = list( - ip.get_routes( - family=socket.AF_INET, dst='8.8.8.8', table="my_table" - ) - ) - assert len(rts) > 0 - assert ( - RtTablesFile().get_rt_name(rts[0].get("RTA_TABLE")) == "my_table" - ) diff --git a/tests/test_linux/test_ipr/test_rule.py b/tests/test_linux/test_ipr/test_rule.py index a131992ca..5068ae6b1 100644 --- a/tests/test_linux/test_ipr/test_rule.py +++ b/tests/test_linux/test_ipr/test_rule.py @@ -10,33 +10,222 @@ def test_flush_rules(context): ifaddr1 = context.new_ipaddr ifaddr2 = context.new_ipaddr - init = len(tuple(context.ipr.get_rules(family=socket.AF_INET))) - assert ( - len(tuple(context.ipr.get_rules(priority=lambda x: 100 < x < 500))) - == 0 - ) + init = len(context.ipr.get_rules(family=socket.AF_INET)) + assert len(context.ipr.get_rules(priority=lambda x: 100 < x < 500)) == 0 context.ipr.rule('add', table=10, priority=110) context.ipr.rule('add', table=15, priority=150, action='FR_ACT_PROHIBIT') context.ipr.rule('add', table=20, priority=200, src=ifaddr1) context.ipr.rule('add', table=25, priority=250, dst=ifaddr2) - assert ( - len(tuple(context.ipr.get_rules(priority=lambda x: 100 < x < 500))) - == 4 - ) - assert len(tuple(context.ipr.get_rules(src=ifaddr1))) == 1 - assert len(tuple(context.ipr.get_rules(dst=ifaddr2))) == 1 + assert len(context.ipr.get_rules(priority=lambda x: 100 < x < 500)) == 4 + assert len(context.ipr.get_rules(src=ifaddr1)) == 1 + assert len(context.ipr.get_rules(dst=ifaddr2)) == 1 context.ipr.flush_rules( family=socket.AF_INET, priority=lambda x: 100 < x < 500 ) + assert len(context.ipr.get_rules(priority=lambda x: 100 < x < 500)) == 0 + assert len(context.ipr.get_rules(src=ifaddr1)) == 0 + assert len(context.ipr.get_rules(dst=ifaddr2)) == 0 + assert len(context.ipr.get_rules(family=socket.AF_INET)) == init + + +def test_basic(context): + context.ipr.rule('add', table=10, priority=32000) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32000 + and x.get_attr('FRA_TABLE') == 10 + ] + ) + == 1 + ) + context.ipr.rule('delete', table=10, priority=32000) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32000 + and x.get_attr('FRA_TABLE') == 10 + ] + ) + == 0 + ) + + +def test_fwmark(context): + context.ipr.rule('add', table=15, priority=32006, fwmark=10) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32006 + and x.get_attr('FRA_TABLE') == 15 + and x.get_attr('FRA_FWMARK') + ] + ) + == 1 + ) + context.ipr.rule('delete', table=15, priority=32006, fwmark=10) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32006 + and x.get_attr('FRA_TABLE') == 15 + and x.get_attr('FRA_FWMARK') + ] + ) + == 0 + ) + + +def test_fwmark_mask_normalized(context): + context.ipr.rule('add', table=15, priority=32006, fwmark=10, fwmask=20) assert ( - len(tuple(context.ipr.get_rules(priority=lambda x: 100 < x < 500))) + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32006 + and x.get_attr('FRA_TABLE') == 15 + and x.get_attr('FRA_FWMARK') + and x.get_attr('FRA_FWMASK') + ] + ) + == 1 + ) + context.ipr.rule('delete', table=15, priority=32006, fwmark=10, fwmask=20) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32006 + and x.get_attr('FRA_TABLE') == 15 + and x.get_attr('FRA_FWMARK') + and x.get_attr('FRA_FWMASK') + ] + ) + == 0 + ) + + +def test_fwmark_mask_raw(context): + context.ipr.rule('add', table=15, priority=32006, fwmark=10, FRA_FWMASK=20) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32006 + and x.get_attr('FRA_TABLE') == 15 + and x.get_attr('FRA_FWMARK') + and x.get_attr('FRA_FWMASK') + ] + ) + == 1 + ) + context.ipr.rule( + 'delete', table=15, priority=32006, fwmark=10, FRA_FWMASK=20 + ) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32006 + and x.get_attr('FRA_TABLE') == 15 + and x.get_attr('FRA_FWMARK') + and x.get_attr('FRA_FWMASK') + ] + ) == 0 ) - assert len(tuple(context.ipr.get_rules(src=ifaddr1))) == 0 - assert len(tuple(context.ipr.get_rules(dst=ifaddr2))) == 0 - assert len(tuple(context.ipr.get_rules(family=socket.AF_INET))) == init def test_bad_table(context): with pytest.raises(struct.error): context.ipr.rule('add', table=-1, priority=32000) + + +def test_big_table(context): + context.ipr.rule('add', table=1024, priority=32000) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32000 + and x.get_attr('FRA_TABLE') == 1024 + ] + ) + == 1 + ) + context.ipr.rule('delete', table=1024, priority=32000) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32000 + and x.get_attr('FRA_TABLE') == 1024 + ] + ) + == 0 + ) + + +def test_src_dst(context): + context.ipr.rule( + 'add', + table=17, + priority=32005, + src='10.0.0.0', + src_len=24, + dst='10.1.0.0', + dst_len=24, + ) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32005 + and x.get_attr('FRA_TABLE') == 17 + and x.get_attr('FRA_SRC') == '10.0.0.0' + and x.get_attr('FRA_DST') == '10.1.0.0' + and x['src_len'] == 24 + and x['dst_len'] == 24 + ] + ) + == 1 + ) + context.ipr.rule( + 'del', + table=17, + priority=32005, + src='10.0.0.0', + src_len=24, + dst='10.1.0.0', + dst_len=24, + ) + assert ( + len( + [ + x + for x in context.ipr.get_rules() + if x.get_attr('FRA_PRIORITY') == 32005 + and x.get_attr('FRA_TABLE') == 17 + and x.get_attr('FRA_SRC') == '10.0.0.0' + and x.get_attr('FRA_DST') == '10.1.0.0' + and x['src_len'] == 24 + and x['dst_len'] == 24 + ] + ) + == 0 + ) diff --git a/tests/test_linux/test_ipr/test_vlan.py b/tests/test_linux/test_ipr/test_vlan.py index c1aaa9dc1..31baf97c7 100644 --- a/tests/test_linux/test_ipr/test_vlan.py +++ b/tests/test_linux/test_ipr/test_vlan.py @@ -74,7 +74,7 @@ def test_qos_add(context, spec, key, check): kind='vlan', link=index, vlan_id=vlan_id, - **spec, + **spec ) (vlan,) = context.ipr.poll(context.ipr.link, 'dump', ifname=vlan_name) diff --git a/tests/test_linux/test_ipvs.py b/tests/test_linux/test_ipvs.py deleted file mode 100644 index e37d62b53..000000000 --- a/tests/test_linux/test_ipvs.py +++ /dev/null @@ -1,57 +0,0 @@ -from socket import IPPROTO_TCP - -import pytest - -from pyroute2 import IPVS, IPVSService - - -class Context: - def __init__(self, request, tmpdir): - self.ipvs = IPVS() - self.services = [] - - def new_service(self, addr, port, protocol): - service = IPVSService(addr=addr, port=port, protocol=protocol) - self.ipvs.service("add", service=service) - self.services.append(service) - return service - - def teardown(self): - for service in self.services: - self.ipvs.service("del", service=service) - self.services = [] - - def service(self, command, service=None): - return self.ipvs.service(command, service) - - def dest(self, command, service, dest=None): - return self.ipvs.dest(command, service, dest) - - -@pytest.fixture -def ipvsadm(request, tmpdir): - ctx = Context(request, tmpdir) - yield ctx - ctx.teardown() - - -def test_basic(ipvsadm, context): - ipaddr = context.new_ipaddr - ( - context.ndb.interfaces[context.default_interface.ifname] - .add_ip(f"{ipaddr}/24") - .commit() - ) - ipvsadm.new_service(addr=ipaddr, port=6000, protocol=IPPROTO_TCP) - buffer = [] - for service in ipvsadm.service("dump"): - if ( - service.get(('service', 'addr')) == ipaddr - and service.get(('service', 'port')) == 6000 - and service.get(('service', 'protocol')) == IPPROTO_TCP - ): - break - buffer.append(service) - else: - raise KeyError('service not found') - print(buffer) diff --git a/tests/test_linux/test_iwutil.py b/tests/test_linux/test_iwutil.py index c9377bc9d..c3b147c15 100644 --- a/tests/test_linux/test_iwutil.py +++ b/tests/test_linux/test_iwutil.py @@ -1,21 +1,11 @@ import collections import errno -from typing import Generator import pytest from pr2test.marks import require_root from pyroute2 import IW, IPRoute from pyroute2.netlink.exceptions import NetlinkError -from pyroute2.netlink.nl80211 import nl80211cmd - -# FIXME: should be fixed after dropping support for Python 3.9 -try: - from types import NoneType -except ImportError: - NoneType = type(None) - -pytestmark = [require_root()] @pytest.fixture @@ -57,48 +47,8 @@ def test_list_dev(ctx): ctx.iw.list_dev() +@require_root def test_scan(ctx): with IPRoute() as ipr: ipr.link('set', index=ctx.index, state='up') ctx.iw.scan(ctx.index) - - -def assert_dump(ctx, dump): - assert len(dump) >= 1 - assert any(map(lambda x: x.get('ifindex') == ctx.index, dump)) - assert any(map(lambda x: x.get('wiphy') == ctx.wiphy, dump)) - - -def test_get_interface_by_phy(ctx): - dump = ctx.iw.get_interface_by_phy(ctx.wiphy) - assert isinstance(dump, list) - assert_dump(ctx, dump) - - -def test_get_interface_by_ifindex(ctx): - dump = ctx.iw.get_interface_by_ifindex(ctx.index) - assert isinstance(dump, list) - assert_dump(ctx, dump) - - -def test_get_interfaces_dict(ctx): - d = ctx.iw.get_interfaces_dict() - assert isinstance(d, dict) - for key, (index, name, address, freq, chan_width) in d.items(): - assert isinstance(key, (NoneType, str)) - assert isinstance(index, (NoneType, int)) - assert isinstance(name, str) - assert isinstance(address, str) and all( - map(lambda x: x >= 0, [int(x, 16) for x in address.split(':')]) - ) - assert isinstance(freq, int) and freq >= 0 - assert isinstance(chan_width, (NoneType, int)) - - -def test_get_stations(ctx): - dump = ctx.iw.get_stations(ctx.index) - assert isinstance(dump, Generator) - materialized = tuple(dump) - assert len(materialized) >= 1 - sta_info = materialized[0].get('NL80211_ATTR_STA_INFO') - assert isinstance(sta_info, nl80211cmd.STAInfo) diff --git a/tests/test_linux/test_ndb/test_altnames.py b/tests/test_linux/test_ndb/test_altnames.py index 2fa274616..28a940e15 100644 --- a/tests/test_linux/test_ndb/test_altnames.py +++ b/tests/test_linux/test_ndb/test_altnames.py @@ -10,7 +10,6 @@ ) -@pytest.mark.xfail(reason='flaky test, to be fixed') @pytest.mark.parametrize('context', test_matrix, indirect=True) @skip_if_not_supported def test_altname_complex(context): diff --git a/tests/test_linux/test_ndb/test_backup.py b/tests/test_linux/test_ndb/test_backup.py index 1c67fdc9d..b2be41380 100644 --- a/tests/test_linux/test_ndb/test_backup.py +++ b/tests/test_linux/test_ndb/test_backup.py @@ -1,10 +1,17 @@ import sqlite3 +import sys import uuid +import pytest + +@pytest.mark.skipif( + sys.version_info < (3, 7), + reason='SQLite3 backup not supported on this Python version', +) def test_file_backup(context): filename = str(uuid.uuid4()) + '-backup.db' - context.ndb.db.backup(filename) + context.ndb.backup(filename) backup = sqlite3.connect(filename) cursor = backup.cursor() cursor.execute('SELECT f_IFLA_IFNAME FROM interfaces WHERE f_index > 0') diff --git a/tests/test_linux/test_ndb/test_examples.py b/tests/test_linux/test_ndb/test_examples.py index a0d7b3767..4cd71181f 100644 --- a/tests/test_linux/test_ndb/test_examples.py +++ b/tests/test_linux/test_ndb/test_examples.py @@ -22,6 +22,13 @@ def get_examples(*argv): } +@pytest.mark.parametrize(**get_examples('examples', 'pyroute2-cli')) +def test_cli_examples(example, pytester, context): + with example.open('r') as text: + result = pytester.run('pyroute2-cli', stdin=text) + assert result.ret == 0 + + @pytest.mark.parametrize(**get_examples('examples', 'ndb')) def test_ndb_examples(example, pytester, context): argv = [] @@ -37,3 +44,11 @@ def test_ndb_examples(example, pytester, context): os.environ[key] = value result = pytester.run(sys.executable, example.as_posix(), *argv) assert result.ret == 0 + + +def test_basic(tmpdir, pytester, context): + pytester.makefile('.pr2', test='interfaces lo mtu') + with open('test.pr2', 'r') as text: + result = pytester.run("pyroute2-cli", stdin=text) + assert result.ret == 0 + assert result.outlines == ['65536'] diff --git a/tests/test_linux/test_ndb/test_netns.py b/tests/test_linux/test_ndb/test_netns.py index 34e6499cd..ffd31c56a 100644 --- a/tests/test_linux/test_ndb/test_netns.py +++ b/tests/test_linux/test_ndb/test_netns.py @@ -1,4 +1,3 @@ -import errno import logging import uuid @@ -7,7 +6,7 @@ from pr2test.marks import require_root from pr2test.tools import address_exists, interface_exists -from pyroute2 import NDB, IPRoute, NetlinkError, netns +from pyroute2 import NDB, netns pytestmark = [require_root()] @@ -15,7 +14,7 @@ @pytest.mark.parametrize('context', test_matrix, indirect=True) -def _test_create_remove(context): +def test_create_remove(context): nsname = context.new_nsname with NDB(log=(context.new_log, logging.DEBUG)) as ndb: # create a netns via ndb.netns @@ -47,43 +46,6 @@ def test_views_contain(context): assert {'ifname': v1, 'target': nsname} in context.ndb.interfaces -@pytest.mark.parametrize('context', test_matrix, indirect=True) -def test_interface_move_altname(context): - ifname1 = context.new_ifname - ifname2 = context.new_ifname - altname = context.new_ifname - nsname = context.new_nsname - - context.ndb.sources.add(netns=nsname) - # create the conflict - idx = 500 - with IPRoute() as ipr: - while idx < 1000: - try: - ipr.link('add', ifname=ifname2, index=idx, kind='dummy') - break - except NetlinkError as e: - if e.code != errno.EEXIST: - raise - idx += 1 - else: - raise RuntimeError('no free interface index available') - with IPRoute(netns=nsname) as ipr: - ipr.link('add', ifname=ifname1, index=idx, kind='dummy') - - link1 = context.ndb.interfaces.wait(target=nsname, ifname=ifname1) - link2 = context.ndb.interfaces.wait(target='localhost', ifname=ifname2) - - link2.add_altname(altname).commit() - - # move the interface - link2.set('net_ns_fd', nsname).commit() - - assert link2['target'] == nsname - assert link2['index'] != link1['index'] - assert len(link2['alt_ifname_list']) == 1 - - @pytest.mark.parametrize('context', test_matrix, indirect=True) def test_interface_move(context): ifname = context.new_ifname diff --git a/tests/test_linux/test_ndb/test_probe.py b/tests/test_linux/test_ndb/test_probe.py deleted file mode 100644 index 72669f9f4..000000000 --- a/tests/test_linux/test_ndb/test_probe.py +++ /dev/null @@ -1,57 +0,0 @@ -import errno - -import pytest -from pr2test.context_manager import make_test_matrix, skip_if_not_supported -from pr2test.marks import require_root - -from pyroute2 import NetlinkError - -pytestmark = [require_root()] -test_matrix = make_test_matrix(targets=['local', 'netns']) - - -@pytest.mark.parametrize('context', test_matrix, indirect=True) -@skip_if_not_supported -def test_ping_ok(context): - index, ifname = context.default_interface - ipaddr = context.new_ipaddr - - with context.ndb.interfaces[ifname] as i: - i.add_ip(address=ipaddr, prefixlen=24) - i.set(state='up') - - with context.ndb.interfaces['lo'] as i: - i.set(state='up') - - context.ndb.probes.create(kind='ping', dst=ipaddr).commit() - - -@pytest.mark.parametrize( - 'context', make_test_matrix(targets=['netns']), indirect=True -) -@skip_if_not_supported -def test_ping_fail_ehostunreach(context): - with context.ndb.interfaces['lo'] as i: - i.set(state='down') - with pytest.raises(NetlinkError) as e: - context.ndb.probes.create(kind='ping', dst='127.0.0.1').commit() - assert e.value.code == errno.EHOSTUNREACH - - -@pytest.mark.parametrize( - 'context', make_test_matrix(targets=['netns']), indirect=True -) -@skip_if_not_supported -def test_ping_fail_etimedout(context): - index, ifname = context.default_interface - ipaddr = context.new_ipaddr - target = context.new_ipaddr - - with context.ndb.interfaces[ifname] as i: - i.add_ip(address=ipaddr, prefixlen=24) - i.set(state='up') - with context.ndb.interfaces['lo'] as i: - i.set(state='up') - with pytest.raises(NetlinkError) as e: - context.ndb.probes.create(kind='ping', dst=target).commit() - assert e.value.code == errno.ETIMEDOUT diff --git a/tests/test_linux/test_ndb/test_routes.py b/tests/test_linux/test_ndb/test_routes.py index 165851d78..ee6b7f7cd 100644 --- a/tests/test_linux/test_ndb/test_routes.py +++ b/tests/test_linux/test_ndb/test_routes.py @@ -457,10 +457,10 @@ def test_metrics_set(context): assert route_exists(context.netns, dst=ipnet, dst_len=24, gateway=gateway) with pytest.raises(KeyError): assert route['metrics']['mtu'] - assert isinstance(route['metrics'].asyncore, MetricsStub) + assert isinstance(route['metrics'], MetricsStub) route['metrics']['mtu'] = target - assert isinstance(route['metrics'].asyncore, Metrics) + assert isinstance(route['metrics'], Metrics) route.commit() assert route_exists( diff --git a/tests/test_linux/test_ndb/test_sources.py b/tests/test_linux/test_ndb/test_sources.py index be9f82bfe..25177df97 100644 --- a/tests/test_linux/test_ndb/test_sources.py +++ b/tests/test_linux/test_ndb/test_sources.py @@ -47,10 +47,7 @@ def test_multiple_sources(context): # NDB() object from the code block above, check # that all the sources are closed too for source in ndb.sources: - # - # a closed event loop will raise RuntimeError - with pytest.raises(RuntimeError): - assert ndb.sources[source] + assert ndb.sources[source].nl.closed @pytest.mark.parametrize('context', test_matrix, indirect=True) @@ -159,7 +156,7 @@ def test_source_netns_restart(context): def count_interfaces(ndb, target): return ( - ndb.db.fetchone( + ndb.task_manager.db_fetchone( ''' SELECT count(*) FROM interfaces WHERE diff --git a/tests/test_linux/test_raw/test_dhcp.py b/tests/test_linux/test_raw/test_dhcp.py new file mode 100644 index 000000000..dc0a8d5e8 --- /dev/null +++ b/tests/test_linux/test_raw/test_dhcp.py @@ -0,0 +1,68 @@ +import collections +import json +import subprocess + +import pytest +from pr2test.marks import require_root + +from pyroute2 import NDB +from pyroute2.common import dqn2int, hexdump, hexload +from pyroute2.dhcp import client + +pytestmark = [require_root()] + + +@pytest.fixture +def ctx(): + ndb = NDB() + index = 0 + ifname = '' + # get a DHCP default route, if exists + with ndb.routes.dump() as dump: + dump.select_records(proto=16, dst='') + for route in dump: + index = route.oif + ifname = ndb.interfaces[index]['ifname'] + break + yield collections.namedtuple('Context', ['ndb', 'index', 'ifname'])( + ndb, index, ifname + ) + ndb.close() + + +def _do_test_client_module(ctx): + if ctx.index == 0: + pytest.skip('no DHCP interfaces detected') + + response = client.action(ctx.ifname) + options = response['options'] + router = response['options']['router'][0] + prefixlen = dqn2int(response['options']['subnet_mask']) + address = response['yiaddr'] + l2addr = response['chaddr'] + + # convert addresses like 96:0:1:45:fa:6c into 96:00:01:45:fa:6c + assert ( + hexdump(hexload(l2addr)) == ctx.ndb.interfaces[ctx.ifname]['address'] + ) + assert router == ctx.ndb.routes['default']['gateway'] + assert { + 'address': address, + 'prefixlen': prefixlen, + 'index': ctx.index, + } in ctx.ndb.addresses + assert options['lease_time'] > 0 + return response + + +def test_client_module(ctx): + _do_test_client_module(ctx) + + +def test_client_console(ctx): + response_from_module = json.loads(json.dumps(_do_test_client_module(ctx))) + client = subprocess.run( + ['pyroute2-dhcp-client', ctx.ifname], stdout=subprocess.PIPE + ) + response_from_console = json.loads(client.stdout) + assert response_from_module == response_from_console diff --git a/tests/test_linux/test_remote.py b/tests/test_linux/test_remote.py new file mode 100644 index 000000000..73962abb7 --- /dev/null +++ b/tests/test_linux/test_remote.py @@ -0,0 +1,14 @@ +from pr2test.context_manager import skip_if_not_supported + +from pyroute2 import IPRoute, RemoteIPRoute + + +@skip_if_not_supported +def test_links(): + with IPRoute() as ipr: + links1 = set([x.get_attr('IFLA_IFNAME') for x in ipr.get_links()]) + + with RemoteIPRoute() as ipr: + links2 = set([x.get_attr('IFLA_IFNAME') for x in ipr.get_links()]) + + assert links1 == links2 diff --git a/tests/test_linux/test_tc/test_basic.py b/tests/test_linux/test_tc/test_basic.py index f26bd1e81..899b8adeb 100644 --- a/tests/test_linux/test_tc/test_basic.py +++ b/tests/test_linux/test_tc/test_basic.py @@ -65,22 +65,6 @@ def test_sfq(context): assert qdisc_exists(context.netns, 'sfq', ifname=ifname, perturb_period=10) -@pytest.mark.parametrize('context', test_matrix, indirect=True) -def test_compat_args4(context): - # get a free link index - idx = max([x.get('index') for x in context.ipr.link('dump')]) + 42 - with pytest.raises(NetlinkError) as e: - context.ipr.tc('add', 'sfq', idx, ':1') - assert e.value.code == errno.ENODEV - - -@pytest.mark.parametrize('context', test_matrix, indirect=True) -def test_compat_args5(context): - idx = max([x.get('index') for x in context.ipr.link('dump')]) + 42 - with pytest.raises(TypeError): - context.ipr.tc('add', 'sfq', idx, ':1', 42) - - @pytest.mark.parametrize('context', test_matrix, indirect=True) def test_tbf(context): index, ifname = context.default_interface diff --git a/tests/test_linux/test_tc/test_bpf.py b/tests/test_linux/test_tc/test_bpf.py index 6fea6b174..1acc360dd 100644 --- a/tests/test_linux/test_tc/test_bpf.py +++ b/tests/test_linux/test_tc/test_bpf.py @@ -210,10 +210,8 @@ def test_filter_delete(context, bpf_cls): classid=1, direct_action=True, ) - filters = tuple( - context.ipr.get_filters( - index=context.default_interface.index, parent='ffff:fff2' - ) + filters = context.ipr.get_filters( + index=context.default_interface.index, parent='ffff:fff2' ) # len == 2: handles 0 and 1 assert len(filters) == 2 @@ -224,9 +222,7 @@ def test_filter_delete(context, bpf_cls): parent='ffff:fff2', info=filters[0]['info'], ) - filters = tuple( - context.ipr.get_filters( - index=context.default_interface.index, parent='ffff:fff2' - ) + filters = context.ipr.get_filters( + index=context.default_interface.index, parent='ffff:fff2' ) assert len(filters) == 0 diff --git a/tests/test_linux/test_tc/test_classful.py b/tests/test_linux/test_tc/test_classful.py index 3e0956e68..a53161b38 100644 --- a/tests/test_linux/test_tc/test_classful.py +++ b/tests/test_linux/test_tc/test_classful.py @@ -23,7 +23,7 @@ def test_drr(context): context.ipr.tc('add-class', 'drr', index=index, handle='1:20', quantum=20) context.ipr.tc('add-class', 'drr', index=index, handle='1:30', quantum=30) assert qdisc_exists(context.netns, 'drr', ifname=ifname) - cls = tuple(context.ipr.get_classes(index=index)) + cls = context.ipr.get_classes(index=index) assert len(cls) == 2 assert cls[0].get_attr('TCA_KIND') == 'drr' assert cls[1].get_attr('TCA_KIND') == 'drr' @@ -46,7 +46,7 @@ def test_hfsc(context): parent='1:0', rsc={'m2': '3mbit'}, ) - cls = tuple(context.ipr.get_classes(index=index)) + cls = context.ipr.get_classes(index=index) assert len(cls) == 2 # implicit root class + the defined one assert cls[0].get_attr('TCA_KIND') == 'hfsc' assert cls[1].get_attr('TCA_KIND') == 'hfsc' diff --git a/tests/test_linux/test_tc/test_htb.py b/tests/test_linux/test_tc/test_htb.py index 4a259ee37..13764334f 100644 --- a/tests/test_linux/test_tc/test_htb.py +++ b/tests/test_linux/test_tc/test_htb.py @@ -9,68 +9,6 @@ test_matrix = make_test_matrix(targets=['local', 'netns']) -@pytest.mark.parametrize('context', test_matrix, indirect=True) -@skip_if_not_supported -def test_htb_over_32gbit(context): - - index, ifname = context.default_interface - - # 8<----------------------------------------------------- - # root queue, '1:0' handle notation - context.ipr.tc('add', 'htb', index=index, handle='1:', default='20:0') - - assert qdisc_exists(context.netns, 'htb', ifname=ifname) - - # 8<----------------------------------------------------- - # classes, both string and int handle notation - context.ipr.tc( - 'add-class', - 'htb', - index=index, - handle=0x10001, - parent=0x10000, - rate='50gbit', - ceil='50gbit', - ) - context.ipr.tc( - 'add-class', - 'htb', - index=index, - handle=0x10010, - parent=0x10001, - rate='35gbit', - ceil='35gbit', - prio=1, - ) - context.ipr.tc( - 'add-class', - 'htb', - index=index, - handle=0x10020, - parent=0x10001, - rate='128kbit', - ceil='128kbit', - prio=2, - ) - - # 8<----------------------------------------------------- - # list the installed classes - classes = tuple(context.ipr.get_classes(index=index)) - assert len(classes) == 3 - rate0 = classes[0].get(('TCA_OPTIONS', 'TCA_HTB_RATE64'), 0) - ceil0 = classes[0].get(('TCA_OPTIONS', 'TCA_HTB_CEIL64'), 0) - rate1 = classes[1].get(('TCA_OPTIONS', 'TCA_HTB_RATE64'), 0) - ceil1 = classes[1].get(('TCA_OPTIONS', 'TCA_HTB_CEIL64'), 0) - rate2 = classes[2].get(('TCA_OPTIONS', 'TCA_HTB_RATE64'), 0) - ceil2 = classes[2].get(('TCA_OPTIONS', 'TCA_HTB_CEIL64'), 0) - assert rate0 == ceil0 - assert rate1 == ceil1 - assert rate2 == ceil2 - assert rate0 > rate2 - assert rate1 > rate2 - assert rate2 == 0 - - @pytest.mark.parametrize('context', test_matrix, indirect=True) @skip_if_not_supported def test_htb(context): @@ -112,7 +50,7 @@ def test_htb(context): burst=1024 * 6, prio=2, ) - cls = tuple(context.ipr.get_classes(index=index)) + cls = context.ipr.get_classes(index=index) assert len(cls) == 3 # 8<----------------------------------------------------- @@ -157,7 +95,7 @@ def test_htb(context): keys=['0x5/0xf+0', '0x10/0xff+33'], ) # 2 filters + 2 autogenerated - fls = tuple(context.ipr.get_filters(index=index)) + fls = context.ipr.get_filters(index=index) assert len(fls) == 4 @@ -177,7 +115,7 @@ def test_replace(context): burst=1024 * 6, prio=3, ) - clss = tuple(context.ipr.get_classes(index=index)) + clss = context.ipr.get_classes(index=index) for cls in clss: if cls['handle'] == 0x10010: break diff --git a/tests/test_minimal/conftest.py b/tests/test_minimal/conftest.py deleted file mode 100644 index cec427ba8..000000000 --- a/tests/test_minimal/conftest.py +++ /dev/null @@ -1,28 +0,0 @@ -import subprocess -from collections import namedtuple - -import pytest - -from pyroute2.common import uifname - -NFTSetup = namedtuple('NFTSetup', ('table', 'chain')) - - -@pytest.fixture -def nft(): - table = uifname() - chain = uifname() - subprocess.call(f'nft add table {table}'.split()) - subprocess.call( - f'nft add chain {table} {chain} ' - f'{{ type filter hook input priority 500 ; }}'.split() - ) - subprocess.call( - f'nft add chain {table} POSTROUTING ' - f'{{ type nat hook postrouting priority 100 ; }}'.split() - ) - yield NFTSetup(table, chain) - subprocess.call(f'nft delete table {table}'.split()) - - -pytest_plugins = ['pyroute2.fixtures.iproute', 'pyroute2.fixtures.plan9'] diff --git a/tests/test_minimal/test_ipr/test_addr_async.py b/tests/test_minimal/test_ipr/test_addr_async.py deleted file mode 100644 index c3d82367c..000000000 --- a/tests/test_minimal/test_ipr/test_addr_async.py +++ /dev/null @@ -1,53 +0,0 @@ -import re - -import pytest -from net_tools import address_exists - -ip4v6 = re.compile('^[.:0-9a-f]*$') - - -@pytest.mark.asyncio -async def test_addr_dump(async_ipr): - async for addr in await async_ipr.addr('dump'): - index = addr.get('index') - address = addr.get('address', '') - prefixlen = addr.get('prefixlen') - assert index > 0 - assert ip4v6.match(address) - assert prefixlen > 0 - - -@pytest.mark.asyncio -async def test_addr_add(async_ipr, test_link_ifname, test_link_index, nsname): - await async_ipr.addr( - 'add', index=test_link_index, address='192.168.145.150', prefixlen=24 - ) - assert address_exists('192.168.145.150', test_link_ifname, netns=nsname) - - -@pytest.mark.parametrize( - 'request_info,assert_info', - ( - ({'preferred': 99}, {'preferred': 99}), - ({'preferred_lft': 109}, {'preferred': 109}), - ({'valid': 119, 'preferred': 100}, {'valid': 119}), - ({'valid_lft': 129, 'preferred': 100}, {'valid': 129}), - ), -) -@pytest.mark.asyncio -async def test_addr_cacheinfo( - async_ipr, - test_link_ifname, - test_link_index, - nsname, - request_info, - assert_info, -): - await async_ipr.addr( - 'add', - index=test_link_index, - address='2001:db8::5678', - mask=128, - **request_info, - ) - assert address_exists('2001:db8::5678', netns=nsname, **assert_info) diff --git a/tests/test_minimal/test_ipr/test_addr_sync.py b/tests/test_minimal/test_ipr/test_addr_sync.py deleted file mode 100644 index 87232a583..000000000 --- a/tests/test_minimal/test_ipr/test_addr_sync.py +++ /dev/null @@ -1,50 +0,0 @@ -import re - -import pytest -from net_tools import address_exists - -ip4v6 = re.compile('^[.:0-9a-f]*$') - - -def test_addr_dump(sync_ipr): - for addr in sync_ipr.addr('dump'): - index = addr.get('index') - address = addr.get('address', '') - prefixlen = addr.get('prefixlen') - assert index > 0 - assert ip4v6.match(address) - assert prefixlen > 0 - - -def test_addr_add(sync_ipr, test_link_ifname, test_link_index, nsname): - sync_ipr.addr( - 'add', index=test_link_index, address='192.168.145.150', prefixlen=24 - ) - assert address_exists('192.168.145.150', test_link_ifname, netns=nsname) - - -@pytest.mark.parametrize( - 'request_info,assert_info', - ( - ({'preferred': 99}, {'preferred': 99}), - ({'preferred_lft': 109}, {'preferred': 109}), - ({'valid': 119, 'preferred': 100}, {'valid': 119}), - ({'valid_lft': 129, 'preferred': 100}, {'valid': 129}), - ), -) -def test_addr_cacheinfo( - sync_ipr, - test_link_ifname, - test_link_index, - nsname, - request_info, - assert_info, -): - sync_ipr.addr( - 'add', - index=test_link_index, - address='2001:db8::5678', - mask=128, - **request_info, - ) - assert address_exists('2001:db8::5678', netns=nsname, **assert_info) diff --git a/tests/test_minimal/test_ipr/test_ensure_async.py b/tests/test_minimal/test_ipr/test_ensure_async.py deleted file mode 100644 index 4ab08618f..000000000 --- a/tests/test_minimal/test_ipr/test_ensure_async.py +++ /dev/null @@ -1,93 +0,0 @@ -import pytest -from net_tools import address_exists, interface_exists, route_exists - - -@pytest.mark.asyncio -async def test_ensure_link_present(async_ipr, tmp_link_ifname, nsname): - await async_ipr.ensure( - async_ipr.link, - present=True, - ifname=tmp_link_ifname, - kind='dummy', - state='up', - ) - assert interface_exists(tmp_link_ifname, netns=nsname) - await async_ipr.ensure( - async_ipr.link, - present=True, - ifname=tmp_link_ifname, - kind='dummy', - state='up', - ) - - -@pytest.mark.asyncio -async def test_ensure_link_absent(async_ipr, nsname, test_link_ifname): - await async_ipr.ensure( - async_ipr.link, present=False, ifname=test_link_ifname - ) - assert not interface_exists(test_link_ifname, netns=nsname) - await async_ipr.ensure( - async_ipr.link, present=False, ifname=test_link_ifname - ) - - -@pytest.mark.asyncio -async def test_ensure_address_exists( - async_ipr, nsname, test_link_index, test_link_ifname -): - await async_ipr.ensure( - async_ipr.addr, - present=True, - index=test_link_index, - address='192.168.145.150/24', - ) - assert address_exists('192.168.145.150', test_link_ifname, netns=nsname) - await async_ipr.ensure( - async_ipr.addr, - present=True, - index=test_link_index, - address='192.168.145.150/24', - ) - - -@pytest.mark.asyncio -async def test_ensure_address_absent( - async_ipr, nsname, test_link_index, test_link_ifname -): - await async_ipr.ensure( - async_ipr.addr, - present=False, - index=test_link_index, - address='192.168.145.150/24', - ) - assert not address_exists( - '192.168.145.150', test_link_ifname, netns=nsname - ) - await async_ipr.ensure( - async_ipr.addr, - present=False, - index=test_link_index, - address='192.168.145.150/24', - ) - - -@pytest.mark.asyncio -async def test_ensure_route(async_ipr, nsname, tmp_link_ifname): - link = await async_ipr.ensure( - async_ipr.link, - present=True, - ifname=tmp_link_ifname, - kind='dummy', - state='up', - ) - await async_ipr.ensure( - async_ipr.addr, present=True, index=link, address='192.168.145.150/24' - ) - await async_ipr.ensure( - async_ipr.route, - present=True, - dst='10.20.30.0/24', - gateway='192.168.145.151', - ) - assert route_exists(dst='10.20.30.0/24', netns=nsname) diff --git a/tests/test_minimal/test_ipr/test_ensure_sync.py b/tests/test_minimal/test_ipr/test_ensure_sync.py deleted file mode 100644 index 51b506f7f..000000000 --- a/tests/test_minimal/test_ipr/test_ensure_sync.py +++ /dev/null @@ -1,83 +0,0 @@ -from net_tools import address_exists, interface_exists, route_exists - - -def test_ensure_link_present(sync_ipr, tmp_link_ifname, nsname): - sync_ipr.ensure( - sync_ipr.link, - present=True, - ifname=tmp_link_ifname, - kind='dummy', - state='up', - ) - assert interface_exists(tmp_link_ifname, netns=nsname) - sync_ipr.ensure( - sync_ipr.link, - present=True, - ifname=tmp_link_ifname, - kind='dummy', - state='up', - ) - - -def test_ensure_link_absent(sync_ipr, nsname, test_link_ifname): - sync_ipr.ensure(sync_ipr.link, present=False, ifname=test_link_ifname) - assert not interface_exists(test_link_ifname, netns=nsname) - sync_ipr.ensure(sync_ipr.link, present=False, ifname=test_link_ifname) - - -def test_ensure_address_exists( - sync_ipr, nsname, test_link_index, test_link_ifname -): - sync_ipr.ensure( - sync_ipr.addr, - present=True, - index=test_link_index, - address='192.168.145.150/24', - ) - assert address_exists('192.168.145.150', test_link_ifname, netns=nsname) - sync_ipr.ensure( - sync_ipr.addr, - present=True, - index=test_link_index, - address='192.168.145.150/24', - ) - - -def test_ensure_address_absent( - sync_ipr, nsname, test_link_index, test_link_ifname -): - sync_ipr.ensure( - sync_ipr.addr, - present=False, - index=test_link_index, - address='192.168.145.150/24', - ) - assert not address_exists( - '192.168.145.150', test_link_ifname, netns=nsname - ) - sync_ipr.ensure( - sync_ipr.addr, - present=False, - index=test_link_index, - address='192.168.145.150/24', - ) - - -def test_ensure_route(sync_ipr, nsname, tmp_link_ifname): - link = sync_ipr.ensure( - sync_ipr.link, - present=True, - ifname=tmp_link_ifname, - kind='dummy', - state='up', - ) - sync_ipr.ensure( - sync_ipr.addr, present=True, index=link, address='192.168.145.150/24' - ) - sync_ipr.ensure( - sync_ipr.route, - present=True, - dst='10.20.30.0/24', - gateway='192.168.145.151', - ) - assert route_exists(dst='10.20.30.0/24', netns=nsname) diff --git a/tests/test_minimal/test_ipr/test_link_async.py b/tests/test_minimal/test_ipr/test_link_async.py deleted file mode 100644 index 40e819b6d..000000000 --- a/tests/test_minimal/test_ipr/test_link_async.py +++ /dev/null @@ -1,41 +0,0 @@ -import pytest -from net_tools import interface_exists - - -@pytest.mark.asyncio -async def test_link_dump(async_ipr): - async for link in await async_ipr.link('dump'): - assert link.get('index') > 0 - assert 1 < len(link.get('ifname')) < 16 - - -@pytest.mark.asyncio -async def test_link_add(async_ipr, tmp_link_ifname, nsname): - await async_ipr.link( - 'add', ifname=tmp_link_ifname, kind='dummy', state='up' - ) - assert interface_exists(tmp_link_ifname, netns=nsname) - - -@pytest.mark.asyncio -async def test_link_get(async_ipr, test_link_ifname): - (link,) = await async_ipr.link('get', ifname=test_link_ifname) - assert link.get('state') == 'up' - assert link.get('index') > 1 - assert link.get('ifname') == test_link_ifname - assert link.get(('linkinfo', 'kind')) == 'dummy' - - -@pytest.mark.asyncio -async def test_link_del_by_index( - async_ipr, test_link_ifname, test_link_index, nsname -): - (link,) = await async_ipr.link('get', ifname=test_link_ifname) - await async_ipr.link('del', index=test_link_index) - assert not interface_exists(test_link_ifname, netns=nsname) - - -@pytest.mark.asyncio -async def test_link_del_by_name(async_ipr, test_link_ifname, nsname): - await async_ipr.link('del', ifname=test_link_ifname) - assert not interface_exists(test_link_ifname, netns=nsname) diff --git a/tests/test_minimal/test_ipr/test_link_sync.py b/tests/test_minimal/test_ipr/test_link_sync.py deleted file mode 100644 index 86101a375..000000000 --- a/tests/test_minimal/test_ipr/test_link_sync.py +++ /dev/null @@ -1,33 +0,0 @@ -from net_tools import interface_exists - - -def test_link_dump(sync_ipr): - for link in sync_ipr.link('dump'): - assert link.get('index') > 0 - assert 1 < len(link.get('ifname')) < 16 - - -def test_link_add(sync_ipr, tmp_link_ifname, nsname): - sync_ipr.link('add', ifname=tmp_link_ifname, kind='dummy', state='up') - assert interface_exists(tmp_link_ifname, netns=nsname) - - -def test_link_get(sync_ipr, test_link_ifname): - (link,) = sync_ipr.link('get', ifname=test_link_ifname) - assert link.get('state') == 'up' - assert link.get('index') > 1 - assert link.get('ifname') == test_link_ifname - assert link.get(('linkinfo', 'kind')) == 'dummy' - - -def test_link_del_by_index( - sync_ipr, test_link_ifname, test_link_index, nsname -): - (link,) = sync_ipr.link('get', ifname=test_link_ifname) - sync_ipr.link('del', index=test_link_index) - assert not interface_exists(test_link_ifname, netns=nsname) - - -def test_link_del_by_name(sync_ipr, test_link_ifname, nsname): - sync_ipr.link('del', ifname=test_link_ifname) - assert not interface_exists(test_link_ifname, netns=nsname) diff --git a/tests/test_minimal/test_ipr/test_route_async.py b/tests/test_minimal/test_ipr/test_route_async.py deleted file mode 100644 index 2d45b7f13..000000000 --- a/tests/test_minimal/test_ipr/test_route_async.py +++ /dev/null @@ -1,40 +0,0 @@ -import pytest - -from pyroute2 import AsyncIPRoute - - -@pytest.mark.parametrize( - "command,kwarg", - [ - ("dump", {"table": 255}), - ("show", {"table": 255}), - ("dump", {"match": {"table": 255}}), - ("show", {"match": {"table": 255}}), - ], -) -@pytest.mark.asyncio -async def test_route_filter(async_ipr, command, kwarg): - assert set( - [ - route.get('table') - async for route in await async_ipr.route(command, **kwarg) - ] - ) == set([255]) - - -@pytest.mark.parametrize( - "command,kwarg", - [ - ("dump", {"table": 255, "family": 1}), - ("show", {"table": 255, "family": 1}), - ], -) -@pytest.mark.asyncio -async def test_route_filter_strict(command, kwarg): - async with AsyncIPRoute(strict_check=True) as ipr: - assert set( - [ - route.get('table') - async for route in await ipr.route(command, **kwarg) - ] - ) == set([255]) diff --git a/tests/test_minimal/test_ipr/test_route_dump_async.py b/tests/test_minimal/test_ipr/test_route_dump_async.py deleted file mode 100644 index 14ca3b6f7..000000000 --- a/tests/test_minimal/test_ipr/test_route_dump_async.py +++ /dev/null @@ -1,112 +0,0 @@ -import io -from socket import AF_INET, AF_INET6, AF_UNSPEC - -import pytest -from net_tools import address_exists, route_exists - -from pyroute2.common import load_dump - -test_dump_data = ''' - -# 10.1.2.0/24 via 127.0.0.8 dev lo table 100 -# 10.1.3.0/24 via 127.0.0.8 dev lo table 100 - -24:12:31:45 3c:00:00:00 18:00:22:00 dc:d0:86:67 -7b:68:00:00 02:18:00:00 64:03:00:01 00:00:00:00 -08:00:0f:00 64:00:00:00 08:00:01:00 0a:01:02:00 -08:00:05:00 7f:00:00:08 08:00:04:00 01:00:00:00 -3c:00:00:00 18:00:22:00 dc:d0:86:67 7b:68:00:00 -02:18:00:00 64:03:00:01 00:00:00:00 08:00:0f:00 -64:00:00:00 08:00:01:00 0a:01:03:00 08:00:05:00 -7f:00:00:08 08:00:04:00 01:00:00:00 -''' - - -@pytest.mark.asyncio -async def test_load(async_ipr, nsname): - assert address_exists('127.0.0.1', ifname='lo', netns=nsname) - assert not route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert not route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - fd = io.BytesIO() - fd.write(load_dump(test_dump_data)) - fd.seek(0) - await async_ipr.route_load(fd) - assert route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - - -@pytest.mark.asyncio -async def test_loads(async_ipr, nsname): - assert address_exists('127.0.0.1', ifname='lo', netns=nsname) - assert not route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert not route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - await async_ipr.route_loads(load_dump(test_dump_data)) - assert route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - - -@pytest.mark.parametrize( - 'family,target_tables,target_families,fmt,offset', - [ - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4), - (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4), - (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4), - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'raw', 0), - (AF_INET, {254, 255}, {AF_INET}, 'raw', 0), - (AF_INET6, {254, 255}, {AF_INET6}, 'raw', 0), - ], - ids=( - 'iproute2/AF_UNSPEC', - 'iproute2/AF_INET', - 'iproute2/AF_INET6', - 'raw/AF_UNSPEC', - 'raw/AF_INET', - 'raw/AF_INET6', - ), -) -@pytest.mark.asyncio -async def test_dump( - async_ipr, family, target_tables, target_families, fmt, offset -): - fd = io.BytesIO() - await async_ipr.route_dump(fd, family=family, fmt=fmt) - tables = set() - families = set() - for route in async_ipr.marshal.parse(fd.getvalue()[offset:]): - tables.add(route.get('table')) - families.add(route.get('family')) - assert tables <= target_tables - assert families == target_families - - -@pytest.mark.parametrize( - 'family,target_tables,target_families,fmt,offset', - [ - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4), - (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4), - (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4), - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'raw', 0), - (AF_INET, {254, 255}, {AF_INET}, 'raw', 0), - (AF_INET6, {254, 255}, {AF_INET6}, 'raw', 0), - ], - ids=( - 'iproute2/AF_UNSPEC', - 'iproute2/AF_INET', - 'iproute2/AF_INET6', - 'raw/AF_UNSPEC', - 'raw/AF_INET', - 'raw/AF_INET6', - ), -) -@pytest.mark.asyncio -async def test_dumps( - async_ipr, family, target_tables, target_families, fmt, offset -): - data = await async_ipr.route_dumps(family=family, fmt=fmt) - tables = set() - families = set() - for route in async_ipr.marshal.parse(data[offset:]): - tables.add(route.get('table')) - families.add(route.get('family')) - assert tables <= target_tables - assert families == target_families diff --git a/tests/test_minimal/test_ipr/test_route_dump_sync.py b/tests/test_minimal/test_ipr/test_route_dump_sync.py deleted file mode 100644 index fd5ee18e9..000000000 --- a/tests/test_minimal/test_ipr/test_route_dump_sync.py +++ /dev/null @@ -1,106 +0,0 @@ -import io -from socket import AF_INET, AF_INET6, AF_UNSPEC - -import pytest -from net_tools import address_exists, route_exists - -from pyroute2.common import load_dump - -test_dump_data = ''' - -# 10.1.2.0/24 via 127.0.0.8 dev lo table 100 -# 10.1.3.0/24 via 127.0.0.8 dev lo table 100 - -24:12:31:45 3c:00:00:00 18:00:22:00 dc:d0:86:67 -7b:68:00:00 02:18:00:00 64:03:00:01 00:00:00:00 -08:00:0f:00 64:00:00:00 08:00:01:00 0a:01:02:00 -08:00:05:00 7f:00:00:08 08:00:04:00 01:00:00:00 -3c:00:00:00 18:00:22:00 dc:d0:86:67 7b:68:00:00 -02:18:00:00 64:03:00:01 00:00:00:00 08:00:0f:00 -64:00:00:00 08:00:01:00 0a:01:03:00 08:00:05:00 -7f:00:00:08 08:00:04:00 01:00:00:00 -''' - - -def test_load(sync_ipr, nsname): - sync_ipr.link('set', index=1, state='up') - assert address_exists('127.0.0.1', ifname='lo', netns=nsname) - assert not route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert not route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - fd = io.BytesIO() - fd.write(load_dump(test_dump_data)) - fd.seek(0) - sync_ipr.route_load(fd) - assert route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - - -def test_loads(sync_ipr, nsname): - sync_ipr.link('set', index=1, state='up') - assert address_exists('127.0.0.1', ifname='lo', netns=nsname) - assert not route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert not route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - sync_ipr.route_loads(load_dump(test_dump_data)) - assert route_exists(dst='10.1.2.0/24', table=100, netns=nsname) - assert route_exists(dst='10.1.3.0/24', table=100, netns=nsname) - - -@pytest.mark.parametrize( - 'family,target_tables,target_families,fmt,offset', - [ - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4), - (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4), - (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4), - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'raw', 0), - (AF_INET, {254, 255}, {AF_INET}, 'raw', 0), - (AF_INET6, {254, 255}, {AF_INET6}, 'raw', 0), - ], - ids=( - 'iproute2/AF_UNSPEC', - 'iproute2/AF_INET', - 'iproute2/AF_INET6', - 'raw/AF_UNSPEC', - 'raw/AF_INET', - 'raw/AF_INET6', - ), -) -def test_dump(sync_ipr, family, target_tables, target_families, fmt, offset): - fd = io.BytesIO() - sync_ipr.route_dump(fd, family=family, fmt=fmt) - tables = set() - families = set() - for route in sync_ipr.marshal.parse(fd.getvalue()[offset:]): - tables.add(route.get('table')) - families.add(route.get('family')) - assert tables <= target_tables - assert families == target_families - - -@pytest.mark.parametrize( - 'family,target_tables,target_families,fmt,offset', - [ - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4), - (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4), - (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4), - (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'raw', 0), - (AF_INET, {254, 255}, {AF_INET}, 'raw', 0), - (AF_INET6, {254, 255}, {AF_INET6}, 'raw', 0), - ], - ids=( - 'iproute2/AF_UNSPEC', - 'iproute2/AF_INET', - 'iproute2/AF_INET6', - 'raw/AF_UNSPEC', - 'raw/AF_INET', - 'raw/AF_INET6', - ), -) -def test_dumps(sync_ipr, family, target_tables, target_families, fmt, offset): - data = sync_ipr.route_dumps(family=family, fmt=fmt) - tables = set() - families = set() - for route in sync_ipr.marshal.parse(data[offset:]): - tables.add(route.get('table')) - families.add(route.get('family')) - assert tables <= target_tables - assert families == target_families diff --git a/tests/test_minimal/test_ipr/test_route_sync.py b/tests/test_minimal/test_ipr/test_route_sync.py deleted file mode 100644 index 43567f019..000000000 --- a/tests/test_minimal/test_ipr/test_route_sync.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest - -from pyroute2 import IPRoute - - -@pytest.mark.parametrize( - "command,kwarg", - [ - ("dump", {"table": 255}), - ("show", {"table": 255}), - ("dump", {"match": {"table": 255}}), - ("show", {"match": {"table": 255}}), - ], -) -def test_route_filter(sync_ipr, command, kwarg): - assert set( - [route.get('table') for route in sync_ipr.route(command, **kwarg)] - ) == set([255]) - - -@pytest.mark.parametrize( - "command,kwarg", - [ - ("dump", {"table": 255, "family": 1}), - ("show", {"table": 255, "family": 1}), - ], -) -def test_route_filter_strict(command, kwarg): - with IPRoute(strict_check=True) as ipr: - assert set( - [route.get('table') for route in ipr.route(command, **kwarg)] - ) == set([255]) diff --git a/tests/test_minimal/test_ipr/test_rule_async.py b/tests/test_minimal/test_ipr/test_rule_async.py deleted file mode 100644 index a5ea6a66f..000000000 --- a/tests/test_minimal/test_ipr/test_rule_async.py +++ /dev/null @@ -1,57 +0,0 @@ -from socket import AF_INET, AF_INET6 - -import pytest -from net_tools import rule_exists - - -@pytest.mark.parametrize( - 'priority,spec', - [ - (30313, {'table': 10}), - (30314, {'table': 10, 'src': None}), - (30315, {'table': 10, 'dst': None}), - (30316, {'table': 10, 'dst': '127.0.0.0/24'}), - (30317, {'table': 10, 'src': '127.0.0.0/24'}), - ], -) -@pytest.mark.parametrize( - 'async_ipr', [{'ext_ack': True, 'strict_check': True}], indirect=True -) -@pytest.mark.asyncio -async def test_rule_strict_src(async_ipr, priority, spec, nsname): - await async_ipr.rule('add', priority=priority, **spec) - assert rule_exists(priority=priority, netns=nsname) - - -@pytest.mark.parametrize( - 'priority,proto,spec', - [ - (20100, AF_INET, {'table': 10}), - (20101, AF_INET, {'table': 10, 'fwmark': 15}), - (20102, AF_INET, {'table': 10, 'fwmark': 15, 'fwmask': 20}), - (20103, AF_INET, {'table': 2048, 'FRA_FWMARK': 10, 'FRA_FWMASK': 12}), - (20104, AF_INET, {'table': 2048, 'src': '127.0.1.0', 'src_len': 24}), - (20105, AF_INET, {'table': 2048, 'dst': '127.0.1.0', 'dst_len': 24}), - (20106, AF_INET6, {'table': 5192, 'src': 'fd00::', 'src_len': 8}), - (20107, AF_INET6, {'table': 5192, 'dst': 'fd00::', 'dst_len': 8}), - ], -) -@pytest.mark.asyncio -async def test_rule_add_del(async_ipr, priority, proto, spec, nsname): - await async_ipr.rule('add', priority=priority, **spec) - assert rule_exists(priority=priority, proto=proto, netns=nsname) - assert ( - len( - [ - x - async for x in await async_ipr.rule( - 'dump', priority=priority, **spec - ) - ] - ) - == 1 - ) - await async_ipr.rule('del', priority=priority, **spec) - assert not rule_exists( - priority=priority, proto=proto, netns=nsname, timeout=0.1 - ) diff --git a/tests/test_minimal/test_ipr/test_rule_sync.py b/tests/test_minimal/test_ipr/test_rule_sync.py deleted file mode 100644 index 011dd7066..000000000 --- a/tests/test_minimal/test_ipr/test_rule_sync.py +++ /dev/null @@ -1,47 +0,0 @@ -from socket import AF_INET, AF_INET6 - -import pytest -from net_tools import rule_exists - - -@pytest.mark.parametrize( - 'priority,spec', - [ - (30313, {'table': 10}), - (30314, {'table': 10, 'src': None}), - (30315, {'table': 10, 'dst': None}), - (30316, {'table': 10, 'dst': '127.0.0.0/24'}), - (30317, {'table': 10, 'src': '127.0.0.0/24'}), - ], -) -@pytest.mark.parametrize( - 'sync_ipr', [{'ext_ack': True, 'strict_check': True}], indirect=True -) -def test_rule_strict_src(sync_ipr, priority, spec, nsname): - sync_ipr.rule('add', priority=priority, **spec) - assert rule_exists(priority=priority, netns=nsname) - - -@pytest.mark.parametrize( - 'priority,proto,spec', - [ - (20100, AF_INET, {'table': 10}), - (20101, AF_INET, {'table': 10, 'fwmark': 15}), - (20102, AF_INET, {'table': 10, 'fwmark': 15, 'fwmask': 20}), - (20103, AF_INET, {'table': 2048, 'FRA_FWMARK': 10, 'FRA_FWMASK': 12}), - (20104, AF_INET, {'table': 2048, 'src': '127.0.1.0', 'src_len': 24}), - (20105, AF_INET, {'table': 2048, 'dst': '127.0.1.0', 'dst_len': 24}), - (20106, AF_INET6, {'table': 5192, 'src': 'fd00::', 'src_len': 8}), - (20107, AF_INET6, {'table': 5192, 'dst': 'fd00::', 'dst_len': 8}), - ], -) -def test_rule_add_del(sync_ipr, priority, proto, spec, nsname): - sync_ipr.rule('add', priority=priority, **spec) - assert rule_exists(priority=priority, proto=proto, netns=nsname) - assert ( - len([x for x in sync_ipr.rule('dump', priority=priority, **spec)]) == 1 - ) - sync_ipr.rule('del', priority=priority, **spec) - assert not rule_exists( - priority=priority, proto=proto, netns=nsname, timeout=0.1 - ) diff --git a/tests/test_minimal/test_ipr/test_tc_async.py b/tests/test_minimal/test_ipr/test_tc_async.py deleted file mode 100644 index 6c7979ddf..000000000 --- a/tests/test_minimal/test_ipr/test_tc_async.py +++ /dev/null @@ -1,258 +0,0 @@ -import pytest -from net_tools import class_exists, filter_exists, qdisc_exists - -from pyroute2 import protocols - - -@pytest.mark.asyncio -async def test_tc_get_qdiscs( - async_ipr, test_link_ifname, test_link_index, nsname -): - root_handle = '1:' - await async_ipr.tc( - 'add', - 'tbf', - index=test_link_index, - handle=root_handle, - rate=256, - burst=256, - latency=1, - ) - assert qdisc_exists( - ifname=test_link_ifname, handle=root_handle, rate=256, netns=nsname - ) - assert ( - len( - [ - x - async for x in await async_ipr.get_qdiscs( - index=test_link_index - ) - ] - ) - == 1 - ) - await async_ipr.tc( - 'del', index=test_link_index, handle=root_handle, root=True - ) - assert not qdisc_exists( - ifname=test_link_ifname, - handle=root_handle, - rate=256, - timeout=0.1, - netns=nsname, - ) - - -@pytest.mark.asyncio -async def test_tc_htb(async_ipr, test_link_ifname, test_link_index, nsname): - root_handle = '1:' - root_options_default = '0x200000' - await async_ipr.tc( - 'add', - 'htb', - index=test_link_index, - handle=root_handle, - default=int(root_options_default, 16), - ) - assert qdisc_exists( - ifname=test_link_ifname, - handle=root_handle, - default=root_options_default, - netns=nsname, - ) - - await async_ipr.tc( - 'add-class', - 'htb', - index=test_link_index, - handle='1:1', - parent='1:0', - rate='256kbit', - burst=1024 * 6, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - netns=nsname, - ) - - await async_ipr.tc( - 'add-class', - 'htb', - index=test_link_index, - handle=0x10010, - parent=0x10001, - rate='192kbit', - burst=1024 * 6, - prio=1, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - netns=nsname, - ) - - await async_ipr.tc( - 'add-class', - 'htb', - index=test_link_index, - handle='1:20', - parent='1:1', - rate='128kbit', - burst=1024 * 6, - prio=2, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - netns=nsname, - ) - - await async_ipr.tc( - 'add-filter', - 'u32', - index=test_link_index, - handle='0:0', - parent='1:0', - prio=10, - protocol=protocols.ETH_P_IP, - target='1:10', - keys=['0x0006/0x00ff+8', '0x0000/0xffc0+2'], - ) - assert filter_exists( - ifname=test_link_ifname, - kind='u32', - parent='1:', - protocol='ip', - match_value="6000000", - match_mask="ff000000", - netns=nsname, - ) - - await async_ipr.tc( - 'add-filter', - 'u32', - index=test_link_index, - handle=0, - parent=0x10000, - prio=10, - protocol=protocols.ETH_P_IP, - target=0x10020, - keys=['0x5/0xf+0', '0x10/0xff+33'], - ) - assert filter_exists( - ifname=test_link_ifname, - kind='u32', - parent='1:', - protocol='ip', - match_value='10000000', - match_mask='ff000000', - netns=nsname, - ) - - # complementary delete commands - await async_ipr.tc( - 'del-filter', index=test_link_index, handle='0:0', parent='1:0' - ) - assert not filter_exists( - ifname=test_link_ifname, - kind='u32', - parent='1:0', - timeout=0.1, - netns=nsname, - ) - - await async_ipr.tc( - 'del-class', index=test_link_index, handle='1:20', parent='1:1' - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - netns=nsname, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - netns=nsname, - ) - - await async_ipr.tc( - 'del-class', index=test_link_index, handle='1:10', parent='1:1' - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - netns=nsname, - ) - - await async_ipr.tc( - 'del-class', index=test_link_index, handle='1:1', parent='1:0' - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - timeout=0.1, - netns=nsname, - ) - - await async_ipr.tc( - 'del', index=test_link_index, handle=root_handle, root=True - ) - assert not qdisc_exists( - ifname=test_link_ifname, handle=root_handle, timeout=0.1, netns=nsname - ) diff --git a/tests/test_minimal/test_ipr/test_tc_sync.py b/tests/test_minimal/test_ipr/test_tc_sync.py deleted file mode 100644 index e50b2b953..000000000 --- a/tests/test_minimal/test_ipr/test_tc_sync.py +++ /dev/null @@ -1,237 +0,0 @@ -from net_tools import class_exists, filter_exists, qdisc_exists - -from pyroute2 import protocols - - -def test_tc_get_qdiscs(sync_ipr, test_link_ifname, test_link_index, nsname): - root_handle = '1:' - sync_ipr.tc( - 'add', - 'tbf', - index=test_link_index, - handle=root_handle, - rate=256, - burst=256, - latency=1, - ) - assert qdisc_exists( - ifname=test_link_ifname, handle=root_handle, rate=256, netns=nsname - ) - assert len([x for x in sync_ipr.get_qdiscs(index=test_link_index)]) == 1 - sync_ipr.tc('del', index=test_link_index, handle=root_handle, root=True) - assert not qdisc_exists( - ifname=test_link_ifname, - handle=root_handle, - rate=256, - timeout=0.1, - netns=nsname, - ) - - -def test_tc_htb(sync_ipr, test_link_ifname, test_link_index, nsname): - root_handle = '1:' - root_options_default = '0x200000' - sync_ipr.tc( - 'add', - 'htb', - index=test_link_index, - handle=root_handle, - default=int(root_options_default, 16), - ) - assert qdisc_exists( - ifname=test_link_ifname, - handle=root_handle, - default=root_options_default, - netns=nsname, - ) - - sync_ipr.tc( - 'add-class', - 'htb', - index=test_link_index, - handle='1:1', - parent='1:0', - rate='256kbit', - burst=1024 * 6, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - netns=nsname, - ) - - sync_ipr.tc( - 'add-class', - 'htb', - index=test_link_index, - handle=0x10010, - parent=0x10001, - rate='192kbit', - burst=1024 * 6, - prio=1, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - netns=nsname, - ) - - sync_ipr.tc( - 'add-class', - 'htb', - index=test_link_index, - handle='1:20', - parent='1:1', - rate='128kbit', - burst=1024 * 6, - prio=2, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - netns=nsname, - ) - - sync_ipr.tc( - 'add-filter', - 'u32', - index=test_link_index, - handle='0:0', - parent='1:0', - prio=10, - protocol=protocols.ETH_P_IP, - target='1:10', - keys=['0x0006/0x00ff+8', '0x0000/0xffc0+2'], - ) - assert filter_exists( - ifname=test_link_ifname, - kind='u32', - parent='1:', - protocol='ip', - match_value="6000000", - match_mask="ff000000", - netns=nsname, - ) - - sync_ipr.tc( - 'add-filter', - 'u32', - index=test_link_index, - handle=0, - parent=0x10000, - prio=10, - protocol=protocols.ETH_P_IP, - target=0x10020, - keys=['0x5/0xf+0', '0x10/0xff+33'], - ) - assert filter_exists( - ifname=test_link_ifname, - kind='u32', - parent='1:', - protocol='ip', - match_value='10000000', - match_mask='ff000000', - netns=nsname, - ) - - # complementary delete commands - sync_ipr.tc( - 'del-filter', index=test_link_index, handle='0:0', parent='1:0' - ) - assert not filter_exists( - ifname=test_link_ifname, - kind='u32', - parent='1:0', - timeout=0.1, - netns=nsname, - ) - - sync_ipr.tc( - 'del-class', index=test_link_index, handle='1:20', parent='1:1' - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - netns=nsname, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - netns=nsname, - ) - - sync_ipr.tc( - 'del-class', index=test_link_index, handle='1:10', parent='1:1' - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - netns=nsname, - ) - - sync_ipr.tc('del-class', index=test_link_index, handle='1:1', parent='1:0') - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:20', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:10', - parent='1:1', - timeout=0.1, - netns=nsname, - ) - assert not class_exists( - ifname=test_link_ifname, - kind='htb', - handle='1:1', - root=True, - timeout=0.1, - netns=nsname, - ) - - sync_ipr.tc('del', index=test_link_index, handle=root_handle, root=True) - assert not qdisc_exists( - ifname=test_link_ifname, handle=root_handle, timeout=0.1, netns=nsname - ) diff --git a/tests/test_minimal/test_iproute.py b/tests/test_minimal/test_iproute.py new file mode 100644 index 000000000..e21b31901 --- /dev/null +++ b/tests/test_minimal/test_iproute.py @@ -0,0 +1,54 @@ +import getpass + +import pytest + +from pyroute2 import IPRoute +from pyroute2.common import uifname +from pyroute2.netlink import nlmsg + +pytestmark = [ + pytest.mark.skipif(getpass.getuser() != 'root', reason='no root access') +] + + +@pytest.fixture +def ipr(): + iproute = IPRoute() + iproute.default_ifname = uifname() + yield iproute + index = iproute.link_lookup(ifname=iproute.default_ifname) + if index: + iproute.link('del', index=index) + iproute.close() + + +def test_dump(ipr): + assert all([isinstance(message, nlmsg) for message in ipr.dump()]) + + +def test_tuntap(ipr): + ipr.link('add', ifname=ipr.default_ifname, kind='tuntap', mode='tun') + ipr.poll( + ipr.link, 'dump', timeout=5, ifname=ipr.default_ifname, kind='tun' + ) + + +def test_bridge(ipr): + ipr.link('add', ifname=ipr.default_ifname, kind='bridge') + interface = ipr.poll( + ipr.link, + 'dump', + timeout=5, + ifname=ipr.default_ifname, + kind='bridge', + br_stp_state=0, + )[0] + ipr.link('set', index=interface['index'], kind='bridge', br_stp_state=1) + ipr.poll( + ipr.link, + 'dump', + timeout=5, + index=interface['index'], + kind='bridge', + br_stp_state=1, + ) diff --git a/tests/test_minimal/test_nfctsocket/test_basic_async.py b/tests/test_minimal/test_nfctsocket/test_basic_async.py deleted file mode 100644 index 25845cf97..000000000 --- a/tests/test_minimal/test_nfctsocket/test_basic_async.py +++ /dev/null @@ -1,80 +0,0 @@ -import os -from typing import AsyncGenerator - -import pytest -import pytest_asyncio - -from pyroute2 import AsyncNFCTSocket -from pyroute2.netlink import nlmsg -from pyroute2.netlink.nfnetlink.nfctsocket import NFCTAttrTuple - - -@pytest_asyncio.fixture -async def nfct(): - async with AsyncNFCTSocket() as s: - yield s - - -@pytest.mark.asyncio -async def test_dump(nfct): - dump = await nfct.dump() - assert isinstance(dump, AsyncGenerator) - async for msg in dump: - assert isinstance(msg, nlmsg) - assert isinstance( - msg.get(('CTA_TUPLE_ORIG', 'CTA_TUPLE_IP', 'CTA_IP_V4_SRC')), str - ) - assert isinstance( - msg.get(('CTA_TUPLE_ORIG', 'CTA_TUPLE_PROTO', 'CTA_PROTO_NUM')), - int, - ) - - -@pytest.mark.asyncio -async def test_count(nfct): - (count,) = await nfct.count() - assert isinstance(count.get('CTA_STATS_GLOBAL_ENTRIES'), int) - assert isinstance(count.get('CTA_STATS_GLOBAL_MAX_ENTRIES'), int) - - -@pytest.mark.asyncio -async def test_stat(nfct): - stat = await nfct.stat() - assert len(stat) == os.cpu_count() - assert all(map(lambda x: isinstance(x, nlmsg), stat)) - assert any(map(lambda x: x.get('insert') > 0, stat)) - - -async def locate_entry(nfct): - async for msg in await nfct.dump(): - if ( - msg.get(('tuple_orig', 'ip', 'v4_src')) == '10.1.2.3' - and msg.get(('tuple_orig', 'proto', 'num')) == 6 - and msg.get(('tuple_orig', 'proto', 'src_port')) == 12345 - ): - return True - return False - - -@pytest.mark.parametrize('cmd,entry_exists', (('add', True), ('del', False))) -@pytest.mark.asyncio -async def _test_entry(nfct, cmd, entry_exists): - await nfct.entry( - cmd, - timeout=10, - tuple_orig=NFCTAttrTuple( - saddr='10.1.2.3', - daddr='10.3.2.1', - proto=6, - sport=12345, - dport=54321, - ), - tuple_reply=NFCTAttrTuple( - saddr='10.3.2.1', - daddr='10.1.2.3', - proto=6, - sport=54321, - dport=12345, - ), - ) - assert await locate_entry(nfct) is entry_exists diff --git a/tests/test_minimal/test_nfctsocket/test_basic_sync.py b/tests/test_minimal/test_nfctsocket/test_basic_sync.py deleted file mode 100644 index 55adad9d7..000000000 --- a/tests/test_minimal/test_nfctsocket/test_basic_sync.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -from typing import Generator - -import pytest - -from pyroute2 import NFCTSocket -from pyroute2.netlink import nlmsg -from pyroute2.netlink.nfnetlink.nfctsocket import NFCTAttrTuple - - -@pytest.fixture -def nfct(): - with NFCTSocket() as s: - yield s - - -def test_dump(nfct): - dump = nfct.dump() - assert isinstance(dump, Generator) - for msg in dump: - assert isinstance(msg, nlmsg) - assert isinstance( - msg.get(('CTA_TUPLE_ORIG', 'CTA_TUPLE_IP', 'CTA_IP_V4_SRC')), str - ) - assert isinstance( - msg.get(('CTA_TUPLE_ORIG', 'CTA_TUPLE_PROTO', 'CTA_PROTO_NUM')), - int, - ) - - -def test_count(nfct): - (count,) = nfct.count() - assert isinstance(count.get('CTA_STATS_GLOBAL_ENTRIES'), int) - assert isinstance(count.get('CTA_STATS_GLOBAL_MAX_ENTRIES'), int) - - -def test_stat(nfct): - stat = nfct.stat() - assert len(stat) == os.cpu_count() - assert all(map(lambda x: isinstance(x, nlmsg), stat)) - assert any(map(lambda x: x.get('insert') > 0, stat)) - - -def locate_entry(nfct): - for msg in nfct.dump(): - if ( - msg.get(('tuple_orig', 'ip', 'v4_src')) == '10.1.2.3' - and msg.get(('tuple_orig', 'proto', 'num')) == 6 - and msg.get(('tuple_orig', 'proto', 'src_port')) == 12345 - ): - return True - return False - - -@pytest.mark.parametrize('cmd,entry_exists', (('add', True), ('del', False))) -def test_entry(nfct, cmd, entry_exists): - nfct.entry( - cmd, - timeout=10, - tuple_orig=NFCTAttrTuple( - saddr='10.1.2.3', - daddr='10.3.2.1', - proto=6, - sport=12345, - dport=54321, - ), - tuple_reply=NFCTAttrTuple( - saddr='10.3.2.1', - daddr='10.1.2.3', - proto=6, - sport=54321, - dport=12345, - ), - ) - assert locate_entry(nfct) is entry_exists diff --git a/tests/test_minimal/test_nfctsocket/test_conntrack_async.py b/tests/test_minimal/test_nfctsocket/test_conntrack_async.py deleted file mode 100644 index e330c48ba..000000000 --- a/tests/test_minimal/test_nfctsocket/test_conntrack_async.py +++ /dev/null @@ -1,148 +0,0 @@ -import os -from functools import partial -from typing import AsyncGenerator - -import pytest -import pytest_asyncio - -from pyroute2 import netns -from pyroute2.common import uifname -from pyroute2.conntrack import AsyncConntrack, ConntrackEntry -from pyroute2.netlink.nfnetlink.nfctsocket import NFCTAttrTuple - - -class ArgMeta(type): - @property - def NONE(cls) -> None: - return None - - @property - def AUTO(cls) -> str: - return uifname() - - -class ArgNetNS(metaclass=ArgMeta): - pass - - -@pytest_asyncio.fixture -async def ct(request): - nsname = getattr(request, 'param', None) - async with AsyncConntrack(netns=nsname) as s: - yield s - if nsname is not None: - netns.remove(nsname) - - -@pytest.mark.asyncio -async def test_dump(ct): - dump = await ct.dump_entries() - assert isinstance(dump, AsyncGenerator) - async for entry in dump: - assert isinstance(entry, ConntrackEntry) - assert isinstance(entry.tuple_orig.saddr, str) - assert isinstance(entry.tuple_orig.proto, int) - - -@pytest.mark.parametrize( - ('ct', 'check_stat'), - ( - (ArgNetNS.NONE, lambda i: any(map(lambda x: x.get('insert') > 0, i))), - (ArgNetNS.AUTO, lambda i: all(map(lambda x: x.get('insert') == 0, i))), - ), - indirect=('ct',), -) -@pytest.mark.asyncio -async def test_stat(ct, check_stat): - stat = await ct.stat() - assert len(stat) == os.cpu_count() - assert check_stat(stat) - - -async def entry_locate(target, ct): - ret = False - async for entry in await ct.dump_entries(): - if ( - entry.tuple_orig.saddr == '10.1.2.3' - and entry.tuple_orig.proto == 6 - and entry.tuple_orig.sport == 12345 - ): - ret = True - break - assert ret is target - - -async def entry_count(func, ct): - count = await ct.count() - assert isinstance(count, int) - assert func(count) - - -async def entry_op(cmd, ct): - await ct.entry( - cmd, - timeout=10, - tuple_orig=NFCTAttrTuple( - saddr='10.1.2.3', - daddr='10.3.2.1', - proto=6, - sport=12345, - dport=54321, - ), - tuple_reply=NFCTAttrTuple( - saddr='10.3.2.1', - daddr='10.1.2.3', - proto=6, - sport=54321, - dport=12345, - ), - ) - - -@pytest.mark.parametrize( - ('ct', 'steps'), - ( - ( - ArgNetNS.NONE, - ( - partial(entry_op, 'add'), - partial(entry_locate, True), - partial(entry_op, 'del'), - partial(entry_locate, False), - ), - ), - ( - ArgNetNS.AUTO, - ( - partial(entry_op, 'add'), - partial(entry_locate, True), - partial(entry_op, 'del'), - partial(entry_locate, False), - ), - ), - ( - ArgNetNS.NONE, - ( - partial(entry_op, 'add'), - partial(entry_count, lambda x: x > 0), - partial(entry_op, 'del'), - ), - ), - ( - ArgNetNS.AUTO, - ( - partial(entry_count, lambda x: x == 0), - partial(entry_op, 'add'), - partial(entry_count, lambda x: x == 1), - partial(entry_op, 'del'), - partial(entry_count, lambda x: x == 0), - ), - ), - ), - indirect=('ct',), - ids=('locate', 'locate (netns)', 'count', 'count (netns)'), -) -@pytest.mark.asyncio -async def test_op(ct, steps): - for step in steps: - await step(ct) diff --git a/tests/test_minimal/test_nfctsocket/test_conntrack_sync.py b/tests/test_minimal/test_nfctsocket/test_conntrack_sync.py deleted file mode 100644 index 0fec482f9..000000000 --- a/tests/test_minimal/test_nfctsocket/test_conntrack_sync.py +++ /dev/null @@ -1,144 +0,0 @@ -import os -from functools import partial -from typing import Generator - -import pytest - -from pyroute2 import netns -from pyroute2.common import uifname -from pyroute2.conntrack import Conntrack, ConntrackEntry -from pyroute2.netlink.nfnetlink.nfctsocket import NFCTAttrTuple - - -class ArgMeta(type): - @property - def NONE(cls) -> None: - return None - - @property - def AUTO(cls) -> str: - return uifname() - - -class ArgNetNS(metaclass=ArgMeta): - pass - - -@pytest.fixture -def ct(request): - nsname = getattr(request, 'param', None) - with Conntrack(netns=nsname) as s: - yield s - if nsname is not None: - netns.remove(nsname) - - -def test_dump(ct): - dump = ct.dump_entries() - assert isinstance(dump, Generator) - for entry in dump: - assert isinstance(entry, ConntrackEntry) - assert isinstance(entry.tuple_orig.saddr, str) - assert isinstance(entry.tuple_orig.proto, int) - - -@pytest.mark.parametrize( - ('ct', 'check_stat'), - ( - (ArgNetNS.NONE, lambda i: any(map(lambda x: x.get('insert') > 0, i))), - (ArgNetNS.AUTO, lambda i: all(map(lambda x: x.get('insert') == 0, i))), - ), - indirect=('ct',), -) -def test_stat(ct, check_stat): - stat = ct.stat() - assert len(stat) == os.cpu_count() - assert check_stat(stat) - - -def entry_locate(target, ct): - ret = False - for entry in ct.dump_entries(): - if ( - entry.tuple_orig.saddr == '10.1.2.3' - and entry.tuple_orig.proto == 6 - and entry.tuple_orig.sport == 12345 - ): - ret = True - break - assert ret is target - - -def entry_count(func, ct): - count = ct.count() - assert isinstance(count, int) - assert func(count) - - -def entry_op(cmd, ct): - ct.entry( - cmd, - timeout=10, - tuple_orig=NFCTAttrTuple( - saddr='10.1.2.3', - daddr='10.3.2.1', - proto=6, - sport=12345, - dport=54321, - ), - tuple_reply=NFCTAttrTuple( - saddr='10.3.2.1', - daddr='10.1.2.3', - proto=6, - sport=54321, - dport=12345, - ), - ) - - -@pytest.mark.parametrize( - ('ct', 'steps'), - ( - ( - ArgNetNS.NONE, - ( - partial(entry_op, 'add'), - partial(entry_locate, True), - partial(entry_op, 'del'), - partial(entry_locate, False), - ), - ), - ( - ArgNetNS.AUTO, - ( - partial(entry_op, 'add'), - partial(entry_locate, True), - partial(entry_op, 'del'), - partial(entry_locate, False), - ), - ), - ( - ArgNetNS.NONE, - ( - partial(entry_op, 'add'), - partial(entry_count, lambda x: x > 0), - partial(entry_op, 'del'), - ), - ), - ( - ArgNetNS.AUTO, - ( - partial(entry_count, lambda x: x == 0), - partial(entry_op, 'add'), - partial(entry_count, lambda x: x == 1), - partial(entry_op, 'del'), - partial(entry_count, lambda x: x == 0), - ), - ), - ), - indirect=('ct',), - ids=('locate', 'locate (netns)', 'count', 'count (netns)'), -) -def test_op(ct, steps): - for step in steps: - step(ct) diff --git a/tests/test_minimal/test_nftables/test_expressions.py b/tests/test_minimal/test_nftables/test_expressions.py deleted file mode 100644 index b326f6fc1..000000000 --- a/tests/test_minimal/test_nftables/test_expressions.py +++ /dev/null @@ -1,70 +0,0 @@ -import pytest - -from pyroute2.common import uifname -from pyroute2.nftables.expressions import ipv4addr, masq, verdict -from pyroute2.nftables.main import AsyncNFTables, NFTables - -MAGIC = 'pyroute2-テスト — pröva UTF-8 kommentar — ' - - -@pytest.mark.parametrize( - 'get_chain,get_target', - ( - (lambda x: 'POSTROUTING', lambda x: masq()), - (lambda x: x.chain, lambda x: verdict(code=1)), - ), - ids=['masq', 'accept'], -) -@pytest.mark.asyncio -async def test_add_rule_async(nft, get_chain, get_target): - global MAGIC - magic = MAGIC + uifname() - async with AsyncNFTables() as cmd: - async for rule in await cmd.get_rules(): - if rule.get('userdata') == magic: - raise RuntimeError('magic exists') - - await cmd.rule( - 'add', - table=nft.table, - chain=get_chain(nft), - expressions=(ipv4addr(src='10.244.0.0/16'), get_target(nft)), - userdata=magic, - ) - - async for rule in await cmd.get_rules(): - if rule.get('userdata') == magic: - break - else: - raise RuntimeError('magic does not exist') - - -@pytest.mark.parametrize( - 'get_chain,get_target', - ( - (lambda x: 'POSTROUTING', lambda x: masq()), - (lambda x: x.chain, lambda x: verdict(code=1)), - ), - ids=['masq', 'accept'], -) -def test_add_rule_sync(nft, get_chain, get_target): - global MAGIC - magic = MAGIC + uifname() - with NFTables() as cmd: - for rule in cmd.get_rules(): - if rule.get('userdata') == magic: - raise RuntimeError('magic exists') - - cmd.rule( - 'add', - table=nft.table, - chain=get_chain(nft), - expressions=(ipv4addr(src='10.244.0.0/16'), get_target(nft)), - userdata=magic, - ) - - for rule in cmd.get_rules(): - if rule.get('userdata') == magic: - break - else: - raise RuntimeError('magic does not exist') diff --git a/tests/test_minimal/test_nftables/test_nftsocket.py b/tests/test_minimal/test_nftables/test_nftsocket.py deleted file mode 100644 index d50470990..000000000 --- a/tests/test_minimal/test_nftables/test_nftsocket.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - -from pyroute2 import AsyncNFTSocket, NFTSocket -from pyroute2.netlink.nfnetlink import nfgen_msg -from pyroute2.netlink.nfnetlink.nftsocket import ( - NFT_MSG_GETCHAIN, - NFT_MSG_GETTABLE, -) - -list_objects = pytest.mark.parametrize( - 'cmd,get_field', - ( - (NFT_MSG_GETTABLE, lambda x: x.table), - (NFT_MSG_GETCHAIN, lambda x: x.chain), - ), - ids=['table', 'chain'], -) - - -@list_objects -def test_list_sync(nft, cmd, get_field): - with NFTSocket() as sock: - objects = [ - msg.get('name') for msg in sock.request_get(nfgen_msg(), cmd) - ] - assert get_field(nft) in objects - - -@list_objects -@pytest.mark.asyncio -async def test_list_async(nft, cmd, get_field): - async with AsyncNFTSocket() as sock: - objects = [ - msg.get('name') - async for msg in await sock.request_get(nfgen_msg(), cmd) - ] - assert get_field(nft) in objects diff --git a/tests/test_neutron/test_ip_lib.py b/tests/test_neutron/test_ip_lib.py index 64672aef2..31d772d41 100644 --- a/tests/test_neutron/test_ip_lib.py +++ b/tests/test_neutron/test_ip_lib.py @@ -1,17 +1,12 @@ from inspect import signature -import eventlet import pytest import pyroute2 from pyroute2 import netlink, netns -from pyroute2.config.eventlet import eventlet_config from pyroute2.netlink import exceptions, rtnl from pyroute2.netlink.rtnl import ifinfmsg, ndmsg -eventlet.monkey_patch() -eventlet_config() - def parameters(func): try: @@ -31,6 +26,7 @@ def test_imports(): assert parameters(rtnl.rt_scope.get) == set(('key', 'default')) assert isinstance(rtnl.rt_proto, dict) and 'static' in rtnl.rt_proto assert parameters(netns._create) == set(('netns', 'libc', 'pid')) + assert parameters(netns.remove) == set(('netns', 'libc')) assert parameters(netns.listnetns) == set(('nspath',)) assert ifinfmsg.IFF_ALLMULTI == 0x200 assert {state[1]: state[0] for state in ndmsg.states.items()} == { @@ -44,9 +40,3 @@ def test_imports(): 64: 'noarp', 128: 'permanent', } - - -def test_dump(): - with pyroute2.IPRoute() as ipr: - assert len(tuple(ipr.route('dump'))) > 0 - assert len(tuple(ipr.link('dump'))) > 0 diff --git a/tests/test_process/test_basic.py b/tests/test_process/test_basic.py deleted file mode 100644 index 6e384290f..000000000 --- a/tests/test_process/test_basic.py +++ /dev/null @@ -1,81 +0,0 @@ -import errno -import multiprocessing as mp - -import pytest - -from pyroute2 import NetlinkError, config -from pyroute2.process import ChildProcess - -exc_map = { - 'OSError': (OSError, (errno.ENOENT, 'message')), - 'NetlinkError': (NetlinkError, (errno.EEXIST, 'message')), - 'RuntimeError': (RuntimeError, ()), - 'AttributeError': (AttributeError, ()), - 'KeyError': (KeyError, ()), -} - - -def _child(arg_str, arg_int, arg_bool): - assert isinstance(arg_str, str) - assert isinstance(arg_int, int) - assert isinstance(arg_bool, bool) - - -def _child_exceptions(exc_type): - global exc_map - spec = exc_map[exc_type] - raise spec[0](*spec[1]) - - -@pytest.mark.parametrize('exc_type', list(exc_map.keys())) -def test_exceptions(exc_type): - with pytest.raises(exc_map[exc_type][0]) as exc_info: - with ChildProcess(target=_child_exceptions, args=[exc_type]) as proc: - proc.communicate() - print(exc_info) - - -@pytest.mark.parametrize( - 'mode,check_attr', - ( - ('fork', lambda x: isinstance(x.pid, int)), - ('mp', lambda x: isinstance(x.proc, mp.Process)), - ), -) -def test_modes(mode, check_attr): - old_mode = config.child_process_mode - config.child_process_mode = mode - with ChildProcess(target=_child, args=["str", 1, True]) as proc: - assert proc.mode == mode - assert check_attr(proc) - proc.communicate() - config.child_process_mode = old_mode - - -@pytest.mark.parametrize( - 'exc,args', ((AssertionError, ['str', 1, 0]), (TypeError, [])) -) -def test_args_fail(exc, args): - with pytest.raises(exc): - with ChildProcess(target=_child, args=args) as proc: - proc.communicate() - - -def child_process_case_01(x): - return b' ' * x - - -def child_process_case_02(): - return None - - -@pytest.mark.parametrize( - 'func,args,ret', - ( - (child_process_case_01, [10], (b' ', [])), - (child_process_case_02, [], (b'', [])), - ), -) -def test_simple_args(func, args, ret): - with ChildProcess(func, args) as proc: - assert proc.communicate() == ret diff --git a/tests/test_process/test_catastrophe.py b/tests/test_process/test_catastrophe.py deleted file mode 100644 index e47b6eab3..000000000 --- a/tests/test_process/test_catastrophe.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import time -from signal import SIGKILL, SIGTERM - -import pytest - -from pyroute2.process import ChildProcess - - -def child_process_timeout(x): - time.sleep(x) - - -def child_process_die(x): - os.kill(os.getpid(), x) - - -@pytest.mark.parametrize( - 'func,argv,catch,kill,exitcode', - ( - (child_process_timeout, [1], TimeoutError, None, -SIGKILL), - (child_process_timeout, [7], TimeoutError, None, -SIGKILL), - (child_process_timeout, [23], TimeoutError, None, -SIGKILL), - (child_process_die, [SIGTERM], RuntimeError, None, -SIGTERM), - (child_process_die, [SIGKILL], RuntimeError, None, -SIGKILL), - (child_process_timeout, [30], RuntimeError, SIGTERM, -SIGTERM), - (child_process_timeout, [30], RuntimeError, SIGKILL, -SIGKILL), - ), - ids=[ - 'timeout-1', - 'timeout-7', - 'timeout-23', - 'die-SIGTERM', - 'die-SIGKILL', - 'kill-SIGTERM', - 'kill-SIGKILL', - ], -) -def test_child_fail(func, argv, catch, kill, exitcode): - cp = ChildProcess(func, argv) - ts_start = time.time() - with pytest.raises(catch): - cp.run() - if kill is not None: - os.kill(cp.pid, kill) - cp.communicate(timeout=0.1) - assert time.time() - ts_start < 1 - assert cp.exitcode == exitcode - cp.close() diff --git a/tests/test_repo/test_noxfile.py b/tests/test_repo/test_noxfile.py index 1965c0df2..a25d6d445 100644 --- a/tests/test_repo/test_noxfile.py +++ b/tests/test_repo/test_noxfile.py @@ -1,7 +1,8 @@ import ast import collections import inspect -import tomllib +import os +import sys import nox import pytest @@ -23,6 +24,9 @@ def session(request): ) +@pytest.mark.skipif( + sys.version_info < (3, 8), reason='unsupported Python version' +) @pytest.mark.parametrize('session', nox_sessions, indirect=True) def test_options_call(session): # walk the AST tree @@ -49,20 +53,11 @@ def test_session_parameters(session): assert args == ['session'] +@pytest.mark.skipif( + sys.version_info < (3, 8), reason='unsupported Python version' +) @pytest.mark.parametrize('session', nox_sessions, indirect=True) -def test_optional_dependencies(session): - keys = set() - flavours = set() - for pyproject in ('pyproject.toml', 'pyproject.minimal.toml'): - with open(pyproject, 'rb') as f: - toml = tomllib.load(f) - current_keys = set(toml['project']['optional-dependencies'].keys()) - if not keys: - keys = current_keys - # here we assert that all the pyproject files have the same - # optional dependencies - assert keys == current_keys - +def test_requirements_files(session): for node in ast.walk(ast.parse(inspect.getsource(session.src_func))): # # inspect function calls, filter direct or indirect install @@ -83,12 +78,13 @@ def test_optional_dependencies(session): .parameters['flavour'] .default ) - flavours.add(flavour) + assert os.stat(f'requirements.{flavour}.txt') elif ( isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.attr == 'install' and node.args[0].value == '-r' ): - raise Exception('no `pip install -r` allowed') - assert keys >= flavours + # + # inspect call `session.install('-r', ...)` -- direct install + assert os.stat(node.args[1].value) diff --git a/tests/test_repo/test_version.py b/tests/test_repo/test_version.py index 006a294d3..5fb825c77 100644 --- a/tests/test_repo/test_version.py +++ b/tests/test_repo/test_version.py @@ -2,7 +2,7 @@ import re import pytest -from packaging.version import InvalidVersion, Version +from setuptools._vendor import packaging @pytest.fixture @@ -19,7 +19,7 @@ def files(): def test_static_version_file(files): assert re.match( - r'^[0-9]\.[0-9]\.[0-9]{1,2}(a[0-9]+|b[0-9]+|rc[0-9]+){0,1}$', + r'^[0-9]\.[0-9]\.[0-9]{1,2}(\.post[0-9]+|\.rc[0-9]+){0,1}$', files['VERSION'].getvalue().strip(), ) @@ -29,11 +29,6 @@ def test_changelog(files): for line in files['CHANGELOG.rst'].readlines(): if line[0] == '*': break - - try: - static_version = Version(files['VERSION'].getvalue().strip()) - last_changelog_version = Version(line.split()[1]) - except InvalidVersion as e: - pytest.fail(f"Invalid version encountered: {e}") - + static_version = packaging.version.parse(files['VERSION'].getvalue()) + last_changelog_version = packaging.version.parse(line.split()[1]) assert static_version >= last_changelog_version diff --git a/tests/test_unit/test_addr_pool.py b/tests/test_unit/test_addr_pool.py index f19e2a98f..e22e3f66d 100644 --- a/tests/test_unit/test_addr_pool.py +++ b/tests/test_unit/test_addr_pool.py @@ -52,3 +52,38 @@ def test_locate(): assert bit2 == bit1 + 1 assert is_allocated1 assert not is_allocated2 + assert ap.allocated == 1 + + +def test_setaddr_allocated(): + ap = AddrPool() + f = ap.alloc() + base, bit, is_allocated = ap.locate(f + 1) + assert not is_allocated + assert ap.allocated == 1 + ap.setaddr(f + 1, 'allocated') + base, bit, is_allocated = ap.locate(f + 1) + assert is_allocated + assert ap.allocated == 2 + ap.free(f + 1) + base, bit, is_allocated = ap.locate(f + 1) + assert not is_allocated + assert ap.allocated == 1 + + +def test_setaddr_free(): + ap = AddrPool() + f = ap.alloc() + base, bit, is_allocated = ap.locate(f + 1) + assert not is_allocated + assert ap.allocated == 1 + ap.setaddr(f + 1, 'free') + base, bit, is_allocated = ap.locate(f + 1) + assert not is_allocated + assert ap.allocated == 1 + ap.setaddr(f, 'free') + base, bit, is_allocated = ap.locate(f) + assert not is_allocated + assert ap.allocated == 0 + with pytest.raises(KeyError): + ap.free(f) diff --git a/tests/test_unit/test_common.py b/tests/test_unit/test_common.py index 50fcd8ac4..274b2062c 100644 --- a/tests/test_unit/test_common.py +++ b/tests/test_unit/test_common.py @@ -1,4 +1,15 @@ -from pyroute2.common import dqn2int, map_namespace, uifname, uuid32 +from pyroute2.common import dqn2int, hexdump, hexload, uifname, uuid32 + + +def test_hexdump(): + binary = b'abcdef5678' + dump1 = hexdump(binary) + dump2 = hexdump(binary, length=6) + assert len(dump1) == 29 + assert len(dump2) == 17 + assert dump1[2] == dump1[-3] == dump2[2] == dump2[-3] == ':' + assert hexload(dump1) == binary + assert hexload(dump2) == binary[:6] def test_uuid32(): @@ -23,56 +34,3 @@ def test_uifname(): nB = uifname() assert nA != nB assert int(nA[2:], 16) != int(nB[2:], 16) - - -def test_map_namespace(): - others = { - 'IFNAMSIZ': 16, - '__name__': 'pyroute2.bsd.pf_route.freebsd', - '__doc__': None, - } - prefixed = {'IFF_UP': 1, 'IFF_NOGROUP': 8388608} - prefixed_by_value = {1: 'IFF_UP', 8388608: 'IFF_NOGROUP'} - ns = prefixed | others - - by_name, by_value = map_namespace('IFF', ns) - - assert by_name == prefixed - assert by_value == prefixed_by_value - - -def test_map_namespace_normalize(): - others = { - 'IFNAMSIZ': 16, - '__name__': 'pyroute2.bsd.pf_route.freebsd', - '__doc__': None, - } - prefixed = {'IFF_UP': 1, 'IFF_NOGROUP': 8388608} - normalized = {'_up': 1, '_nogroup': 8388608} - normalized_by_value = {1: '_up', 8388608: '_nogroup'} - ns = prefixed | others - - by_name, by_value = map_namespace('IFF', ns, True) - - assert by_name == normalized - assert by_value == normalized_by_value - - -def test_map_namespace_normalize_function(): - def normalizer(s: str) -> str: - return s.removeprefix('IFF_') - - others = { - 'IFNAMSIZ': 16, - '__name__': 'pyroute2.bsd.pf_route.freebsd', - '__doc__': None, - } - prefixed = {'IFF_UP': 1, 'IFF_NOGROUP': 8388608} - normalized = {'UP': 1, 'NOGROUP': 8388608} - normalized_by_value = {1: 'UP', 8388608: 'NOGROUP'} - ns = prefixed | others - - by_name, by_value = map_namespace('IFF', ns, normalizer) - - assert by_name == normalized - assert by_value == normalized_by_value diff --git a/tests/test_unit/test_iproute_match/test_match.py b/tests/test_unit/test_iproute_match/test_match.py index f7b3148e3..f8ba8cc7d 100644 --- a/tests/test_unit/test_iproute_match/test_match.py +++ b/tests/test_unit/test_iproute_match/test_match.py @@ -62,10 +62,10 @@ def ipr(): ) def test_get_leaf(ipr, spec, query, result): spec = RequestProcessor(context=spec, prime=spec) - spec.add_filter(LinkFieldFilter()) - spec.add_filter(LinkIPRouteFilter('dump')) + spec.apply_filter(LinkFieldFilter()) + spec.apply_filter(LinkIPRouteFilter('dump')) spec.finalize() - msg = tuple(ipr.filter_messages(spec, ifinfmsg_sample)) + msg = ipr.filter_messages(spec, ifinfmsg_sample) assert len(msg) == 1 assert msg[0].get_nested(*query) == result diff --git a/tests/test_unit/test_nlmsg/test_marshal.py b/tests/test_unit/test_nlmsg/test_marshal.py index f2d7e833b..573f22367 100644 --- a/tests/test_unit/test_nlmsg/test_marshal.py +++ b/tests/test_unit/test_nlmsg/test_marshal.py @@ -10,7 +10,6 @@ from pyroute2.netlink.nl80211 import MarshalNl80211 from pyroute2.netlink.rtnl import RTM_NEWADDR, RTM_NEWLINK from pyroute2.netlink.rtnl.iprsocket import MarshalRtnl -from pyroute2.netlink.uevent import MarshalUevent def load_sample(sample): @@ -43,8 +42,6 @@ def run_using_marshal(sample, marshal): ('test_unit/test_nlmsg/gre_01.dump', MarshalRtnl()), ('test_unit/test_nlmsg/iw_info_rsp.dump', MarshalNl80211()), ('test_unit/test_nlmsg/iw_scan_rsp.dump', MarshalNl80211()), - ('test_unit/test_nlmsg/uevent_udevd_backlight.dump', MarshalUevent()), - ('test_unit/test_nlmsg/uevent_kernel_backlight.dump', MarshalUevent()), ), ) def test_marshal(sample, marshal): diff --git a/tests/test_unit/test_nlmsg/uevent_kernel_backlight.dump b/tests/test_unit/test_nlmsg/uevent_kernel_backlight.dump deleted file mode 100644 index 59b9b61f3..000000000 --- a/tests/test_unit/test_nlmsg/uevent_kernel_backlight.dump +++ /dev/null @@ -1,45 +0,0 @@ -# sample -6c:69:62:75:64:65:76:00:fe:ed:ca:fe:28:00:00:00:28:00:00:00:a1:01:00:00:c3:70 -b3:02:00:00:00:00:02:08:04:00:10:c0:00:01:55:44:45:56:5f:44:41:54:41:42:41:53 -45:5f:56:45:52:53:49:4f:4e:3d:31:00:41:43:54:49:4f:4e:3d:63:68:61:6e:67:65:00 -44:45:56:50:41:54:48:3d:2f:64:65:76:69:63:65:73:2f:70:63:69:30:30:30:30:3a:30 -30:2f:30:30:30:30:3a:30:30:3a:30:32:2e:30:2f:64:72:6d:2f:63:61:72:64:30:2f:63 -61:72:64:30:2d:65:44:50:2d:31:2f:69:6e:74:65:6c:5f:62:61:63:6b:6c:69:67:68:74 -00:53:55:42:53:59:53:54:45:4d:3d:62:61:63:6b:6c:69:67:68:74:00:53:4f:55:52:43 -45:3d:73:79:73:66:73:00:53:45:51:4e:55:4d:3d:31:39:33:37:31:00:55:53:45:43:5f -49:4e:49:54:49:41:4c:49:5a:45:44:3d:31:35:35:38:34:38:33:37:00:54:41:47:53:3d -3a:73:79:73:74:65:6d:64:3a:73:65:61:74:3a:00:4e:56:4d:45:5f:48:4f:53:54:5f:49 -46:41:43:45:3d:6e:6f:6e:65:00:43:55:52:52:45:4e:54:5f:54:41:47:53:3d:3a:73:79 -73:74:65:6d:64:3a:73:65:61:74:3a:00:49:44:5f:50:41:54:48:3d:70:63:69:2d:30:30 -30:30:3a:30:30:3a:30:32:2e:30:00:49:44:5f:50:41:54:48:5f:54:41:47:3d:70:63:69 -2d:30:30:30:30:5f:30:30:5f:30:32:5f:30:00:49:44:5f:46:4f:52:5f:53:45:41:54:3d -62:61:63:6b:6c:69:67:68:74:2d:70:63:69:2d:30:30:30:30:5f:30:30:5f:30:32:5f:30 -00:53:59:53:54:45:4d:44:5f:57:41:4e:54:53:3d:73:79:73:74:65:6d:64:2d:62:61:63 -6b:6c:69:67:68:74:40:62:61:63:6b:6c:69:67:68:74:3a:69:6e:74:65:6c:5f:62:61:63 -6b:6c:69:67:68:74:2e:73:65:72:76:69:63:65:00 - -# parsed data should match primes below -#: application/x-python-code -( - { - 'attrs': [], - 'header': { - 'sequence_number': 0, - 'message': 'libudev', - 'unparsed': b'' - }, - 'ACTION': 'change', - 'DEVPATH': '/devices/pci0000:00/0000:00:02.0/drm/card0/card0-eDP-1/intel_backlight', - 'SUBSYSTEM': 'backlight', - 'SOURCE': 'sysfs', - 'SEQNUM': '19371', - 'USEC_INITIALIZED': '15584837', - 'TAGS': ':systemd:seat:', - 'NVME_HOST_IFACE': 'none', - 'CURRENT_TAGS': ':systemd:seat:', - 'ID_PATH': 'pci-0000:00:02.0', - 'ID_PATH_TAG': 'pci-0000_00_02_0', - 'ID_FOR_SEAT': 'backlight-pci-0000_00_02_0', - 'SYSTEMD_WANTS': 'systemd-backlight@backlight:intel_backlight.service' - }, -) diff --git a/tests/test_unit/test_nlmsg/uevent_udevd_backlight.dump b/tests/test_unit/test_nlmsg/uevent_udevd_backlight.dump deleted file mode 100644 index 61ba8bcfa..000000000 --- a/tests/test_unit/test_nlmsg/uevent_udevd_backlight.dump +++ /dev/null @@ -1,28 +0,0 @@ -# sample -63:68:61:6e:67:65:40:2f:64:65:76:69:63:65:73:2f:70:63:69:30:30:30:30:3a:30:30 -2f:30:30:30:30:3a:30:30:3a:30:32:2e:30:2f:64:72:6d:2f:63:61:72:64:30:2f:63:61 -72:64:30:2d:65:44:50:2d:31:2f:69:6e:74:65:6c:5f:62:61:63:6b:6c:69:67:68:74:00 -41:43:54:49:4f:4e:3d:63:68:61:6e:67:65:00:44:45:56:50:41:54:48:3d:2f:64:65:76 -69:63:65:73:2f:70:63:69:30:30:30:30:3a:30:30:2f:30:30:30:30:3a:30:30:3a:30:32 -2e:30:2f:64:72:6d:2f:63:61:72:64:30:2f:63:61:72:64:30:2d:65:44:50:2d:31:2f:69 -6e:74:65:6c:5f:62:61:63:6b:6c:69:67:68:74:00:53:55:42:53:59:53:54:45:4d:3d:62 -61:63:6b:6c:69:67:68:74:00:53:4f:55:52:43:45:3d:73:79:73:66:73:00:53:45:51:4e -55:4d:3d:31:39:33:37:32:00 - -# parsed data should match primes below -#: application/x-python-code -( - { - 'attrs': [], - 'header': { - 'sequence_number': 0, - 'message': 'change@/devices/pci0000:00/0000:00:02.0/drm/card0/card0-eDP-1/intel_backlight', - 'unparsed': b'' - }, - 'ACTION': 'change', - 'DEVPATH': '/devices/pci0000:00/0000:00:02.0/drm/card0/card0-eDP-1/intel_backlight', - 'SUBSYSTEM': 'backlight', - 'SOURCE': 'sysfs', - 'SEQNUM': '19372' - }, -) diff --git a/tests/test_unit/test_requests/common.py b/tests/test_unit/test_requests/common.py index cab8518f0..e96b80eeb 100644 --- a/tests/test_unit/test_requests/common.py +++ b/tests/test_unit/test_requests/common.py @@ -12,6 +12,6 @@ class Result(dict): def run_test(config, spec, result): processor = RequestProcessor(context=spec, prime=spec) for fspec in config['filters']: - processor.add_filter(fspec['class'](*fspec['argv'])) + processor.apply_filter(fspec['class'](*fspec['argv'])) processor.finalize() assert Result(processor) == result diff --git a/tests/test_unit/test_requests/test_address.py b/tests/test_unit/test_requests/test_address.py index 9db610605..ba4d4e138 100644 --- a/tests/test_unit/test_requests/test_address.py +++ b/tests/test_unit/test_requests/test_address.py @@ -13,139 +13,6 @@ } -## -# -# IPv6 cacheinfo -# -@pytest.mark.parametrize( - 'spec,result', - ( - ( - Request( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'preferred_lft': 99, - } - ), - Result( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'family': AF_INET6, - 'cacheinfo': { - 'ifa_preferred': 99, - 'ifa_valid': 4294967295, - }, - } - ), - ), - ( - Request( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'preferred': 99, - } - ), - Result( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'family': AF_INET6, - 'cacheinfo': { - 'ifa_preferred': 99, - 'ifa_valid': 4294967295, - }, - } - ), - ), - ( - Request( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'valid_lft': 109, - 'preferred': 99, - } - ), - Result( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'family': AF_INET6, - 'cacheinfo': {'ifa_preferred': 99, 'ifa_valid': 109}, - } - ), - ), - ( - Request( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'valid': 109, - 'preferred': 99, - } - ), - Result( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'family': AF_INET6, - 'cacheinfo': {'ifa_preferred': 99, 'ifa_valid': 109}, - } - ), - ), - ), - ids=('preferred_lft', 'preferred', 'valid_lft', 'valid'), -) -def test_cacheinfo(spec, result): - return run_test(config, spec, result) - - -@pytest.mark.parametrize( - 'spec,result', - ( - ( - Request( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'valid': 109, - 'preferred': 990, - } - ), - Result({}), - ), - ( - Request( - { - 'index': 1, - 'address': '2001:db8::5678', - 'prefixlen': 128, - 'valid': 109, - } - ), - Result({}), - ), - ), - ids=('with-preferred', 'without-preferred'), -) -def test_cacheinfo_valid_fail(spec, result): - with pytest.raises(ValueError) as e: - run_test(config, spec, result) - assert e.value.args == ('preferred_lft is greater than valid_lft',) - - ## # # broadcast tests: bool diff --git a/tests/test_unit/test_requests/test_link.py b/tests/test_unit/test_requests/test_link.py index df9ea8435..e6c461dcd 100644 --- a/tests/test_unit/test_requests/test_link.py +++ b/tests/test_unit/test_requests/test_link.py @@ -1,7 +1,6 @@ import pytest from common import Request, Result, run_test -from pyroute2.netlink.rtnl.ifinfmsg import IFF_NOARP, IFF_UP from pyroute2.requests.link import LinkFieldFilter, LinkIPRouteFilter config_add = { @@ -90,35 +89,3 @@ def test_index(config, spec, result): ) def test_dump_specific(spec, result): return run_test(config_dump, spec, result) - - -@pytest.mark.parametrize( - 'spec,result', - ( - ( - Request({'kind': 'bridge', 'state': 'up'}), - Result( - { - 'flags': IFF_UP, - 'change': IFF_UP, - 'index': 0, - 'IFLA_LINKINFO': {'attrs': [['IFLA_INFO_KIND', 'bridge']]}, - } - ), - ), - ( - Request({'kind': 'bridge', 'state': 'up', 'noarp': True}), - Result( - { - 'flags': IFF_UP | IFF_NOARP, - 'change': IFF_UP | IFF_NOARP, - 'index': 0, - 'IFLA_LINKINFO': {'attrs': [['IFLA_INFO_KIND', 'bridge']]}, - } - ), - ), - ), - ids=['flags1', 'flags2'], -) -def test_add_filter(spec, result): - return run_test(config_add, spec, result) diff --git a/tests/test_unit/test_requests/test_neighbour.py b/tests/test_unit/test_requests/test_neighbour.py index 31910a5a1..4ee61960e 100644 --- a/tests/test_unit/test_requests/test_neighbour.py +++ b/tests/test_unit/test_requests/test_neighbour.py @@ -17,7 +17,7 @@ } -result = Result({'ifindex': 1, 'state': NUD_PERMANENT}) +result = Result({'ifindex': 1, 'family': AF_INET, 'state': NUD_PERMANENT}) @pytest.mark.parametrize( @@ -73,27 +73,27 @@ def test_family(spec, result): ( ( Request({'ifindex': 1, 'state': 'permanent'}), - Result({'ifindex': 1, 'state': NUD_PERMANENT}), + Result({'ifindex': 1, 'state': NUD_PERMANENT, 'family': AF_INET}), ), ( Request({'ifindex': 1, 'state': 'failed'}), - Result({'ifindex': 1, 'state': NUD_FAILED}), + Result({'ifindex': 1, 'state': NUD_FAILED, 'family': AF_INET}), ), ( Request({'ifindex': 1, 'nud': 'permanent'}), - Result({'ifindex': 1, 'state': NUD_PERMANENT}), + Result({'ifindex': 1, 'state': NUD_PERMANENT, 'family': AF_INET}), ), ( Request({'ifindex': 1, 'nud': 'failed'}), - Result({'ifindex': 1, 'state': NUD_FAILED}), + Result({'ifindex': 1, 'state': NUD_FAILED, 'family': AF_INET}), ), ( Request({'ifindex': 1, 'nud': NUD_PERMANENT}), - Result({'ifindex': 1, 'state': NUD_PERMANENT}), + Result({'ifindex': 1, 'state': NUD_PERMANENT, 'family': AF_INET}), ), ( Request({'ifindex': 1, 'nud': NUD_FAILED}), - Result({'ifindex': 1, 'state': NUD_FAILED}), + Result({'ifindex': 1, 'state': NUD_FAILED, 'family': AF_INET}), ), ), ids=[ diff --git a/tests/utils.py b/tests/utils.py index 77e2ac0f0..ca67a7ff2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,6 +5,7 @@ import re import stat import subprocess +import sys import uuid from socket import AF_INET, AF_INET6 @@ -104,6 +105,11 @@ def require_kernel(major, minor=None): pytest.skip('incompatible kernel version') +def require_python(target): + if sys.version_info[0] != target: + pytest.skip('test requires Python %i' % target) + + def require_8021q(): try: os.stat('/proc/net/vlan/config') diff --git a/util/make_lab_templates.py b/util/make_lab_templates.py new file mode 100755 index 000000000..bc096e622 --- /dev/null +++ b/util/make_lab_templates.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import pathlib +import sys + +from docutils.core import publish_parts +from jinja2 import Environment, FileSystemLoader + +env = Environment(loader=FileSystemLoader('lab/_templates/')) +# js template +template = env.get_template('conf.js') +with open('lab/_static/conf.js', 'w') as f: + f.write(template.render(distfile=sys.argv[1])) + print('created lab/_static/conf.js') + +# html template +template = env.get_template('form_template.html') +root = pathlib.Path('examples/lab') +for example in root.iterdir(): + if not example.is_dir(): + continue + readme = publish_parts( + example.joinpath('README.rst').read_text(), writer_name='html' + )['html_body'] + setup = example.joinpath('setup.py').read_text() + task = example.joinpath('task.py').read_text() + check = '' + with example.joinpath('check.py').open('r') as f: + for line in f.readlines(): + if 'import' not in line: + check += line + name = example.name + with open(f'lab/{name}.html', 'w') as f: + f.write( + template.render( + readme=readme, setup=setup, task=task, check=check, name=name + ) + ) + print(f'created lab/{name}.html')