diff --git a/Cargo.lock b/Cargo.lock index c6d327a4..c887384b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,368 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "add-syntax" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6415d4d71daf492fee3ffe63ab90322db5ebf7122d7bd46d09dbcd336fae34" +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +name = "add-syntax" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6415d4d71daf492fee3ffe63ab90322db5ebf7122d7bd46d09dbcd336fae34" + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "bitflags", + "clap_lex", + "indexmap", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + [[package]] name = "malloc_size_of" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d719de8b8f230028cf8192ae4c1b25267cd6b8a99d2747d345a70b8c81aa13" +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -32,6 +382,70 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "serde" version = "1.0.229" @@ -39,6 +453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", + "serde_derive", ] [[package]] @@ -58,7 +473,20 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", ] [[package]] @@ -70,16 +498,35 @@ dependencies = [ "serde", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "2.0.0-alpha.12" dependencies = [ + "add-syntax", "bytes", + "criterion", "malloc_size_of", "serde_core", "serde_test", ] +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "3.0.3" @@ -91,8 +538,161 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 19391ffb..acfb20be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,9 +24,16 @@ internals = [] bytes = { version = "1", optional = true, default-features = false } serde_core = { version = "1.0.221", optional = true, default-features = false } malloc_size_of = { version = "0.1.1", optional = true, default-features = false } +add-syntax = "0.1.0" [dev-dependencies] serde_test = "1.0" +criterion = "0.4.0" + +[[bench]] +name = "bench" +path = "benches/bench.rs" +harness = false [package.metadata.docs.rs] all-features = true diff --git a/benches/bench.rs b/benches/bench.rs index 23864009..e881130a 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -1,10 +1,9 @@ -#![feature(test)] #![allow(deprecated)] -extern crate test; - +use criterion::{criterion_group, criterion_main, Bencher, Criterion}; use smallvec::{smallvec, SmallVec}; -use test::Bencher; +use std::hint::black_box; +use std::time::Duration; const VEC_SIZE: usize = 16; const SPILLED_SIZE: usize = 100; @@ -18,87 +17,91 @@ trait Vector: for<'a> From<&'a [T]> + Extend { fn from_elem(val: T, n: usize) -> Self; fn from_elems(val: &[T]) -> Self; fn extend_from_slice(&mut self, other: &[T]); + fn retain_mut(&mut self, f: F) + where + F: FnMut(&mut T) -> bool; } impl Vector for Vec { fn new() -> Self { Self::with_capacity(VEC_SIZE) } - fn push(&mut self, val: T) { self.push(val) } - fn pop(&mut self) -> Option { self.pop() } - fn remove(&mut self, p: usize) -> T { self.remove(p) } - fn insert(&mut self, n: usize, val: T) { self.insert(n, val) } - fn from_elem(val: T, n: usize) -> Self { vec![val; n] } - fn from_elems(val: &[T]) -> Self { val.to_owned() } - fn extend_from_slice(&mut self, other: &[T]) { Vec::extend_from_slice(self, other) } + fn retain_mut(&mut self, f: F) + where + F: FnMut(&mut T) -> bool, + { + self.retain_mut(f) + } } impl Vector for SmallVec { fn new() -> Self { Self::new() } - fn push(&mut self, val: T) { self.push(val) } - fn pop(&mut self) -> Option { self.pop() } - fn remove(&mut self, p: usize) -> T { self.remove(p) } - fn insert(&mut self, n: usize, val: T) { self.insert(n, val) } - fn from_elem(val: T, n: usize) -> Self { smallvec![val; n] } - fn from_elems(val: &[T]) -> Self { SmallVec::from(val) } - fn extend_from_slice(&mut self, other: &[T]) { SmallVec::extend_from_slice(self, other) } + fn retain_mut(&mut self, f: F) + where + F: FnMut(&mut T) -> bool, + { + self.retain_mut(f) + } } macro_rules! make_benches { ($typ:ty { $($b_name:ident => $g_name:ident($($args:expr),*),)* }) => { $( - #[bench] - fn $b_name(b: &mut Bencher) { - $g_name::<$typ>($($args,)* b) + fn $b_name(c: &mut Criterion) { + c.bench_function(stringify!($b_name), |b: &mut Bencher| { + $g_name::<$typ>($($args,)* b) + }); } )* } } +/* ---------- Bench generation (same list, just using the new macro) + * ---------- */ make_benches! { SmallVec { bench_push => gen_push(SPILLED_SIZE as _), @@ -122,6 +125,12 @@ make_benches! { bench_macro_from_elem => gen_from_elem(SPILLED_SIZE as _), bench_macro_from_elem_small => gen_from_elem(VEC_SIZE as _), bench_pushpop => gen_pushpop(), + bench_retain_mut_half => gen_retain_mut_half(SPILLED_SIZE as _), + bench_retain_mut_half_small => gen_retain_mut_half(VEC_SIZE as _), + bench_retain_mut_all => gen_retain_mut_all(SPILLED_SIZE as _), + bench_retain_mut_all_small => gen_retain_mut_all(VEC_SIZE as _), + bench_retain_mut_none => gen_retain_mut_none(SPILLED_SIZE as _), + bench_retain_mut_none_small => gen_retain_mut_none(VEC_SIZE as _), } } @@ -148,156 +157,269 @@ make_benches! { bench_macro_from_elem_vec => gen_from_elem(SPILLED_SIZE as _), bench_macro_from_elem_vec_small => gen_from_elem(VEC_SIZE as _), bench_pushpop_vec => gen_pushpop(), + bench_retain_mut_vec_half => gen_retain_mut_half(SPILLED_SIZE as _), + bench_retain_mut_vec_half_small => gen_retain_mut_half(VEC_SIZE as _), + bench_retain_mut_vec_all => gen_retain_mut_all(SPILLED_SIZE as _), + bench_retain_mut_vec_all_small => gen_retain_mut_all(VEC_SIZE as _), + bench_retain_mut_vec_none => gen_retain_mut_none(SPILLED_SIZE as _), + bench_retain_mut_vec_none_small => gen_retain_mut_none(VEC_SIZE as _), } } fn gen_push>(n: u64, b: &mut Bencher) { #[inline(never)] fn push_noinline>(vec: &mut V, x: u64) { - vec.push(x); + vec.push(black_box(x)); } b.iter(|| { + let n = black_box(n); let mut vec = V::new(); for x in 0..n { push_noinline(&mut vec, x); } - vec + black_box(vec) }); } fn gen_insert_push>(n: u64, b: &mut Bencher) { #[inline(never)] fn insert_push_noinline>(vec: &mut V, x: u64) { - vec.insert(x as usize, x); + vec.insert(black_box(x) as usize, black_box(x)); } b.iter(|| { + let n = black_box(n); let mut vec = V::new(); for x in 0..n { insert_push_noinline(&mut vec, x); } - vec + black_box(vec) }); } fn gen_insert>(n: u64, b: &mut Bencher) { #[inline(never)] fn insert_noinline>(vec: &mut V, p: usize, x: u64) { - vec.insert(p, x) + vec.insert(black_box(p), black_box(x)) } - b.iter(|| { - let mut vec = V::new(); - // Always insert at position 0 so that we are subject to shifts of - // many different lengths. - vec.push(0); - for x in 0..n { - insert_noinline(&mut vec, 0, x); - } - vec - }); + b.iter_with_setup( + || { + let mut vec = V::new(); + vec.push(0); + vec + }, + |mut vec| { + let n = black_box(n); + for x in 0..n { + insert_noinline(&mut vec, 0, x); + } + vec + }, + ); } fn gen_remove>(n: usize, b: &mut Bencher) { #[inline(never)] fn remove_noinline>(vec: &mut V, p: usize) -> u64 { - vec.remove(p) + vec.remove(black_box(p)) } - b.iter(|| { - let mut vec = V::from_elem(0, n as _); - - for _ in 0..n { - remove_noinline(&mut vec, 0); - } - }); + b.iter_with_setup( + || V::from_elem(0, black_box(n)), + |mut vec| { + for _ in 0..n { + black_box(remove_noinline(&mut vec, 0)); + } + vec + }, + ); } fn gen_extend>(n: u64, b: &mut Bencher) { b.iter(|| { + let n = black_box(n); let mut vec = V::new(); vec.extend(0..n); - vec + black_box(vec) }); } fn gen_extend_filtered>(n: u64, b: &mut Bencher) { b.iter(|| { let mut vec = V::new(); - vec.extend((0..n).filter(|i| i % 2 == 0)); - vec + vec.extend((0..black_box(n)).filter(|i| black_box(*i) % 2 == 0)); + black_box(vec) }); } fn gen_from_iter>(n: u64, b: &mut Bencher) { - let v: Vec = (0..n).collect(); + let v: Vec = (0..black_box(n)).collect(); b.iter(|| { - let vec = V::from(&v); - vec + let vec = V::from(black_box(&v)); + black_box(vec) }); } fn gen_from_slice>(n: u64, b: &mut Bencher) { - let v: Vec = (0..n).collect(); + let v: Vec = (0..black_box(n)).collect(); b.iter(|| { - let vec = V::from_elems(&v); - vec + let vec = V::from_elems(black_box(&v)); + black_box(vec) }); } fn gen_extend_from_slice>(n: u64, b: &mut Bencher) { - let v: Vec = (0..n).collect(); + let v: Vec = (0..black_box(n)).collect(); b.iter(|| { let mut vec = V::new(); - vec.extend_from_slice(&v); - vec + vec.extend_from_slice(black_box(&v)); + black_box(vec) }); } fn gen_pushpop>(b: &mut Bencher) { #[inline(never)] fn pushpop_noinline>(vec: &mut V, x: u64) -> Option { - vec.push(x); + vec.push(black_box(x)); vec.pop() } b.iter(|| { let mut vec = V::new(); for x in 0..SPILLED_SIZE as _ { - pushpop_noinline(&mut vec, x); + black_box(pushpop_noinline(&mut vec, x)); } - vec + black_box(vec) }); } fn gen_from_elem>(n: usize, b: &mut Bencher) { b.iter(|| { - let vec = V::from_elem(42, n); - vec + let n = black_box(n); + let vec = V::from_elem(black_box(42), n); + black_box(vec) }); } -#[bench] -fn bench_macro_from_list(b: &mut Bencher) { - b.iter(|| { - let vec: SmallVec = smallvec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 24, 32, 36, 0x40, 0x80, - 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, - 0x80000, 0x100000, - ]; - vec +fn gen_retain_mut_half>(n: usize, b: &mut Bencher) { + b.iter_with_setup( + || V::from_elem(16, black_box(n)), + |mut vec| { + vec.retain_mut(|x| black_box(*x) % 2 == 0); + vec + }, + ); +} + +fn gen_retain_mut_all>(n: usize, b: &mut Bencher) { + b.iter_with_setup( + || V::from_elem(16, black_box(n)), + |mut vec| { + vec.retain_mut(|_| true); + vec + }, + ); +} + +fn gen_retain_mut_none>(n: usize, b: &mut Bencher) { + b.iter_with_setup( + || V::from_elem(16, black_box(n)), + |mut vec| { + vec.retain_mut(|_| false); + vec + }, + ); +} + +fn bench_macro_from_list(c: &mut Criterion) { + c.bench_function("bench_macro_from_list", |b| { + b.iter(|| { + let vec: SmallVec = smallvec![ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 24, 32, 36, 0x40, + 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, + 0x40000, 0x80000, 0x100000, + ]; + vec + }) }); } -#[bench] -fn bench_macro_from_list_vec(b: &mut Bencher) { - b.iter(|| { - let vec: Vec = vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 24, 32, 36, 0x40, 0x80, - 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, - 0x80000, 0x100000, - ]; - vec +fn bench_macro_from_list_vec(c: &mut Criterion) { + c.bench_function("bench_macro_from_list_vec", |b| { + b.iter(|| { + let vec: Vec = vec![ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 24, 32, 36, 0x40, + 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, + 0x40000, 0x80000, 0x100000, + ]; + vec + }) }); } + +criterion_group!( + name = benches; + config = Criterion::default() + .warm_up_time(Duration::from_millis(200)) + .measurement_time(Duration::from_millis(700)); + targets = + bench_push, + bench_push_small, + bench_insert_push, + bench_insert_push_small, + bench_insert, + bench_insert_small, + bench_remove, + bench_remove_small, + bench_extend, + bench_extend_small, + bench_extend_filtered, + bench_extend_filtered_small, + bench_from_iter, + bench_from_iter_small, + bench_from_slice, + bench_from_slice_small, + bench_extend_from_slice, + bench_extend_from_slice_small, + bench_macro_from_elem, + bench_macro_from_elem_small, + bench_pushpop, + bench_retain_mut_half, + bench_retain_mut_half_small, + bench_retain_mut_all, + bench_retain_mut_all_small, + bench_retain_mut_none, + bench_retain_mut_none_small, + bench_push_vec, + bench_push_vec_small, + bench_insert_push_vec, + bench_insert_push_vec_small, + bench_insert_vec, + bench_insert_vec_small, + bench_remove_vec, + bench_remove_vec_small, + bench_extend_vec, + bench_extend_vec_small, + bench_extend_vec_filtered, + bench_extend_vec_filtered_small, + bench_from_iter_vec, + bench_from_iter_vec_small, + bench_from_slice_vec, + bench_from_slice_vec_small, + bench_extend_from_slice_vec, + bench_extend_from_slice_vec_small, + bench_macro_from_elem_vec, + bench_macro_from_elem_vec_small, + bench_pushpop_vec, + bench_retain_mut_vec_half, + bench_retain_mut_vec_half_small, + bench_retain_mut_vec_all, + bench_retain_mut_vec_all_small, + bench_retain_mut_vec_none, + bench_retain_mut_vec_none_small, + bench_macro_from_list, + bench_macro_from_list_vec +); +criterion_main!(benches); diff --git a/rustfmt.toml b/rustfmt.toml index 5171db17..6f31944f 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,4 +1,9 @@ wrap_comments = true -imports_granularity = "Preserve" +imports_granularity = "One" group_imports = "One" -format_code_in_doc_comments = true \ No newline at end of file +format_code_in_doc_comments = true +error_on_line_overflow = true +error_on_unformatted = true +blank_lines_lower_bound = 0 +blank_lines_upper_bound = 1 +fn_single_line = true \ No newline at end of file diff --git a/src/bytes.rs b/src/bytes.rs new file mode 100644 index 00000000..c89c5754 --- /dev/null +++ b/src/bytes.rs @@ -0,0 +1,59 @@ +use { + super::SmallVec, + bytes::{buf::UninitSlice, BufMut}, +}; + +unsafe impl BufMut for SmallVec { + fn remaining_mut(&self) -> usize { + // A vector can never have more than isize::MAX bytes + isize::MAX as usize - self.len() + } + unsafe fn advance_mut(&mut self, cnt: usize) { + let len = self.len(); + let remaining = self.capacity() - len; + + if remaining < cnt { + panic!("advance out of bounds: the len is {remaining} but advancing by {cnt}"); + } + + // Addition will not overflow since the sum is at most the capacity. + self.set_len(len + cnt); + } + fn chunk_mut(&mut self) -> &mut UninitSlice { + if self.capacity() == self.len() { + self.reserve(64); // Grow the smallvec + } + + let cap = self.capacity(); + let len = self.len(); + + let ptr = self.as_mut_ptr(); + // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be + // valid for `cap - len` bytes. The subtraction will not underflow since + // `len <= cap`. + unsafe { UninitSlice::from_raw_parts_mut(ptr.add(len), cap - len) } + } + + // Specialize these methods so they can skip checking `remaining_mut` + // and `advance_mut`. + fn put(&mut self, mut src: T) + where + Self: Sized, + { + // In case the src isn't contiguous, reserve upfront. + self.reserve(src.remaining()); + + while src.has_remaining() { + let s = src.chunk(); + let l = s.len(); + self.extend_from_slice(s); + src.advance(l); + } + } + fn put_slice(&mut self, src: &[u8]) { self.extend_from_slice(src); } + fn put_bytes(&mut self, val: u8, cnt: usize) { + // If the addition overflows, then the `resize` will fail. + let new_len = self.len().saturating_add(cnt); + self.resize(new_len, val); + } +} diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 00000000..fc78fc48 --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,22 @@ +use core::{ + alloc::Layout, + error::Error, + fmt::{Display, Formatter, Result as FormatResult}, +}; + +/// Error type for APIs with fallible heap allocation +#[derive(Debug)] +pub enum CollectionAllocErr { + /// Overflow `usize::MAX` or other error during size computation + CapacityOverflow, + /// The allocator return an error + AllocErr { + /// The layout that was passed to the allocator + layout: Layout, + }, +} +impl Display for CollectionAllocErr { + fn fmt(&self, f: &mut Formatter) -> FormatResult { write!(f, "Allocation error: {:?}", self) } +} + +impl Error for CollectionAllocErr {} diff --git a/src/lib.rs b/src/lib.rs index 3995c8ed..43c9ae50 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,6 @@ //! When this feature is enabled, traits available from `std` are implemented: //! //! * `SmallVec` implements the [`std::io::Write`] trait. -//! * [`CollectionAllocErr`] implements [`std::error::Error`]. //! //! This feature is not compatible with `#![no_std]` programs. //! @@ -65,60 +64,40 @@ pub extern crate alloc; #[cfg(any(test, feature = "std"))] extern crate std; +#[cfg(feature = "bytes")] +mod bytes; +mod errors; +mod macros; +#[cfg(feature = "malloc_size_of")] +mod mallocsizeof; mod rawsmallvec; +mod references; +#[cfg(feature = "serde")] +mod serde; +mod taggedlen; #[cfg(test)] mod tests; -use alloc::alloc::Layout; -use alloc::boxed::Box; -use alloc::vec; -use alloc::vec::Vec; -#[cfg(feature = "bytes")] -use bytes::{buf::UninitSlice, BufMut}; -use core::borrow::Borrow; -use core::borrow::BorrowMut; -use core::fmt::Debug; -use core::hash::{Hash, Hasher}; -use core::marker::PhantomData; -use core::mem::align_of; -use core::mem::size_of; -use core::mem::ManuallyDrop; -use core::mem::MaybeUninit; -use core::ptr::copy; -use core::ptr::copy_nonoverlapping; -use core::ptr::NonNull; -#[cfg(feature = "malloc_size_of")] -use malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}; +pub use errors::CollectionAllocErr; #[cfg(feature = "internals")] pub use rawsmallvec::RawSmallVec; #[cfg(not(feature = "internals"))] use rawsmallvec::RawSmallVec; -#[cfg(feature = "serde")] -use serde_core::{ - de::{Deserialize, Deserializer, SeqAccess, Visitor}, - ser::{Serialize, SerializeSeq, Serializer}, -}; #[cfg(feature = "std")] -use std::io; - -/// Error type for APIs with fallible heap allocation -#[derive(Debug)] -pub enum CollectionAllocErr { - /// Overflow `usize::MAX` or other error during size computation - CapacityOverflow, - /// The allocator return an error - AllocErr { - /// The layout that was passed to the allocator - layout: Layout, +use std::io::{Result as WriteResult, Write}; +use { + add_syntax::prepend, + alloc::{alloc::Layout, boxed::Box, vec::Vec}, + core::{ + fmt::Debug, + hash::{Hash, Hasher}, + iter::repeat_n, + marker::PhantomData, + mem::{align_of, size_of, ManuallyDrop, MaybeUninit}, + ptr::{copy, copy_nonoverlapping, NonNull}, }, -} -impl core::fmt::Display for CollectionAllocErr { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "Allocation error: {:?}", self) - } -} - -impl core::error::Error for CollectionAllocErr {} + taggedlen::TaggedLen, +}; #[inline] fn infallible(result: Result) -> T { @@ -129,12 +108,6 @@ fn infallible(result: Result) -> T { } } -/// Helper function to check if a type is a ZST. -#[inline] -const fn is_zst() -> bool { - const { size_of::() == 0 } -} - #[inline] /// A local copy of [`core::slice::range`]. The latter function is unstable /// and thus cannot be used yet. @@ -170,173 +143,6 @@ where core::ops::Range { start, end } } -impl RawSmallVec { - const IS_ZST: bool = is_zst::(); - - #[inline] - const fn new() -> Self { - Self::new_inline(MaybeUninit::uninit()) - } - #[inline] - const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { - Self { - inline: ManuallyDrop::new(inline), - } - } - #[inline] - const fn new_heap(ptr: NonNull, capacity: usize) -> Self { - Self { - heap: (ptr, capacity), - } - } - - #[inline] - const fn as_ptr_inline(&self) -> *const T { - // SAFETY: it is safe because we aren't reading the value, just getting a - // reference to it. reading it would be UB potentially, but for that downstream - // unsafe is required - (unsafe { &raw const self.inline }) as *mut T - } - - #[inline] - const fn as_mut_ptr_inline(&mut self) -> *mut T { - // SAFETY: same as above - (unsafe { &raw mut self.inline }) as *mut T - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_ptr_heap(&self) -> *const T { - self.heap.0.as_ptr() - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { - self.heap.0.as_ptr() - } - - /// # Safety - /// - /// `new_capacity` must be non zero, and greater or equal to the length. - /// T must not be a ZST. - unsafe fn try_grow_raw( - &mut self, - len: TaggedLen, - new_capacity: usize, - ) -> Result<(), CollectionAllocErr> { - use alloc::alloc::{alloc, realloc}; - debug_assert!(!Self::IS_ZST); - debug_assert!(new_capacity > 0); - debug_assert!(new_capacity >= len.value()); - - let was_on_heap = len.on_heap(); - let ptr = if was_on_heap { - self.as_mut_ptr_heap() - } else { - self.as_mut_ptr_inline() - }; - let len = len.value(); - - let new_layout = - Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; - if new_layout.size() > isize::MAX as usize { - return Err(CollectionAllocErr::CapacityOverflow); - } - - let new_ptr = if !was_on_heap { - // get a fresh allocation - let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. - let new_ptr = - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; - copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); - new_ptr - } else { - // use realloc - - // this can't overflow since we already constructed an equivalent layout during - // the previous allocation - let old_layout = - Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); - - // SAFETY: ptr was allocated with this allocator - // old_layout is the same as the layout used to allocate the previous memory - // block new_layout.size() is greater than zero - // does not overflow when rounded up to alignment. since it was constructed - // with Layout::array - let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? - }; - *self = Self::new_heap(new_ptr, new_capacity); - Ok(()) - } -} - -/// Vec guarantees that its length is always less than [`isize::MAX`] in -/// *bytes*. -/// -/// For a non ZST, this means that the length is less than `isize::MAX` objects, -/// which implies we have at least one free bit we can use. We use the least -/// significant bit for the tag. And store the length in the `usize::BITS - 1` -/// most significant bits. -/// -/// For a ZST, we never use the heap, so we just store the length directly. -#[repr(transparent)] -struct TaggedLen(usize, PhantomData); - -// Clone and Copy must be manually implemented because the generic interferes -// with the derive attribute implementations. -impl Clone for TaggedLen { - #[inline] - fn clone(&self) -> Self { - Self(self.0, PhantomData) - } - - #[inline] - fn clone_from(&mut self, source: &Self) { - self.0 = source.0; - } -} - -impl Copy for TaggedLen {} - -impl TaggedLen { - const IS_ZST: bool = is_zst::(); - #[inline] - pub const fn new(len: usize, on_heap: bool) -> Self { - if Self::IS_ZST { - debug_assert!(!on_heap); - Self(len, PhantomData) - } else { - debug_assert!(len < isize::MAX as usize); - Self((len << 1) | on_heap as usize, PhantomData) - } - } - - #[inline] - #[must_use] - pub const fn on_heap(self) -> bool { - if Self::IS_ZST { - false - } else { - (self.0 & 1_usize) == 1 - } - } - - #[inline] - pub const fn value(self) -> usize { - if Self::IS_ZST { - self.0 - } else { - self.0 >> 1 - } - } -} - #[repr(C)] pub struct SmallVec { len: TaggedLen, @@ -349,9 +155,7 @@ unsafe impl Sync for SmallVec {} impl Default for SmallVec { #[inline] - fn default() -> Self { - Self::new() - } + fn default() -> Self { Self::new() } } /// An iterator that removes the items from a `SmallVec` and yields them by @@ -387,9 +191,7 @@ impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { } #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } + fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { @@ -404,9 +206,7 @@ impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { impl ExactSizeIterator for Drain<'_, T, N> { #[inline] - fn len(&self) -> usize { - self.iter.len() - } + fn len(&self) -> usize { self.iter.len() } } impl core::iter::FusedIterator for Drain<'_, T, N> {} @@ -487,9 +287,7 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { impl Drain<'_, T, N> { #[must_use] - pub fn as_slice(&self) -> &[T] { - self.iter.as_slice() - } + pub fn as_slice(&self) -> &[T] { self.iter.as_slice() } /// The range from `self.vec.len` to `self.tail_start` contains elements /// that have been moved out. @@ -606,9 +404,7 @@ where } } - fn size_hint(&self) -> (usize, Option) { - (0, Some(self.end - self.idx)) - } + fn size_hint(&self) -> (usize, Option) { (0, Some(self.end - self.idx)) } } impl Drop for ExtractIf<'_, T, N, F> @@ -653,19 +449,13 @@ where impl Iterator for Splice<'_, I, N> { type Item = I::Item; - fn next(&mut self) -> Option { - self.drain.next() - } + fn next(&mut self) -> Option { self.drain.next() } - fn size_hint(&self) -> (usize, Option) { - self.drain.size_hint() - } + fn size_hint(&self) -> (usize, Option) { self.drain.size_hint() } } impl DoubleEndedIterator for Splice<'_, I, N> { - fn next_back(&mut self) -> Option { - self.drain.next_back() - } + fn next_back(&mut self) -> Option { self.drain.next_back() } } impl ExactSizeIterator for Splice<'_, I, N> {} @@ -913,8 +703,7 @@ impl SmallVec { /// # Examples /// /// ``` - /// use smallvec::SmallVec; - /// use std::mem::MaybeUninit; + /// use {smallvec::SmallVec, std::mem::MaybeUninit}; /// /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; /// let small_vec = unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) }; @@ -937,7 +726,7 @@ impl SmallVec { } impl SmallVec { - const IS_ZST: bool = is_zst::(); + const IS_ZST: bool = size_of::() == 0; #[inline] pub fn from_vec(vec: Vec) -> Self { @@ -983,9 +772,7 @@ impl SmallVec { /// /// The active union member must be the self.raw.heap #[inline] - unsafe fn set_on_heap(&mut self) { - self.len = TaggedLen::new(self.len(), true); - } + unsafe fn set_on_heap(&mut self) { self.len = TaggedLen::new(self.len(), true); } /// Sets the tag to be inline /// @@ -993,9 +780,7 @@ impl SmallVec { /// /// The active union member must be the self.raw.inline #[inline] - unsafe fn set_inline(&mut self) { - self.len = TaggedLen::new(self.len(), false); - } + unsafe fn set_inline(&mut self) { self.len = TaggedLen::new(self.len(), false); } /// Sets the length of a vector. /// @@ -1024,15 +809,11 @@ impl SmallVec { } #[inline] - pub const fn len(&self) -> usize { - self.len.value() - } + pub const fn len(&self) -> usize { self.len.value() } #[must_use] #[inline] - pub const fn is_empty(&self) -> bool { - self.len() == 0 - } + pub const fn is_empty(&self) -> bool { self.len() == 0 } #[inline] pub const fn capacity(&self) -> usize { @@ -1045,9 +826,7 @@ impl SmallVec { } #[inline] - pub const fn spilled(&self) -> bool { - self.len.on_heap() - } + pub const fn spilled(&self) -> bool { self.len.on_heap() } /// Splits the collection into two at the given index. /// @@ -1239,9 +1018,7 @@ impl SmallVec { } #[inline] - pub fn push(&mut self, value: T) { - _ = self.push_mut(value); - } + pub fn push(&mut self, value: T) { _ = self.push_mut(value); } #[inline] #[must_use] @@ -1277,18 +1054,17 @@ impl SmallVec { #[inline] pub fn pop(&mut self) -> Option { - if self.is_empty() { - None - } else { - let len = self.len() - 1; - // SAFETY: len < old_len since this can't overflow, because the old length is - // non zero - unsafe { self.set_len(len) }; - // SAFETY: this element was initialized and we just gave up ownership of it, so - // we can give it away - let value = unsafe { self.as_mut_ptr().add(len).read() }; - Some(value) + let len = self.len(); + if len == 0 { + return None; } + let new_len = len - 1; + // SAFETY: new_len < len since len is non-zero + unsafe { self.set_len(new_len) }; + // SAFETY: this element was initialized and we just gave up ownership of it, so + // we can give it away + let value = unsafe { self.as_mut_ptr().add(new_len).read() }; + Some(value) } #[inline] @@ -1322,9 +1098,7 @@ impl SmallVec { } #[inline] - pub fn grow(&mut self, new_capacity: usize) { - infallible(self.try_grow(new_capacity)); - } + pub fn grow(&mut self, new_capacity: usize) { infallible(self.try_grow(new_capacity)); } #[cold] pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), CollectionAllocErr> { @@ -1551,9 +1325,7 @@ impl SmallVec { } #[inline] - pub fn insert(&mut self, index: usize, value: T) { - _ = self.insert_mut(index, value); - } + pub fn insert(&mut self, index: usize, value: T) { _ = self.insert_mut(index, value); } #[inline] #[must_use] @@ -1664,9 +1436,7 @@ impl SmallVec { } #[inline] - pub fn into_boxed_slice(self) -> Box<[T]> { - self.into_vec().into_boxed_slice() - } + pub fn into_boxed_slice(self) -> Box<[T]> { self.into_vec().into_boxed_slice() } #[inline] pub fn into_inner(self) -> Result<[T; N], Self> { @@ -1686,27 +1456,32 @@ impl SmallVec { } #[inline] - pub fn retain bool>(&mut self, mut f: F) { - self.retain_mut(|elem| f(elem)) - } + pub fn retain bool>(&mut self, mut f: F) { self.retain_mut(|elem| f(elem)) } #[inline] pub fn retain_mut bool>(&mut self, mut f: F) { - let mut del = 0; let len = self.len(); + + if len == 0 { + // return early as hint to llvm, like what std does + return; + } + let ptr = self.as_mut_ptr(); - for i in 0..len { + let mut write_idx = 0; + + for read_idx in 0..len { // SAFETY: all the pointers are in bounds - // `i - del` never overflows since `del <= i` is a maintained invariant unsafe { - if !f(&mut *ptr.add(i)) { - del += 1; - } else if del > 0 { - core::ptr::swap(ptr.add(i), ptr.add(i - del)); + if f(&mut *ptr.add(read_idx)) { + if write_idx < read_idx { + core::ptr::swap(ptr.add(read_idx), ptr.add(write_idx)); + } + write_idx += 1; } } } - self.truncate(len - del); + self.truncate(write_idx); } #[inline] @@ -1895,9 +1670,7 @@ impl SmallVec { } #[inline] - pub fn extend_from_slice(&mut self, other: &[T]) { - self.extend(other.iter()) - } + pub fn extend_from_slice(&mut self, other: &[T]) { self.extend(other.iter()) } pub fn extend_from_within(&mut self, src: R) where @@ -2037,8 +1810,8 @@ impl Drop for DropDealloc { } } -#[cfg(feature = "may_dangle")] -unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { +#[cfg_attr(feature = "may_dangle", prepend(unsafe))] +impl<#[cfg_attr(feature = "may_dangle", may_dangle)] T, const N: usize> Drop for SmallVec { fn drop(&mut self) { let on_heap = self.spilled(); let len = self.len(); @@ -2061,29 +1834,6 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { } } -#[cfg(not(feature = "may_dangle"))] -impl Drop for SmallVec { - fn drop(&mut self) { - let on_heap = self.spilled(); - let len = self.len(); - let ptr = self.as_mut_ptr(); - // SAFETY: see above - unsafe { - let _drop_dealloc = if on_heap { - let capacity = self.capacity(); - Some(DropDealloc { - ptr: NonNull::new_unchecked(ptr as *mut u8), - size_bytes: capacity * size_of::(), - align: align_of::(), - }) - } else { - None - }; - core::ptr::slice_from_raw_parts_mut(ptr, len).drop_in_place(); - } - } -} - impl Drop for IntoIter { fn drop(&mut self) { // SAFETY: see above @@ -2107,21 +1857,6 @@ impl Drop for IntoIter { } } -impl core::ops::Deref for SmallVec { - type Target = [T]; - - #[inline] - fn deref(&self) -> &Self::Target { - self.as_slice() - } -} -impl core::ops::DerefMut for SmallVec { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - self.as_mut_slice() - } -} - /// This function is used in the [`smallvec`] macro. /// It is recommended to use the macro instead of using thís function. #[doc(hidden)] @@ -2129,7 +1864,7 @@ impl core::ops::DerefMut for SmallVec { pub fn from_elem(elem: T, n: usize) -> SmallVec { if n > SmallVec::::inline_size() { // Standard Rust vectors are already specialized. - SmallVec::::from_vec(vec![elem; n]) + SmallVec::from_iter(repeat_n(elem, n)) } else { #[cfg(feature = "specialization")] { @@ -2209,9 +1944,7 @@ mod spec_traits { I: Iterator, { #[inline] - default fn spec_extend(&mut self, iter: I) { - self.extend_fallback(iter); - } + default fn spec_extend(&mut self, iter: I) { self.extend_fallback(iter); } } impl SpecExtend for SmallVec @@ -2277,9 +2010,7 @@ mod spec_traits { T: Clone, { #[inline] - default fn spec_extend(&mut self, iterator: I) { - self.spec_extend(iterator.cloned()) - } + default fn spec_extend(&mut self, iterator: I) { self.spec_extend(iterator.cloned()) } } impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec @@ -2369,9 +2100,7 @@ mod spec_traits { I: Iterator, { #[inline] - default fn spec_from_iter(iter: I) -> Self { - Self::from_iter_fallback(iter) - } + default fn spec_from_iter(iter: I) -> Self { Self::from_iter_fallback(iter) } } impl SpecFromIterator for SmallVec @@ -2402,9 +2131,7 @@ mod spec_traits { impl SpecCloneFrom for SmallVec { #[inline] - default fn spec_clone_from(&mut self, source: &[T]) { - self.clone_from_fallback(source); - } + default fn spec_clone_from(&mut self, source: &[T]) { self.clone_from_fallback(source); } } impl SpecCloneFrom for SmallVec { @@ -2643,23 +2370,17 @@ impl From<&[T]> for SmallVec { impl From<&mut [T]> for SmallVec { #[inline] - fn from(slice: &mut [T]) -> Self { - Self::from(slice as &[T]) - } + fn from(slice: &mut [T]) -> Self { Self::from(slice as &[T]) } } impl From<&[T; M]> for SmallVec { #[inline] - fn from(slice: &[T; M]) -> Self { - Self::from(slice as &[T]) - } + fn from(slice: &[T; M]) -> Self { Self::from(slice as &[T]) } } impl From<&mut [T; M]> for SmallVec { #[inline] - fn from(slice: &mut [T; M]) -> Self { - Self::from(slice as &[T]) - } + fn from(slice: &mut [T; M]) -> Self { Self::from(slice as &[T]) } } impl From<[T; M]> for SmallVec { @@ -2684,16 +2405,12 @@ impl From<[T; M]> for SmallVec { } impl From> for SmallVec { - fn from(array: Vec) -> Self { - Self::from_vec(array) - } + fn from(array: Vec) -> Self { Self::from_vec(array) } } impl Clone for SmallVec { #[inline] - fn clone(&self) -> SmallVec { - SmallVec::from(self.as_slice()) - } + fn clone(&self) -> SmallVec { SmallVec::from(self.as_slice()) } #[inline] fn clone_from(&mut self, source: &Self) { @@ -2711,9 +2428,7 @@ impl Clone for SmallVec { impl Clone for IntoIter { #[inline] - fn clone(&self) -> IntoIter { - SmallVec::from(self.as_slice()).into_iter() - } + fn clone(&self) -> IntoIter { SmallVec::from(self.as_slice()).into_iter() } } impl Extend for SmallVec { @@ -2761,28 +2476,6 @@ impl core::iter::FromIterator for SmallVec { } } -#[deprecated] -#[macro_export] -macro_rules! smallvec { - ($elem:expr; $n:expr) => ({ - $crate::from_elem($elem, $n) - }); - ($($($x:expr),+$(,)?)?) => ({ - $crate::SmallVec::from([$($($x),+)?]) - }); -} - -#[deprecated] -#[macro_export] -macro_rules! smallvec_inline { - ($elem:expr; $n:expr) => ({ - $crate::SmallVec::<_, $n>::from_buf([$elem; $n]) - }); - ($($($x:expr),+$(,)?)?) => ({ - $crate::SmallVec::from_buf([$($($x),+)?]) - }); -} - impl IntoIterator for SmallVec { type IntoIter = IntoIter; type Item = T; @@ -2805,17 +2498,13 @@ impl IntoIterator for SmallVec { impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { type IntoIter = core::slice::Iter<'a, T>; type Item = &'a T; - fn into_iter(self) -> Self::IntoIter { - self.iter() - } + fn into_iter(self) -> Self::IntoIter { self.iter() } } impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { type IntoIter = core::slice::IterMut<'a, T>; type Item = &'a mut T; - fn into_iter(self) -> Self::IntoIter { - self.iter_mut() - } + fn into_iter(self) -> Self::IntoIter { self.iter_mut() } } impl PartialEq> for SmallVec @@ -2823,9 +2512,7 @@ where T: PartialEq, { #[inline] - fn eq(&self, other: &SmallVec) -> bool { - self.as_slice().eq(other.as_slice()) - } + fn eq(&self, other: &SmallVec) -> bool { self.as_slice().eq(other.as_slice()) } } impl Eq for SmallVec where T: Eq {} @@ -2834,9 +2521,7 @@ where T: PartialEq, { #[inline] - fn eq(&self, other: &[U; M]) -> bool { - self[..] == other[..] - } + fn eq(&self, other: &[U; M]) -> bool { self[..] == other[..] } } impl PartialEq<&[U; M]> for SmallVec @@ -2844,9 +2529,7 @@ where T: PartialEq, { #[inline] - fn eq(&self, other: &&[U; M]) -> bool { - self[..] == other[..] - } + fn eq(&self, other: &&[U; M]) -> bool { self[..] == other[..] } } impl PartialEq<[U]> for SmallVec @@ -2854,9 +2537,7 @@ where T: PartialEq, { #[inline] - fn eq(&self, other: &[U]) -> bool { - self[..] == other[..] - } + fn eq(&self, other: &[U]) -> bool { self[..] == other[..] } } impl PartialEq<&[U]> for SmallVec @@ -2864,9 +2545,7 @@ where T: PartialEq, { #[inline] - fn eq(&self, other: &&[U]) -> bool { - self[..] == other[..] - } + fn eq(&self, other: &&[U]) -> bool { self[..] == other[..] } } impl PartialEq<&mut [U]> for SmallVec @@ -2874,9 +2553,7 @@ where T: PartialEq, { #[inline] - fn eq(&self, other: &&mut [U]) -> bool { - self[..] == other[..] - } + fn eq(&self, other: &&mut [U]) -> bool { self[..] == other[..] } } impl PartialOrd for SmallVec @@ -2900,37 +2577,7 @@ where } impl Hash for SmallVec { - fn hash(&self, state: &mut H) { - self.as_slice().hash(state) - } -} - -impl Borrow<[T]> for SmallVec { - #[inline] - fn borrow(&self) -> &[T] { - self.as_slice() - } -} - -impl BorrowMut<[T]> for SmallVec { - #[inline] - fn borrow_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } -} - -impl AsRef<[T]> for SmallVec { - #[inline] - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} - -impl AsMut<[T]> for SmallVec { - #[inline] - fn as_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } + fn hash(&self, state: &mut H) { self.as_slice().hash(state) } } impl Debug for SmallVec { @@ -2951,174 +2598,20 @@ impl Debug for Drain<'_, T, N> { } } -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl Serialize for SmallVec -where - T: Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - let mut state = serializer.serialize_seq(Some(self.len()))?; - for item in self { - state.serialize_element(item)?; - } - state.end() - } -} - -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl<'de, T, const N: usize> Deserialize<'de> for SmallVec -where - T: Deserialize<'de>, -{ - fn deserialize>(deserializer: D) -> Result { - deserializer.deserialize_seq(SmallVecVisitor { - phantom: PhantomData, - }) - } -} - -#[cfg(feature = "serde")] -struct SmallVecVisitor { - phantom: PhantomData, -} - -#[cfg(feature = "serde")] -impl<'de, T, const N: usize> Visitor<'de> for SmallVecVisitor -where - T: Deserialize<'de>, -{ - type Value = SmallVec; - - fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - formatter.write_str("a sequence") - } - - fn visit_seq(self, mut seq: B) -> Result - where - B: SeqAccess<'de>, - { - use serde_core::de::Error; - let len = seq.size_hint().unwrap_or(0); - let mut values = SmallVec::new(); - values.try_reserve(len).map_err(B::Error::custom)?; - - while let Some(value) = seq.next_element()? { - values.push(value); - } - - Ok(values) - } -} - -#[cfg(feature = "malloc_size_of")] -impl MallocShallowSizeOf for SmallVec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if self.spilled() { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } else { - 0 - } - } -} - -#[cfg(feature = "malloc_size_of")] -impl MallocSizeOf for SmallVec { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); - for elem in self.iter() { - n += elem.size_of(ops); - } - n - } -} - #[cfg(feature = "std")] -#[cfg_attr(docsrs, doc(cfg(feature = "std")))] -impl io::Write for SmallVec { +impl Write for SmallVec { #[inline] - fn write(&mut self, buf: &[u8]) -> io::Result { + fn write(&mut self, buf: &[u8]) -> WriteResult { self.extend_from_slice(buf); Ok(buf.len()) } #[inline] - fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + fn write_all(&mut self, buf: &[u8]) -> WriteResult<()> { self.extend_from_slice(buf); Ok(()) } #[inline] - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -#[cfg(feature = "bytes")] -unsafe impl BufMut for SmallVec { - #[inline] - fn remaining_mut(&self) -> usize { - // A vector can never have more than isize::MAX bytes - isize::MAX as usize - self.len() - } - - #[inline] - unsafe fn advance_mut(&mut self, cnt: usize) { - let len = self.len(); - let remaining = self.capacity() - len; - - if remaining < cnt { - panic!("advance out of bounds: the len is {remaining} but advancing by {cnt}"); - } - - // Addition will not overflow since the sum is at most the capacity. - self.set_len(len + cnt); - } - - #[inline] - fn chunk_mut(&mut self) -> &mut UninitSlice { - if self.capacity() == self.len() { - self.reserve(64); // Grow the smallvec - } - - let cap = self.capacity(); - let len = self.len(); - - let ptr = self.as_mut_ptr(); - // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be - // valid for `cap - len` bytes. The subtraction will not underflow since - // `len <= cap`. - unsafe { UninitSlice::from_raw_parts_mut(ptr.add(len), cap - len) } - } - - // Specialize these methods so they can skip checking `remaining_mut` - // and `advance_mut`. - #[inline] - fn put(&mut self, mut src: T) - where - Self: Sized, - { - // In case the src isn't contiguous, reserve upfront. - self.reserve(src.remaining()); - - while src.has_remaining() { - let s = src.chunk(); - let l = s.len(); - self.extend_from_slice(s); - src.advance(l); - } - } - - #[inline] - fn put_slice(&mut self, src: &[u8]) { - self.extend_from_slice(src); - } - - #[inline] - fn put_bytes(&mut self, val: u8, cnt: usize) { - // If the addition overflows, then the `resize` will fail. - let new_len = self.len().saturating_add(cnt); - self.resize(new_len, val); - } + fn flush(&mut self) -> WriteResult<()> { Ok(()) } } diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 00000000..a3b718da --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,21 @@ +#[deprecated] +#[macro_export] +macro_rules! smallvec { + ($elem:expr; $n:expr) => ({ + $crate::from_elem($elem, $n) + }); + ($($($x:expr),+$(,)?)?) => ({ + $crate::SmallVec::from([$($($x),+)?]) + }); +} + +#[deprecated] +#[macro_export] +macro_rules! smallvec_inline { + ($elem:expr; $n:expr) => ({ + $crate::SmallVec::<_, $n>::from_buf([$elem; $n]) + }); + ($($($x:expr),+$(,)?)?) => ({ + $crate::SmallVec::from_buf([$($($x),+)?]) + }); +} diff --git a/src/mallocsizeof.rs b/src/mallocsizeof.rs new file mode 100644 index 00000000..6377b635 --- /dev/null +++ b/src/mallocsizeof.rs @@ -0,0 +1,24 @@ +use { + super::SmallVec, + malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}, +}; + +impl MallocShallowSizeOf for SmallVec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if self.spilled() { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } else { + 0 + } + } +} + +impl MallocSizeOf for SmallVec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index dadbf958..b4aafc4e 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -1,5 +1,11 @@ -use core::mem::{ManuallyDrop, MaybeUninit}; -use core::ptr::NonNull; +use { + super::{errors::CollectionAllocErr, taggedlen::TaggedLen}, + core::{ + alloc::Layout, + mem::{ManuallyDrop, MaybeUninit}, + ptr::{copy_nonoverlapping, NonNull}, + }, +}; /// Either a stack array with `length <= N` or a heap array /// whose pointer and capacity are stored here. @@ -9,5 +15,189 @@ use core::ptr::NonNull; #[repr(C)] pub union RawSmallVec { pub inline: ManuallyDrop>, - pub heap: (NonNull, usize), + pub heap: (NonNull, usize) +} + +impl RawSmallVec { + const IS_ZST: bool = size_of::() == 0; + pub const fn new() -> Self { Self::new_inline(MaybeUninit::uninit()) } + pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + Self { + inline: ManuallyDrop::new(inline), + } + } + pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + Self { + heap: (ptr, capacity), + } + } + #[allow(unused_unsafe, reason = "requires unsafe in MSRV 1.83.0")] + pub const fn as_ptr_inline(&self) -> *const T { + // SAFETY: it is safe because we aren't reading the value, just getting a + // reference to it. reading it would be UB potentially, but for that downstream + // unsafe is required + (unsafe { &raw const self.inline }) as *mut T + } + #[allow(unused_unsafe, reason = "requires unsafe in MSRV 1.83.0")] + pub const fn as_mut_ptr_inline(&mut self) -> *mut T { + // SAFETY: same as above + (unsafe { &raw mut self.inline }) as *mut T + } + + /// # Safety + /// + /// The vector must be on the heap + pub const unsafe fn as_ptr_heap(&self) -> *const T { self.heap.0.as_ptr() } + + /// # Safety + /// + /// The vector must be on the heap + pub const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { self.heap.0.as_ptr() } + + /// # Safety + /// + /// `new_capacity` must be non zero, and greater or equal to the length. + /// T must not be a ZST. + pub unsafe fn try_grow_raw( + &mut self, + len: TaggedLen, + new_capacity: usize, + ) -> Result<(), CollectionAllocErr> { + use alloc::alloc::{alloc, realloc}; + debug_assert!(!Self::IS_ZST); + debug_assert!(new_capacity > 0); + debug_assert!(new_capacity >= len.value()); + + let was_on_heap = len.on_heap(); + let ptr = if was_on_heap { + self.as_mut_ptr_heap() + } else { + self.as_mut_ptr_inline() + }; + let len = len.value(); + + let new_layout = + Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; + if new_layout.size() > isize::MAX as usize { + return Err(CollectionAllocErr::CapacityOverflow); + } + + let new_ptr = if !was_on_heap { + // get a fresh allocation + let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. + let new_ptr = + NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; + copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); + new_ptr + } else { + // use realloc + + // this can't overflow since we already constructed an equivalent layout during + // the previous allocation + let old_layout = + Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); + + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to allocate the previous memory + // block new_layout.size() is greater than zero + // does not overflow when rounded up to alignment. since it was constructed + // with Layout::array + let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; + NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? + }; + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } +} + +impl RawSmallVec { + const IS_ZST: bool = size_of::() == 0; + pub const fn new() -> Self { Self::new_inline(MaybeUninit::uninit()) } + pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + Self { + inline: ManuallyDrop::new(inline), + } + } + pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + Self { + heap: (ptr, capacity), + } + } + #[allow(unused_unsafe, reason = "requires unsafe in MSRV 1.83.0")] + pub const fn as_ptr_inline(&self) -> *const T { + // SAFETY: it is safe because we aren't reading the value, just getting a + // reference to it. reading it would be UB potentially, but for that downstream + // unsafe is required + (unsafe { &raw const self.inline }) as *mut T + } + #[allow(unused_unsafe, reason = "requires unsafe in MSRV 1.83.0")] + pub const fn as_mut_ptr_inline(&mut self) -> *mut T { + // SAFETY: same as above + (unsafe { &raw mut self.inline }) as *mut T + } + + /// # Safety + /// + /// The vector must be on the heap + pub const unsafe fn as_ptr_heap(&self) -> *const T { self.heap.0.as_ptr() } + + /// # Safety + /// + /// The vector must be on the heap + pub const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { self.heap.0.as_ptr() } + + /// # Safety + /// + /// `new_capacity` must be non zero, and greater or equal to the length. + /// T must not be a ZST. + pub unsafe fn try_grow_raw( + &mut self, + len: TaggedLen, + new_capacity: usize, + ) -> Result<(), CollectionAllocErr> { + use alloc::alloc::{alloc, realloc}; + debug_assert!(!Self::IS_ZST); + debug_assert!(new_capacity > 0); + debug_assert!(new_capacity >= len.value()); + + let was_on_heap = len.on_heap(); + let ptr = if was_on_heap { + self.as_mut_ptr_heap() + } else { + self.as_mut_ptr_inline() + }; + let len = len.value(); + + let new_layout = + Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; + if new_layout.size() > isize::MAX as usize { + return Err(CollectionAllocErr::CapacityOverflow); + } + + let new_ptr = if !was_on_heap { + // get a fresh allocation + let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. + let new_ptr = + NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; + copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); + new_ptr + } else { + // use realloc + + // this can't overflow since we already constructed an equivalent layout during + // the previous allocation + let old_layout = + Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); + + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to allocate the previous memory + // block new_layout.size() is greater than zero + // does not overflow when rounded up to alignment. since it was constructed + // with Layout::array + let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; + NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? + }; + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } } diff --git a/src/references.rs b/src/references.rs new file mode 100644 index 00000000..3b157685 --- /dev/null +++ b/src/references.rs @@ -0,0 +1,38 @@ +use { + super::SmallVec, + core::{ + borrow::{Borrow, BorrowMut}, + ops::{Deref, DerefMut}, + }, +}; + +impl AsRef<[T]> for SmallVec { + #[inline] + fn as_ref(&self) -> &[T] { self.as_slice() } +} + +impl AsMut<[T]> for SmallVec { + #[inline] + fn as_mut(&mut self) -> &mut [T] { self.as_mut_slice() } +} + +impl Borrow<[T]> for SmallVec { + #[inline] + fn borrow(&self) -> &[T] { self.as_slice() } +} + +impl BorrowMut<[T]> for SmallVec { + #[inline] + fn borrow_mut(&mut self) -> &mut [T] { self.as_mut_slice() } +} + +impl Deref for SmallVec { + type Target = [T]; + + #[inline] + fn deref(&self) -> &Self::Target { self.as_slice() } +} +impl DerefMut for SmallVec { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { self.as_mut_slice() } +} diff --git a/src/serde.rs b/src/serde.rs new file mode 100644 index 00000000..0f7d7e8a --- /dev/null +++ b/src/serde.rs @@ -0,0 +1,64 @@ +use { + super::SmallVec, + core::marker::PhantomData, + serde_core::{ + de::{SeqAccess, Visitor}, + ser::SerializeSeq, + Deserialize, Deserializer, Serialize, Serializer, + }, +}; + +impl Serialize for SmallVec +where + T: Serialize, +{ + fn serialize(&self, serializer: S) -> Result { + let mut state = serializer.serialize_seq(Some(self.len()))?; + for item in self { + state.serialize_element(item)?; + } + state.end() + } +} + +impl<'de, T, const N: usize> Deserialize<'de> for SmallVec +where + T: Deserialize<'de>, +{ + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_seq(SmallVecVisitor { + phantom: PhantomData, + }) + } +} + +struct SmallVecVisitor { + phantom: PhantomData, +} + +impl<'de, T, const N: usize> Visitor<'de> for SmallVecVisitor +where + T: Deserialize<'de>, +{ + type Value = SmallVec; + + fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter.write_str("a sequence") + } + + fn visit_seq(self, mut seq: B) -> Result + where + B: SeqAccess<'de>, + { + use serde_core::de::Error; + let len = seq.size_hint().unwrap_or(0); + let mut values = SmallVec::new(); + values.try_reserve(len).map_err(B::Error::custom)?; + + while let Some(value) = seq.next_element()? { + values.push(value); + } + + Ok(values) + } +} diff --git a/src/taggedlen.rs b/src/taggedlen.rs new file mode 100644 index 00000000..e10748d7 --- /dev/null +++ b/src/taggedlen.rs @@ -0,0 +1,58 @@ +use core::marker::PhantomData; + +/// Vec guarantees that its length is always less than [`isize::MAX`] in +/// *bytes*. +/// +/// For a non ZST, this means that the length is less than `isize::MAX` objects, +/// which implies we have at least one free bit we can use. We use the least +/// significant bit for the tag. And store the length in the `usize::BITS - 1` +/// most significant bits. +/// +/// For a ZST, we never use the heap, so we just store the length directly. +#[repr(transparent)] +pub struct TaggedLen(usize, PhantomData); + +// Clone and Copy must be manually implemented because the generic interferes +// with the derive attribute implementations. +impl Clone for TaggedLen { + #[inline] + fn clone(&self) -> Self { Self(self.0, PhantomData) } + + #[inline] + fn clone_from(&mut self, source: &Self) { self.0 = source.0; } +} + +impl Copy for TaggedLen {} + +impl TaggedLen { + const IS_ZST: bool = size_of::() == 0; + #[inline] + pub const fn new(len: usize, on_heap: bool) -> Self { + if Self::IS_ZST { + debug_assert!(!on_heap); + Self(len, PhantomData) + } else { + debug_assert!(len < isize::MAX as usize); + Self((len << 1) | on_heap as usize, PhantomData) + } + } + + #[inline] + #[must_use] + pub const fn on_heap(self) -> bool { + if Self::IS_ZST { + false + } else { + (self.0 & 1_usize) == 1 + } + } + + #[inline] + pub const fn value(self) -> usize { + if Self::IS_ZST { + self.0 + } else { + self.0 >> 1 + } + } +} diff --git a/src/tests.rs b/src/tests.rs index d344de7f..22eaa854 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,10 +1,8 @@ -use crate::{smallvec, SmallVec}; -use alloc::borrow::ToOwned; -use alloc::boxed::Box; -use alloc::rc::Rc; -use alloc::{vec, vec::Vec}; -use core::hash::Hasher; -use core::iter::FromIterator; +use { + crate::SmallVec, + alloc::{borrow::ToOwned, boxed::Box, rc::Rc, vec::Vec}, + core::{hash::Hasher, iter::FromIterator}, +}; #[test] pub fn test_zero() { @@ -106,9 +104,7 @@ pub fn test_double_spill() { // https://github.com/servo/rust-smallvec/issues/4 #[test] -fn issue_4() { - SmallVec::, 2>::new(); -} +fn issue_4() { SmallVec::, 2>::new(); } // https://github.com/servo/rust-smallvec/issues/5 #[test] @@ -167,7 +163,7 @@ fn drain_rev() { #[test] fn drain_forget() { - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6, 7]); std::mem::forget(v.drain(2..5)); assert_eq!(v.len(), 2); } @@ -175,21 +171,21 @@ fn drain_forget() { #[test] fn splice() { // The range starts right before the end. - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6]); let new = [7, 8, 9, 10]; let u: SmallVec = v.splice(6.., new).collect(); assert_eq!(v, [0, 1, 2, 3, 4, 5, 7, 8, 9, 10]); assert_eq!(u, [6]); // The range is empty. - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6]); let new = [7, 8, 9, 10]; let u: SmallVec = v.splice(1..1, new).collect(); assert_eq!(v, [0, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6]); assert_eq!(u, [0u8; 0]); // The range is at the beginning and nonempty. - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6]); let new = [7, 8, 9, 10]; let u: SmallVec = v.splice(..3, new).collect(); assert_eq!(v, [7, 8, 9, 10, 3, 4, 5, 6]); @@ -231,9 +227,7 @@ fn into_iter_drop() { struct DropCounter<'a>(&'a Cell); impl<'a> Drop for DropCounter<'a> { - fn drop(&mut self) { - self.0.set(self.0.get() + 1); - } + fn drop(&mut self) { self.0.set(self.0.get() + 1); } } { @@ -317,7 +311,7 @@ fn test_truncate() { #[test] fn test_truncate_references() { - let mut v = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut v = Vec::from([0, 1, 2, 3, 4, 5, 6, 7]); let mut i = 8; let mut v: SmallVec<&mut u8, 8> = v.iter_mut().collect(); @@ -338,7 +332,7 @@ fn test_truncate_references() { #[test] fn test_split_off() { - let mut vec: SmallVec = smallvec![1, 2, 3, 4, 5, 6]; + let mut vec: SmallVec = SmallVec::from([1, 2, 3, 4, 5, 6]); let orig_ptr = vec.as_ptr(); let orig_capacity = vec.capacity(); @@ -400,7 +394,7 @@ fn test_invalid_grow() { #[test] #[should_panic] fn drain_overflow() { - let mut v: SmallVec = smallvec![0]; + let mut v: SmallVec = SmallVec::from([0]); v.drain(..=usize::MAX); } @@ -420,7 +414,7 @@ fn test_extend_from_slice() { #[test] fn test_extend_from_within() { - let mut v: SmallVec = smallvec![0, 1, 2, 3]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3]); v.extend_from_within(1..3); assert_eq!( &v.iter().map(|v| *v).collect::>(), @@ -486,8 +480,7 @@ fn test_ord() { #[test] fn test_hash() { - use std::collections::hash_map::DefaultHasher; - use std::hash::Hash; + use std::{collections::hash_map::DefaultHasher, hash::Hash}; fn hash(value: impl Hash) -> u64 { let mut hasher = DefaultHasher::new(); @@ -567,17 +560,17 @@ fn test_from() { assert_eq!(&SmallVec::::from(&[1][..])[..], [1]); assert_eq!(&SmallVec::::from(&[1, 2, 3][..])[..], [1, 2, 3]); - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); @@ -589,7 +582,7 @@ fn test_from() { let array = [99; 128]; let small_vec: SmallVec = SmallVec::from(array); - assert_eq!(&*small_vec, vec![99u8; 128].as_slice()); + assert_eq!(&*small_vec, Vec::from([99u8; 128]).as_slice()); drop(small_vec); #[derive(PartialEq, Eq, Debug)] @@ -599,14 +592,14 @@ fn test_from() { assert_eq!(&*small_vec, &[NoClone(42)]); drop(small_vec); - let vec = vec![NoClone(42)]; + let vec = Vec::from([NoClone(42)]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[NoClone(42)]); drop(small_vec); let array = [1; 128]; let small_vec: SmallVec = SmallVec::from(array); - assert_eq!(&*small_vec, vec![1; 128].as_slice()); + assert_eq!(&*small_vec, Vec::from([1; 128]).as_slice()); drop(small_vec); let array = [99]; @@ -686,7 +679,7 @@ fn shrink_to_fit_unspill() { #[test] fn shrink_after_from_empty_vec() { - let mut v = SmallVec::::from_vec(vec![]); + let mut v = SmallVec::::from_vec(Vec::new()); v.shrink_to_fit(); assert!(!v.spilled()) } @@ -694,10 +687,10 @@ fn shrink_after_from_empty_vec() { #[test] fn test_into_vec() { let vec = SmallVec::::from_iter(0..2); - assert_eq!(vec.into_vec(), vec![0, 1]); + assert_eq!(vec.into_vec(), Vec::from([0, 1])); let vec = SmallVec::::from_iter(0..3); - assert_eq!(vec.into_vec(), vec![0, 1, 2]); + assert_eq!(vec.into_vec(), Vec::from([0, 1, 2])); } #[test] @@ -714,32 +707,32 @@ fn test_into_inner() { #[test] fn test_from_vec() { - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - let vec = vec![1]; + let vec = Vec::from([1]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1]); drop(small_vec); - let vec = vec![1, 2, 3]; + let vec = Vec::from([1, 2, 3]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); @@ -903,9 +896,7 @@ fn grow_spilled_same_size() { } #[test] -fn const_generics() { - let _v = SmallVec::::default(); -} +fn const_generics() { let _v = SmallVec::::default(); } #[test] fn const_new() { @@ -922,25 +913,15 @@ fn const_new() { assert_eq!(v[0], 1); assert_eq!(v[1], 4); } -const fn const_new_inner() -> SmallVec { - SmallVec::::new() -} -const fn const_new_inline_sized() -> SmallVec { - crate::smallvec_inline![1; 4] -} -const fn const_new_inline_args() -> SmallVec { - crate::smallvec_inline![1, 4] -} +const fn const_new_inner() -> SmallVec { SmallVec::::new() } +const fn const_new_inline_sized() -> SmallVec { SmallVec::from_buf([1; 4]) } +const fn const_new_inline_args() -> SmallVec { SmallVec::from_buf([1, 4]) } #[test] -fn empty_macro() { - let _v: SmallVec = smallvec![]; -} +fn empty_macro() { let _v: SmallVec = SmallVec::new(); } #[test] -fn zero_size_items() { - SmallVec::<(), 0>::new().push(()); -} +fn zero_size_items() { SmallVec::<(), 0>::new().push(()); } #[test] fn test_clone_from() { @@ -966,7 +947,7 @@ fn test_clone_from() { #[test] fn test_extract_if() { - let mut a: SmallVec = smallvec![0, 1u8, 2, 3, 4, 5, 6, 7, 8, 0]; + let mut a: SmallVec = SmallVec::from([0, 1u8, 2, 3, 4, 5, 6, 7, 8, 0]); let b: SmallVec = a.extract_if(1..9, |x| *x % 3 == 0).collect(); @@ -983,7 +964,7 @@ fn test_extract_if() { /// wrong" args. #[test] fn max_dont_panic() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); let _ = sv.get(usize::MAX); sv.truncate(usize::MAX); } @@ -991,21 +972,21 @@ fn max_dont_panic() { #[test] #[should_panic] fn max_remove() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); sv.remove(usize::MAX); } #[test] #[should_panic] fn max_swap_remove() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); sv.swap_remove(usize::MAX); } #[test] #[should_panic] fn max_insert() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); sv.insert(usize::MAX, 0); } @@ -1016,9 +997,7 @@ fn collect_from_iter() { impl Iterator for IterNoHint { type Item = I::Item; - fn next(&mut self) -> Option { - self.0.next() - } + fn next(&mut self) -> Option { self.0.next() } // no implementation of size_hint means it returns (0, None) - which forces // from_iter to grow the allocated space iteratively. @@ -1027,7 +1006,11 @@ fn collect_from_iter() { // A length of 3 is fine to trigger this bug under valgrind, but making the // vector 1 million elements makes it crash - which is much easier to // detect. - let iter = IterNoHint(std::iter::repeat(1u8).take(1_000_000)); + #[cfg(miri)] + const ELEMENTS: usize = 1000; + #[cfg(not(miri))] + const ELEMENTS: usize = 1_000_000; + let iter = IterNoHint(std::iter::repeat(1u8).take(ELEMENTS)); let _y: SmallVec = SmallVec::from_iter(iter); }