diff --git a/test/exactly/README.rst b/test/exactly/README.rst new file mode 100644 index 00000000..9f1c5a57 --- /dev/null +++ b/test/exactly/README.rst @@ -0,0 +1,153 @@ +=============================================================================== +Some tests of Criterion using Exactly +=============================================================================== + +An implementation of the tests in *Exactly*, corresponding to the tests +implemented in *Cram* in the official repository. + +- Criterion_ +- Exactly_ +- Cram_ + +Some C++ tests corresponding to C tests have been added. + + +Organisation +=============================================================================== + +Test root directory +------------------------------------------------------------------------------- + +``c.suite`` + Suite file for the tests of the C library. + +``cxx.suite`` + Suite file for the tests of the C++ library. + +``exactly.suite`` + Suite file for all tests - C and C++. + + +A sub suite corresponding to each Cram file (``.t``) +------------------------------------------------------------------------------- + +Each Cram test file (``.t``) is implemented as an Exactly sub suite, +located in its own subdirectory. + +Most sub suites share a common structure: + +``c`` + Directory representing a sub suite for the tests of the C library. + +``cxx`` + Directory representing a sub suite for the tests of the C++ library. + +``c.suite`` + Suite file for the tests of the C library. + +``cxx.suite`` + Suite file for the tests of the C++ library. + +``exactly.suite`` + Suite file for all tests - C and C++. + +A Cram test file contains multiple test cases, +and each such test case is represented by a single Exactly test case file +(``.case``). + + +Execution +=============================================================================== + + +Exactly basics +------------------------------------------------------------------------------- + +Executing a suite via its suite file ``NAME.suite``:: + + > exactly suite NAME.suite + +Executing a case via its case file ``NAME.case``:: + + > exactly NAME.case + +Executing the *action to check* of a test case file ``NAME.case``:: + + > exactly --act NAME.case + +This will ignore assertions and output exit-code, stdout and stderr +of the program run in ``[act]``. + + +Executing Exactly tests (suites and cases) directly +------------------------------------------------------------------------------- + +To run tests independently of the build system (for debugging, development), +the PATH needs to be extended with the paths of test binaries. + +To do this, follow the instructions in ``test/exactly/common.xly`` + + +Evaluation +=============================================================================== + + +Status +------------------------------------------------------------------------------- + +Only tested on Linux (Debian). + + +Pros +------------------------------------------------------------------------------- + +- More detailed tests, and test of more props: + - exit code + - stdout + - stderr +- Ability to run each test case independently. +- Test cases are named (via file names) - may help + understand the tests. +- Execpted output text can be put in separate files. +- Resue of resources + - Common output in external files that can be reused. + - Replacement of name-of-sourc-file in expected output + means that the same expected output can be used for + both C and C++. +- No dependency on a shell. +- Test setup is handled by the test tool. + + The Cram tests depends on setup (environment variables) + handled by the build tool (``meson``). + + +Cons +------------------------------------------------------------------------------- + +- Problem - replace refs (see ``asserts/c/fail-messages.case``). + + Cram's feature for checking text (mixing constants and reg-exps) + is nice and could be added Exactly. +- Challenging to organise and run via ``meson``. +- Exactly cannot check prerequesites. + + These has to be implemented via the build system (``meson``). +- Exactly is more complex - may take more time to learn. + + +New features of Exactly that would help here +------------------------------------------------------------------------------- + +- text-matcher for Cram's text matching - mixed constants, reg-exps, globs +- Automatic inclusion of package file (package.xly) (as planned) + + This would make the file structure simpler. +- Phase for prerequesites (as planned) +- Giving config (home, act-home) and symbol definitions via command line + + This could help having test executables in variable locations (the build dir) + + +.. _Criterion: https://github.com/Snaipe/Criterion +.. _Exactly: https://github.com/emilkarlen/exactly +.. _Cram: https://github.com/brodie/cram diff --git a/test/exactly/asserts/c.suite b/test/exactly/asserts/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/asserts/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/asserts/c/asserts.case b/test/exactly/asserts/c/asserts.case new file mode 100644 index 00000000..1147e5d7 --- /dev/null +++ b/test/exactly/asserts/c/asserts.case @@ -0,0 +1,24 @@ +[act] + +% asserts.c.bin + +[assert] + +stderr equals <'. +# +# This means that the test will not detect if an expected occurence of (a): +# 0.33333333333333331 +# happens to be +# 0.333333333333333333342 +# since both strings matches (b) and thus are replaced with ''. +# In this situation the test will give a false positive - the test will pass. +############################################################################### + +[act] + +% failmessages.c.bin + +[before-assert] + +def text-transformer replace_tabs_and_refs = + replace '\t' '' | + replace '0\.3333333333333333(33342|1)' '' + +[assert] + +stderr + -transformed-by replace_tabs_and_refs + equals -contents-of fail-messages.txt diff --git a/test/exactly/asserts/c/fail-messages.txt b/test/exactly/asserts/c/fail-messages.txt new file mode 100644 index 00000000..fe903fdf --- /dev/null +++ b/test/exactly/asserts/c/fail-messages.txt @@ -0,0 +1,472 @@ +[----] failmessages.c:213: Assertion Failed +[----] eq(i32, 1, 0): +[----] diff: [-1-]{+0+} +[----] failmessages.c:214: Assertion Failed +[----] failmessages.c:215: Assertion Failed +[----] eq(i32, 1, 1): +[----] diff: [-1-]{+1+} +[FAIL] message::compo +[----] failmessages.c:165: Assertion Failed +[----] lt(i8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:166: Assertion Failed +[----] lt(i16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:167: Assertion Failed +[----] lt(i32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:168: Assertion Failed +[----] lt(i64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:169: Assertion Failed +[----] lt(u8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:170: Assertion Failed +[----] lt(u16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:171: Assertion Failed +[----] lt(u32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:172: Assertion Failed +[----] lt(u64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.c:173: Assertion Failed +[----] lt(iptr, 1, 0): +[----] actual: 0x1 +[----] reference: 0x0 +[----] le(iptr, 1, 0): +[----] actual: 0x1 +[----] reference: 0x0 +[----] gt(iptr, 0, 1): +[----] actual: 0x0 +[----] reference: 0x1 +[----] ge(iptr, 0, 1): +[----] actual: 0x0 +[----] reference: 0x1 +[----] failmessages.c:174: Assertion Failed +[----] lt(uptr, 1, 0): +[----] actual: 0x1 +[----] reference: 0x0 +[----] le(uptr, 1, 0): +[----] actual: 0x1 +[----] reference: 0x0 +[----] gt(uptr, 0, 1): +[----] actual: 0x0 +[----] reference: 0x1 +[----] ge(uptr, 0, 1): +[----] actual: 0x0 +[----] reference: 0x1 +[----] failmessages.c:175: Assertion Failed +[----] lt(flt, 1 / 3.f, 0): +[----] actual: 0.333333343 +[----] reference: 0 +[----] le(flt, 1 / 3.f, 0): +[----] actual: 0.333333343 +[----] reference: 0 +[----] gt(flt, 0, 1 / 3.f): +[----] actual: 0 +[----] reference: 0.333333343 +[----] ge(flt, 0, 1 / 3.f): +[----] actual: 0 +[----] reference: 0.333333343 +[----] failmessages.c:176: Assertion Failed +[----] lt(dbl, 1 / 3., 0): +[----] actual: +[----] reference: 0 +[----] le(dbl, 1 / 3., 0): +[----] actual: +[----] reference: 0 +[----] gt(dbl, 0, 1 / 3.): +[----] actual: 0 +[----] reference: +[----] ge(dbl, 0, 1 / 3.): +[----] actual: 0 +[----] reference: +[----] failmessages.c:177: Assertion Failed +[----] lt(ldbl, 1 / 3.l, 0): +[----] actual: +[----] reference: 0 +[----] le(ldbl, 1 / 3.l, 0): +[----] actual: +[----] reference: 0 +[----] gt(ldbl, 0, 1 / 3.l): +[----] actual: 0 +[----] reference: +[----] ge(ldbl, 0, 1 / 3.l): +[----] actual: 0 +[----] reference: +[----] failmessages.c:180: Assertion Failed +[----] lt(ptr, (void *) 1, (void *) 0): +[----] actual: 0x1 +[----] reference: 0x0 +[----] le(ptr, (void *) 1, (void *) 0): +[----] actual: 0x1 +[----] reference: 0x0 +[----] gt(ptr, (void *) 0, (void *) 1): +[----] actual: 0x0 +[----] reference: 0x1 +[----] ge(ptr, (void *) 0, (void *) 1): +[----] actual: 0x0 +[----] reference: 0x1 +[----] failmessages.c:182: Assertion Failed +[----] lt(str, "cba", "abc"): +[----] actual: "cba" +[----] reference: "abc" +[----] le(str, "cba", "abc"): +[----] actual: "cba" +[----] reference: "abc" +[----] gt(str, "abc", "cba"): +[----] actual: "abc" +[----] reference: "cba" +[----] ge(str, "abc", "cba"): +[----] actual: "abc" +[----] reference: "cba" +[----] failmessages.c:183: Assertion Failed +[----] lt(str, "cba\ncba", "abc\nabc"): +[----] actual: "cba\n" +[----] "cba" +[----] reference: "abc\n" +[----] "abc" +[----] le(str, "cba\ncba", "abc\nabc"): +[----] actual: "cba\n" +[----] "cba" +[----] reference: "abc\n" +[----] "abc" +[----] gt(str, "abc\nabc", "cba\ncba"): +[----] actual: "abc\n" +[----] "abc" +[----] reference: "cba\n" +[----] "cba" +[----] ge(str, "abc\nabc", "cba\ncba"): +[----] actual: "abc\n" +[----] "abc" +[----] reference: "cba\n" +[----] "cba" +[----] failmessages.c:185: Assertion Failed +[----] lt(wcs, L"cba", L"abc"): +[----] actual: L"cba" +[----] reference: L"abc" +[----] le(wcs, L"cba", L"abc"): +[----] actual: L"cba" +[----] reference: L"abc" +[----] gt(wcs, L"abc", L"cba"): +[----] actual: L"abc" +[----] reference: L"cba" +[----] ge(wcs, L"abc", L"cba"): +[----] actual: L"abc" +[----] reference: L"cba" +[----] failmessages.c:186: Assertion Failed +[----] lt(wcs, L"cba\ncba", L"abc\nabc"): +[----] actual: L"cba\n" +[----] L"cba" +[----] reference: L"abc\n" +[----] L"abc" +[----] le(wcs, L"cba\ncba", L"abc\nabc"): +[----] actual: L"cba\n" +[----] L"cba" +[----] reference: L"abc\n" +[----] L"abc" +[----] gt(wcs, L"abc\nabc", L"cba\ncba"): +[----] actual: L"abc\n" +[----] L"abc" +[----] reference: L"cba\n" +[----] L"cba" +[----] ge(wcs, L"abc\nabc", L"cba\ncba"): +[----] actual: L"abc\n" +[----] L"abc" +[----] reference: L"cba\n" +[----] L"cba" +[----] failmessages.c:206: Assertion Failed +[----] lt(stream, shi, slo): +[----] actual: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[----] reference: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] le(stream, shi, slo): +[----] actual: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[----] reference: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] gt(stream, slo, shi): +[----] actual: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] reference: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[----] ge(stream, slo, shi): +[----] actual: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] reference: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[FAIL] messages::cmp +[----] failmessages.c:219: Assertion Failed +[----] failmessages.c:220: Assertion Failed +[----] +[----] foo bar +[----] +[FAIL] messages::default +[----] failmessages.c:80: Assertion Failed +[----] eq(i8, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:81: Assertion Failed +[----] eq(i16, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:82: Assertion Failed +[----] eq(i32, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:83: Assertion Failed +[----] eq(i64, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:84: Assertion Failed +[----] eq(u8, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:85: Assertion Failed +[----] eq(u16, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:86: Assertion Failed +[----] eq(u32, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:87: Assertion Failed +[----] eq(u64, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.c:88: Assertion Failed +[----] eq(iptr, 0, 1): +[----] diff: [-0x0-]{+0x1+} +[----] failmessages.c:89: Assertion Failed +[----] eq(uptr, 0, 1): +[----] diff: [-0x0-]{+0x1+} +[----] failmessages.c:90: Assertion Failed +[----] eq(flt, 0, 1 / 3.f): +[----] diff: [-0-]{+0.333333343+} +[----] failmessages.c:91: Assertion Failed +[----] eq(dbl, 0, 1 / 3.): +[----] diff: [-0-]{++} +[----] failmessages.c:92: Assertion Failed +[----] eq(ldbl, 0, 1 / 3.l): +[----] diff: [-0-]{++} +[----] failmessages.c:95: Assertion Failed +[----] eq(ptr, (void *) 1, (void *) 0): +[----] diff: [-0x1-]{+0x0+} +[----] failmessages.c:97: Assertion Failed +[----] eq(str, "", "foo"): +[----] diff: [-""-]{+"foo"+} +[----] failmessages.c:98: Assertion Failed +[----] eq(str, "reallyreallylongstringindeedmygoodsirormadam", "yetanotherreallyreallylongstring"): +[----] @@ -1,1 +1,1 @@ +[----] -"reallyreallylongstringindeedmygoodsirormadam" +[----] +"yetanotherreallyreallylongstring" +[----] failmessages.c:99: Assertion Failed +[----] eq(str, "foo\nbar", "foo\nbaz"): +[----] @@ -1,2 +1,2 @@ +[----] "foo\n" +[----] -"bar" +[----] +"baz" +[----] failmessages.c:101: Assertion Failed +[----] eq(wcs, L"", L"foo"): +[----] diff: [-L""-]{+L"foo"+} +[----] failmessages.c:102: Assertion Failed +[----] eq(wcs, L"reallyreallylongstringindeedmygoodsirormadam", L"yetanotherreallyreallylongstring"): +[----] @@ -1,1 +1,1 @@ +[----] -L"reallyreallylongstringindeedmygoodsirormadam" +[----] +L"yetanotherreallyreallylongstring" +[----] failmessages.c:103: Assertion Failed +[----] eq(wcs, L"foo\nbar", L"foo\nbaz"): +[----] @@ -1,2 +1,2 @@ +[----] L"foo\n" +[----] -L"bar" +[----] +L"baz" +[----] failmessages.c:111: Assertion Failed +[----] eq(mem, ma, mb): +[----] @@ -1,2 +1,2 @@ +[----] -00: 00000000 .... +[----] +00: 01000000 .... +[----] +[----] failmessages.c:112: Assertion Failed +[----] eq(int[1], &a, &b): +[----] @@ -1,3 +1,3 @@ +[----] (int[1]) { +[----] -[0] = 0, +[----] +[0] = 1, +[----] } +[----] failmessages.c:125: Assertion Failed +[----] eq(mem, marra, marrb): +[----] @@ -1,3 +1,3 @@ +[----] -00: 00000000 01000000 02000000 03000000 ................ +[----] -10: 04000000 .... +[----] +00: 04000000 03000000 02000000 01000000 ................ +[----] +10: 00000000 .... +[----] +[----] failmessages.c:126: Assertion Failed +[----] eq(int[sizeof (arra) / sizeof (int)], arra, arrb): +[----] @@ -1,7 +1,7 @@ +[----] (int[5]) { +[----] -[0] = 0, +[----] -[1] = 1, +[----] +[0] = 4, +[----] +[1] = 3, +[----] [2] = 2, +[----] -[3] = 3, +[----] -[4] = 4, +[----] +[3] = 1, +[----] +[4] = 0, +[----] } +[----] failmessages.c:131: Assertion Failed +[----] eq(type(struct dummy_struct), dummy1, dummy2): +[----] @@ -1,4 +1,4 @@ +[----] (struct dummy_struct) { +[----] .a = 42, +[----] -.b = 24 +[----] +.b = 42 +[----] } +[----] failmessages.c:132: Assertion Failed +[----] eq(type(struct dummy_struct)[1], &dummy1, &dummy2): +[----] @@ -1,6 +1,6 @@ +[----] (struct dummy_struct[1]) { +[----] [0] = (struct dummy_struct) { +[----] .a = 42, +[----] -.b = 24 +[----] +.b = 42 +[----] }, +[----] } +[----] failmessages.c:152: Assertion Failed +[----] eq(stream, s1, s2): +[----] @@ -1,2 +1,2 @@ +[----] -00: 68656c6c 6f20776f 726c6400 hello world. +[----] +00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[FAIL] messages::eq +[----] failmessages.c:53: Assertion Failed +[----] zero(i8, 0): +[----] value: 0 +[----] failmessages.c:54: Assertion Failed +[----] zero(i16, 0): +[----] value: 0 +[----] failmessages.c:55: Assertion Failed +[----] zero(i32, 0): +[----] value: 0 +[----] failmessages.c:56: Assertion Failed +[----] zero(i64, 0): +[----] value: 0 +[----] failmessages.c:57: Assertion Failed +[----] zero(u8, 0): +[----] value: 0 +[----] failmessages.c:58: Assertion Failed +[----] zero(u16, 0): +[----] value: 0 +[----] failmessages.c:59: Assertion Failed +[----] zero(u32, 0): +[----] value: 0 +[----] failmessages.c:60: Assertion Failed +[----] zero(u64, 0): +[----] value: 0 +[----] failmessages.c:61: Assertion Failed +[----] zero(iptr, 0): +[----] value: 0x0 +[----] failmessages.c:62: Assertion Failed +[----] zero(uptr, 0): +[----] value: 0x0 +[----] failmessages.c:63: Assertion Failed +[----] zero(flt, 0): +[----] value: 0 +[----] failmessages.c:64: Assertion Failed +[----] zero(dbl, 0): +[----] value: 0 +[----] failmessages.c:65: Assertion Failed +[----] zero(ldbl, 0): +[----] value: 0 +[----] failmessages.c:68: Assertion Failed +[----] zero(ptr, 0): +[----] value: 0x0 +[----] failmessages.c:69: Assertion Failed +[----] zero(str, ""): +[----] value: "" +[----] failmessages.c:70: Assertion Failed +[----] zero(wcs, L""): +[----] value: L"" +[----] failmessages.c:75: Assertion Failed +[----] zero(type(struct dummy_struct), dummy1): +[----] value: (struct dummy_struct) { +[----] .a = 0, +[----] .b = 0 +[----] } +[FAIL] messages::zero +[====] Synthesis: Tested: 5 | Passing: 0 | Failing: 5 | Crashing: 0 diff --git a/test/exactly/asserts/c/long-messages.case b/test/exactly/asserts/c/long-messages.case new file mode 100644 index 00000000..76aee0b5 --- /dev/null +++ b/test/exactly/asserts/c/long-messages.case @@ -0,0 +1,17 @@ +[act] + +% long-messages.c.bin + +[assert] + +stderr equals < +[----] asserts.cc:169: Assertion Failed +[----] throw(std::bad_alloc, throw std::invalid_argument("some other message")): +[----] message: "some other message" +[FAIL] asserts::exception +[----] asserts.cc:21: Assertion Failed +[----] +[----] You can fail an assertion with a message from anywhere +[----] +[----] asserts.cc:22: Assertion Failed +[FAIL] asserts::old_school +[====] Synthesis: Tested: 9 | Passing: 6 | Failing: 3 | Crashing: 0 +EOF diff --git a/test/exactly/asserts/cxx/exactly.suite b/test/exactly/asserts/cxx/exactly.suite new file mode 100644 index 00000000..b93e2dc4 --- /dev/null +++ b/test/exactly/asserts/cxx/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/asserts/cxx/fail-messages.case b/test/exactly/asserts/cxx/fail-messages.case new file mode 100644 index 00000000..0bf3732f --- /dev/null +++ b/test/exactly/asserts/cxx/fail-messages.case @@ -0,0 +1,30 @@ +############################################################################### +# Drawback - false positives +# +# This impl of the test is not able to distinguish between +# (a) (contant) 0.33333333333333331 +# and +# (b) (reg-ex) 0\.3333333333333333(33342|1) +# since (b) matches (a), and both are replaced with ''. +# +# This means that the test will not detect if an expected occurence of (a): +# 0.33333333333333331 +# happens to be +# 0.333333333333333333342 +# since both strings matches (b) and thus are replaced with ''. +# In this situation the test will give a false positive - the test will pass. +############################################################################### + +[act] + +% failmessages.cc.bin + +[before-assert] + +def text-transformer replace_tabs = replace '\t' '' + +[assert] + +stderr + -transformed-by replace_tabs + equals -contents-of fail-messages.txt diff --git a/test/exactly/asserts/cxx/fail-messages.txt b/test/exactly/asserts/cxx/fail-messages.txt new file mode 100644 index 00000000..66b97799 --- /dev/null +++ b/test/exactly/asserts/cxx/fail-messages.txt @@ -0,0 +1,479 @@ +[----] failmessages.cc:193: Assertion Failed +[----] eq(i32, 1, 0): +[----] diff: [-1-]{+0+} +[----] failmessages.cc:194: Assertion Failed +[----] failmessages.cc:195: Assertion Failed +[----] eq(i32, 1, 1): +[----] diff: [-1-]{+1+} +[FAIL] message::compo +[----] failmessages.cc:199: Assertion Failed +[----] throw(std::bad_alloc, throw std::invalid_argument("exception message")): +[----] message: "exception message" +[----] failmessages.cc:200: Assertion Failed +[----] failmessages.cc:201: Assertion Failed +[----] nothrow(throw std::invalid_argument("exception message")): +[----] message: "exception message" +[----] failmessages.cc:202: Assertion Failed +[FAIL] message::exception +[----] failmessages.cc:151: Assertion Failed +[----] lt(i8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:152: Assertion Failed +[----] lt(i16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:153: Assertion Failed +[----] lt(i32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:154: Assertion Failed +[----] lt(i64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(i64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(i64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(i64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:155: Assertion Failed +[----] lt(u8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u8, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u8, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:156: Assertion Failed +[----] lt(u16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u16, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u16, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:157: Assertion Failed +[----] lt(u32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u32, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u32, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:158: Assertion Failed +[----] lt(u64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(u64, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(u64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(u64, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:159: Assertion Failed +[----] lt(iptr, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(iptr, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(iptr, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(iptr, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:160: Assertion Failed +[----] lt(uptr, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] le(uptr, 1, 0): +[----] actual: 1 +[----] reference: 0 +[----] gt(uptr, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] ge(uptr, 0, 1): +[----] actual: 0 +[----] reference: 1 +[----] failmessages.cc:161: Assertion Failed +[----] lt(flt, 1 / 3.f, 0): +[----] actual: 0.333333 +[----] reference: 0 +[----] le(flt, 1 / 3.f, 0): +[----] actual: 0.333333 +[----] reference: 0 +[----] gt(flt, 0, 1 / 3.f): +[----] actual: 0 +[----] reference: 0.333333 +[----] ge(flt, 0, 1 / 3.f): +[----] actual: 0 +[----] reference: 0.333333 +[----] failmessages.cc:162: Assertion Failed +[----] lt(dbl, 1 / 3., 0): +[----] actual: 0.333333 +[----] reference: 0 +[----] le(dbl, 1 / 3., 0): +[----] actual: 0.333333 +[----] reference: 0 +[----] gt(dbl, 0, 1 / 3.): +[----] actual: 0 +[----] reference: 0.333333 +[----] ge(dbl, 0, 1 / 3.): +[----] actual: 0 +[----] reference: 0.333333 +[----] failmessages.cc:163: Assertion Failed +[----] lt(ldbl, 1 / 3.l, 0): +[----] actual: 0.333333 +[----] reference: 0 +[----] le(ldbl, 1 / 3.l, 0): +[----] actual: 0.333333 +[----] reference: 0 +[----] gt(ldbl, 0, 1 / 3.l): +[----] actual: 0 +[----] reference: 0.333333 +[----] ge(ldbl, 0, 1 / 3.l): +[----] actual: 0 +[----] reference: 0.333333 +[----] failmessages.cc:166: Assertion Failed +[----] lt(ptr, (void *) 1, (void *) 0): +[----] actual: @1 +[----] reference: nullptr +[----] le(ptr, (void *) 1, (void *) 0): +[----] actual: @1 +[----] reference: nullptr +[----] gt(ptr, (void *) 0, (void *) 1): +[----] actual: nullptr +[----] reference: @1 +[----] ge(ptr, (void *) 0, (void *) 1): +[----] actual: nullptr +[----] reference: @1 +[----] failmessages.cc:168: Assertion Failed +[----] lt(str, "cba", "abc"): +[----] actual: "cba" +[----] reference: "abc" +[----] le(str, "cba", "abc"): +[----] actual: "cba" +[----] reference: "abc" +[----] gt(str, "abc", "cba"): +[----] actual: "abc" +[----] reference: "cba" +[----] ge(str, "abc", "cba"): +[----] actual: "abc" +[----] reference: "cba" +[----] failmessages.cc:169: Assertion Failed +[----] lt(str, "cba\ncba", "abc\nabc"): +[----] actual: "cba\n" +[----] "cba" +[----] reference: "abc\n" +[----] "abc" +[----] le(str, "cba\ncba", "abc\nabc"): +[----] actual: "cba\n" +[----] "cba" +[----] reference: "abc\n" +[----] "abc" +[----] gt(str, "abc\nabc", "cba\ncba"): +[----] actual: "abc\n" +[----] "abc" +[----] reference: "cba\n" +[----] "cba" +[----] ge(str, "abc\nabc", "cba\ncba"): +[----] actual: "abc\n" +[----] "abc" +[----] reference: "cba\n" +[----] "cba" +[----] failmessages.cc:171: Assertion Failed +[----] lt(wcs, L"cba", L"abc"): +[----] actual: L"cba" +[----] reference: L"abc" +[----] le(wcs, L"cba", L"abc"): +[----] actual: L"cba" +[----] reference: L"abc" +[----] gt(wcs, L"abc", L"cba"): +[----] actual: L"abc" +[----] reference: L"cba" +[----] ge(wcs, L"abc", L"cba"): +[----] actual: L"abc" +[----] reference: L"cba" +[----] failmessages.cc:172: Assertion Failed +[----] lt(wcs, L"cba\ncba", L"abc\nabc"): +[----] actual: L"cba\n" +[----] L"cba" +[----] reference: L"abc\n" +[----] L"abc" +[----] le(wcs, L"cba\ncba", L"abc\nabc"): +[----] actual: L"cba\n" +[----] L"cba" +[----] reference: L"abc\n" +[----] L"abc" +[----] gt(wcs, L"abc\nabc", L"cba\ncba"): +[----] actual: L"abc\n" +[----] L"abc" +[----] reference: L"cba\n" +[----] L"cba" +[----] ge(wcs, L"abc\nabc", L"cba\ncba"): +[----] actual: L"abc\n" +[----] L"abc" +[----] reference: L"cba\n" +[----] L"cba" +[----] failmessages.cc:189: Assertion Failed +[----] lt(stream, shi, slo): +[----] actual: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[----] reference: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] le(stream, shi, slo): +[----] actual: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[----] reference: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] gt(stream, slo, shi): +[----] actual: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] reference: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[----] ge(stream, slo, shi): +[----] actual: 00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[----] reference: 00: 68656c6c 6f20776f 726c6400 hello world. +[----] +[FAIL] messages::cmp +[----] failmessages.cc:206: Assertion Failed +[----] failmessages.cc:207: Assertion Failed +[----] +[----] foo bar +[----] +[FAIL] messages::default +[----] failmessages.cc:69: Assertion Failed +[----] eq(i8, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:70: Assertion Failed +[----] eq(i16, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:71: Assertion Failed +[----] eq(i32, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:72: Assertion Failed +[----] eq(i64, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:73: Assertion Failed +[----] eq(u8, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:74: Assertion Failed +[----] eq(u16, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:75: Assertion Failed +[----] eq(u32, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:76: Assertion Failed +[----] eq(u64, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:77: Assertion Failed +[----] eq(iptr, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:78: Assertion Failed +[----] eq(uptr, 0, 1): +[----] diff: [-0-]{+1+} +[----] failmessages.cc:79: Assertion Failed +[----] eq(flt, 0, 1 / 3.f): +[----] diff: [-0-]{+0.333333+} +[----] failmessages.cc:80: Assertion Failed +[----] eq(dbl, 0, 1 / 3.): +[----] diff: [-0-]{+0.333333+} +[----] failmessages.cc:81: Assertion Failed +[----] eq(ldbl, 0, 1 / 3.l): +[----] diff: [-0-]{+0.333333+} +[----] failmessages.cc:84: Assertion Failed +[----] eq(ptr, (void *) 1, (void *) 0): +[----] diff: [-@1-]{+nullptr+} +[----] failmessages.cc:86: Assertion Failed +[----] eq(str, "", "foo"): +[----] diff: [-""-]{+"foo"+} +[----] failmessages.cc:87: Assertion Failed +[----] eq(str, "reallyreallylongstringindeedmygoodsirormadam", "yetanotherreallyreallylongstring"): +[----] @@ -1,1 +1,1 @@ +[----] -"reallyreallylongstringindeedmygoodsirormadam" +[----] +"yetanotherreallyreallylongstring" +[----] failmessages.cc:88: Assertion Failed +[----] eq(str, "foo\nbar", "foo\nbaz"): +[----] @@ -1,2 +1,2 @@ +[----] "foo\n" +[----] -"bar" +[----] +"baz" +[----] failmessages.cc:90: Assertion Failed +[----] eq(wcs, L"", L"foo"): +[----] diff: [-L""-]{+L"foo"+} +[----] failmessages.cc:91: Assertion Failed +[----] eq(wcs, L"reallyreallylongstringindeedmygoodsirormadam", L"yetanotherreallyreallylongstring"): +[----] @@ -1,1 +1,1 @@ +[----] -L"reallyreallylongstringindeedmygoodsirormadam" +[----] +L"yetanotherreallyreallylongstring" +[----] failmessages.cc:92: Assertion Failed +[----] eq(wcs, L"foo\nbar", L"foo\nbaz"): +[----] @@ -1,2 +1,2 @@ +[----] L"foo\n" +[----] -L"bar" +[----] +L"baz" +[----] failmessages.cc:100: Assertion Failed +[----] eq(mem, ma, mb): +[----] @@ -1,2 +1,2 @@ +[----] -00: 00 . +[----] +00: 01 . +[----] +[----] failmessages.cc:101: Assertion Failed +[----] eq(int[1], &a, &b): +[----] @@ -1,3 +1,3 @@ +[----] (int[1]) { +[----] -[0] = 0, +[----] +[0] = 1, +[----] } +[----] failmessages.cc:114: Assertion Failed +[----] eq(mem, marra, marrb): +[----] @@ -1,2 +1,2 @@ +[----] -00: 00000000 01 ..... +[----] +00: 04000000 03 ..... +[----] +[----] failmessages.cc:115: Assertion Failed +[----] eq(int[sizeof (arra) / sizeof (int)], arra, arrb): +[----] @@ -1,7 +1,7 @@ +[----] (int[5]) { +[----] -[0] = 0, +[----] -[1] = 1, +[----] +[0] = 4, +[----] +[1] = 3, +[----] [2] = 2, +[----] -[3] = 3, +[----] -[4] = 4, +[----] +[3] = 1, +[----] +[4] = 0, +[----] } +[----] failmessages.cc:120: Assertion Failed +[----] eq(type(struct dummy_struct), dummy1, dummy2): +[----] @@ -1,4 +1,4 @@ +[----] (struct dummy_struct) { +[----] .a = 42, +[----] -.b = 24 +[----] +.b = 42 +[----] } +[----] failmessages.cc:121: Assertion Failed +[----] eq(type(struct dummy_struct)[1], &dummy1, &dummy2): +[----] @@ -1,6 +1,6 @@ +[----] (struct dummy_struct[1]) { +[----] [0] = (struct dummy_struct) { +[----] .a = 42, +[----] -.b = 24 +[----] +.b = 42 +[----] }, +[----] } +[----] failmessages.cc:138: Assertion Failed +[----] eq(stream, s1, s2): +[----] @@ -1,2 +1,2 @@ +[----] -00: 68656c6c 6f20776f 726c6400 hello world. +[----] +00: 646c726f 77206f6c 6c656800 dlrow olleh. +[----] +[FAIL] messages::eq +[----] failmessages.cc:45: Assertion Failed +[----] zero(i8, 0): +[----] value: 0 +[----] failmessages.cc:46: Assertion Failed +[----] zero(i16, 0): +[----] value: 0 +[----] failmessages.cc:47: Assertion Failed +[----] zero(i32, 0): +[----] value: 0 +[----] failmessages.cc:48: Assertion Failed +[----] zero(i64, 0): +[----] value: 0 +[----] failmessages.cc:49: Assertion Failed +[----] zero(u8, 0): +[----] value: 0 +[----] failmessages.cc:50: Assertion Failed +[----] zero(u16, 0): +[----] value: 0 +[----] failmessages.cc:51: Assertion Failed +[----] zero(u32, 0): +[----] value: 0 +[----] failmessages.cc:52: Assertion Failed +[----] zero(u64, 0): +[----] value: 0 +[----] failmessages.cc:53: Assertion Failed +[----] zero(iptr, 0): +[----] value: 0 +[----] failmessages.cc:54: Assertion Failed +[----] zero(uptr, 0): +[----] value: 0 +[----] failmessages.cc:55: Assertion Failed +[----] zero(flt, 0): +[----] value: 0 +[----] failmessages.cc:56: Assertion Failed +[----] zero(dbl, 0): +[----] value: 0 +[----] failmessages.cc:57: Assertion Failed +[----] zero(ldbl, 0): +[----] value: 0 +[----] failmessages.cc:60: Assertion Failed +[----] zero(ptr, 0): +[----] value: nullptr +[----] failmessages.cc:61: Assertion Failed +[----] zero(str, ""): +[----] value: "" +[----] failmessages.cc:62: Assertion Failed +[----] zero(wcs, L""): +[----] value: L"" +[----] failmessages.cc:64: Assertion Failed +[----] zero(type(dummy_struct), dummy_struct{}): +[----] value: (struct dummy_struct) { +[----] .a = 0, +[----] .b = 0 +[----] } +[FAIL] messages::zero +[====] Synthesis: Tested: 6 | Passing: 0 | Failing: 6 | Crashing: 0 diff --git a/test/exactly/asserts/cxx/long-messages.case b/test/exactly/asserts/cxx/long-messages.case new file mode 100644 index 00000000..f34ba5b7 --- /dev/null +++ b/test/exactly/asserts/cxx/long-messages.case @@ -0,0 +1,17 @@ +[act] + +% long-messages.cc.bin + +[assert] + +stderr equals <' diff --git a/test/exactly/core/c.suite b/test/exactly/core/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/core/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/core/c/descriptions.case b/test/exactly/core/c/descriptions.case new file mode 100644 index 00000000..4f62b3c8 --- /dev/null +++ b/test/exactly/core/c/descriptions.case @@ -0,0 +1,27 @@ +[act] + +% description.c.bin --verbose + +[assert] + +exit-code == 0 + +stdout is-empty + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] Just a failing test +[----] description.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[SKIP] misc::skipped: Test is disabled +[----] This one is skipped +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/core/c/exactly.suite b/test/exactly/core/c/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/core/c/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/core/c/fixtures.case b/test/exactly/core/c/fixtures.case new file mode 100644 index 00000000..07748411 --- /dev/null +++ b/test/exactly/core/c/fixtures.case @@ -0,0 +1,18 @@ +[act] + +% fixtures.c.bin + +[assert] + +exit-code == 0 + +stdout equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/core/c/simple.case b/test/exactly/core/c/simple.case new file mode 100644 index 00000000..0870faac --- /dev/null +++ b/test/exactly/core/c/simple.case @@ -0,0 +1,19 @@ +[act] + +% simple.c.bin + +[assert] + +exit-code == 0 + +stdout is-empty + +stderr equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] Just a failing test +[----] description.cc:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[SKIP] misc::skipped: Test is disabled +[----] This one is skipped +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/core/cxx/exactly.suite b/test/exactly/core/cxx/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/core/cxx/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/core/cxx/fixtures.case b/test/exactly/core/cxx/fixtures.case new file mode 100644 index 00000000..f38001b8 --- /dev/null +++ b/test/exactly/core/cxx/fixtures.case @@ -0,0 +1,18 @@ +[act] + +% fixtures.cc.bin + +[assert] + +exit-code == 0 + +stdout equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.cc:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/core/cxx/simple.case b/test/exactly/core/cxx/simple.case new file mode 100644 index 00000000..035598a1 --- /dev/null +++ b/test/exactly/core/cxx/simple.case @@ -0,0 +1,19 @@ +[act] + +% simple.cc.bin + +[assert] + +exit-code == 0 + +stdout is-empty + +stderr equals +< +[====] Running 2 tests from misc: +[RUN ] misc::setup_crash +[----] Warning! The test `misc::setup_crash` crashed during its setup or teardown. +[RUN ] misc::teardown_crash +[PASS] misc::teardown_crash +[----] Warning! The test `misc::teardown_crash` crashed during its setup or teardown. +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 1 +EOF diff --git a/test/exactly/crashes/c/other-crashes.case b/test/exactly/crashes/c/other-crashes.case new file mode 100644 index 00000000..b6aea171 --- /dev/null +++ b/test/exactly/crashes/c/other-crashes.case @@ -0,0 +1,14 @@ +[act] + +% other-crashes.c.bin + +[assert] + +`Crashes outside of the test` + +stderr equals +< +[====] Running 3 tests from simple: +[RUN ] simple::caught +[PASS] simple::caught +[RUN ] simple::uncaught +[----] signal.c:14: Unexpected signal caught below this line! +[FAIL] simple::uncaught: CRASH! +[RUN ] simple::wrong_signal +[FAIL] simple::wrong_signal +[====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1 +EOF diff --git a/test/exactly/crashes/c/signal.case b/test/exactly/crashes/c/signal.case new file mode 100644 index 00000000..de68d552 --- /dev/null +++ b/test/exactly/crashes/c/signal.case @@ -0,0 +1,12 @@ +[act] + +% signal.c.bin + +[assert] + +stderr equals < +[====] Running 6 tests from exception: +[RUN ] exception::raise_any +[----] exception::raise_any: Caught some unexpected exception during the test execution. +[FAIL] exception::raise_any +[RUN ] exception::raise_any_fini +[PASS] exception::raise_any_fini +[----] exception::raise_any_fini: Caught some unexpected exception during the test finalization. +[RUN ] exception::raise_any_init +[----] exception::raise_any_init: Caught some unexpected exception during the test initialization. +[FAIL] exception::raise_any_init +[RUN ] exception::raise_std +[----] exception::raise_std: Caught an unexpected exception during the test execution: Some exception message. +[FAIL] exception::raise_std +[RUN ] exception::raise_std_fini +[PASS] exception::raise_std_fini +[----] exception::raise_std_fini: Caught an unexpected exception during the test finalization: Some exception message. +[RUN ] exception::raise_std_init +[----] exception::raise_std_init: Caught an unexpected exception during the test initialization: Some exception message. +[FAIL] exception::raise_std_init +[====] Synthesis: Tested: 6 | Passing: 2 | Failing: 4 | Crashing: 0 +EOF diff --git a/test/exactly/crashes/cxx/exception.case b/test/exactly/crashes/cxx/exception.case new file mode 100644 index 00000000..9e23000d --- /dev/null +++ b/test/exactly/crashes/cxx/exception.case @@ -0,0 +1,19 @@ +[act] + +% exception.cc.bin + +[assert] + +stderr equals < +[====] Running 2 tests from misc: +[RUN ] misc::setup_crash +[----] Warning! The test `misc::setup_crash` crashed during its setup or teardown. +[RUN ] misc::teardown_crash +[PASS] misc::teardown_crash +[----] Warning! The test `misc::teardown_crash` crashed during its setup or teardown. +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 1 +EOF diff --git a/test/exactly/crashes/cxx/other-crashes.case b/test/exactly/crashes/cxx/other-crashes.case new file mode 100644 index 00000000..e206649a --- /dev/null +++ b/test/exactly/crashes/cxx/other-crashes.case @@ -0,0 +1,14 @@ +[act] + +% other-crashes.cc.bin + +[assert] + +`Crashes outside of the test` + +stderr equals +< +[====] Running 3 tests from simple: +[RUN ] simple::caught +[PASS] simple::caught +[RUN ] simple::uncaught +[----] signal.cc:14: Unexpected signal caught below this line! +[FAIL] simple::uncaught: CRASH! +[RUN ] simple::wrong_signal +[FAIL] simple::wrong_signal +[====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1 +EOF diff --git a/test/exactly/crashes/cxx/signal.case b/test/exactly/crashes/cxx/signal.case new file mode 100644 index 00000000..534a6a34 --- /dev/null +++ b/test/exactly/crashes/cxx/signal.case @@ -0,0 +1,12 @@ +[act] + +% signal.cc.bin + +[assert] + +stderr equals < +[====] Running 3 tests from exit: +[RUN ] exit::expected_exit +[PASS] exit::expected_exit +[RUN ] exit::normal +[PASS] exit::normal +[RUN ] exit::unexpected_exit +[FAIL] exit::unexpected_exit +[====] Running 2 tests from exit_with_fixtures: +[RUN ] exit_with_fixtures::fini_exits +[PASS] exit_with_fixtures::fini_exits +[----] Warning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown. +[RUN ] exit_with_fixtures::init_exits +[----] Warning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown. +[====] Synthesis: Tested: 5 | Passing: 3 | Failing: 2 | Crashing: 1 +EOF diff --git a/test/exactly/exit/common.xly b/test/exactly/exit/common.xly new file mode 100644 index 00000000..1b84bbb3 --- /dev/null +++ b/test/exactly/exit/common.xly @@ -0,0 +1,3 @@ +[setup] + +including ../common.xly diff --git a/test/exactly/exit/cxx.suite b/test/exactly/exit/cxx.suite new file mode 100644 index 00000000..874a9fbb --- /dev/null +++ b/test/exactly/exit/cxx.suite @@ -0,0 +1,3 @@ +[suites] + +cxx diff --git a/test/exactly/exit/cxx/exactly.suite b/test/exactly/exit/cxx/exactly.suite new file mode 100644 index 00000000..b93e2dc4 --- /dev/null +++ b/test/exactly/exit/cxx/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/exit/cxx/exit.case b/test/exactly/exit/cxx/exit.case new file mode 100644 index 00000000..f988ee14 --- /dev/null +++ b/test/exactly/exit/cxx/exit.case @@ -0,0 +1,26 @@ +[act] + +% exit.cc.bin --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 3 tests from exit: +[RUN ] exit::expected_exit +[PASS] exit::expected_exit +[RUN ] exit::normal +[PASS] exit::normal +[RUN ] exit::unexpected_exit +[FAIL] exit::unexpected_exit +[====] Running 2 tests from exit_with_fixtures: +[RUN ] exit_with_fixtures::fini_exits +[PASS] exit_with_fixtures::fini_exits +[----] Warning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown. +[RUN ] exit_with_fixtures::init_exits +[----] Warning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown. +[====] Synthesis: Tested: 5 | Passing: 3 | Failing: 2 | Crashing: 1 +EOF diff --git a/test/exactly/exit/exactly.suite b/test/exactly/exit/exactly.suite new file mode 100644 index 00000000..b56d011a --- /dev/null +++ b/test/exactly/exit/exactly.suite @@ -0,0 +1,4 @@ +[suites] + +c +cxx diff --git a/test/exactly/fail-fast/c.suite b/test/exactly/fail-fast/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/fail-fast/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/fail-fast/c/cli-option.case b/test/exactly/fail-fast/c/cli-option.case new file mode 100644 index 00000000..c45b4072 --- /dev/null +++ b/test/exactly/fail-fast/c/cli-option.case @@ -0,0 +1,13 @@ +[act] + +% simple.c.bin --fail-fast --verbose + +[assert] + +exit-code == 1 + +stdout is-empty + +stderr + -transformed-by replace_version + equals -contents-of err-output.txt diff --git a/test/exactly/fail-fast/c/env-var.case b/test/exactly/fail-fast/c/env-var.case new file mode 100644 index 00000000..0f9da1af --- /dev/null +++ b/test/exactly/fail-fast/c/env-var.case @@ -0,0 +1,17 @@ +[setup] + +env -of act CRITERION_FAIL_FAST = 1 + +[act] + +% simple.c.bin --verbose + +[assert] + +exit-code == 1 + +stdout is-empty + +stderr + -transformed-by replace_version + equals -contents-of err-output.txt diff --git a/test/exactly/fail-fast/c/err-output.txt b/test/exactly/fail-fast/c/err-output.txt new file mode 100644 index 00000000..e2fc23bc --- /dev/null +++ b/test/exactly/fail-fast/c/err-output.txt @@ -0,0 +1,9 @@ +[----] Criterion +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 diff --git a/test/exactly/fail-fast/c/exactly.suite b/test/exactly/fail-fast/c/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/fail-fast/c/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/fail-fast/common.xly b/test/exactly/fail-fast/common.xly new file mode 100644 index 00000000..1b84bbb3 --- /dev/null +++ b/test/exactly/fail-fast/common.xly @@ -0,0 +1,3 @@ +[setup] + +including ../common.xly diff --git a/test/exactly/fail-fast/cxx.suite b/test/exactly/fail-fast/cxx.suite new file mode 100644 index 00000000..874a9fbb --- /dev/null +++ b/test/exactly/fail-fast/cxx.suite @@ -0,0 +1,3 @@ +[suites] + +cxx diff --git a/test/exactly/fail-fast/cxx/cli-option.case b/test/exactly/fail-fast/cxx/cli-option.case new file mode 100644 index 00000000..7d2ec52a --- /dev/null +++ b/test/exactly/fail-fast/cxx/cli-option.case @@ -0,0 +1,13 @@ +[act] + +% simple.cc.bin --fail-fast --verbose + +[assert] + +exit-code == 1 + +stdout is-empty + +stderr + -transformed-by replace_version + equals -contents-of err-output.txt diff --git a/test/exactly/fail-fast/cxx/env-var.case b/test/exactly/fail-fast/cxx/env-var.case new file mode 100644 index 00000000..e839a5e1 --- /dev/null +++ b/test/exactly/fail-fast/cxx/env-var.case @@ -0,0 +1,17 @@ +[setup] + +env -of act CRITERION_FAIL_FAST = 1 + +[act] + +% simple.cc.bin --verbose + +[assert] + +exit-code == 1 + +stdout is-empty + +stderr + -transformed-by replace_version + equals -contents-of err-output.txt diff --git a/test/exactly/fail-fast/cxx/err-output.txt b/test/exactly/fail-fast/cxx/err-output.txt new file mode 100644 index 00000000..660c65ad --- /dev/null +++ b/test/exactly/fail-fast/cxx/err-output.txt @@ -0,0 +1,9 @@ +[----] Criterion +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.cc:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 diff --git a/test/exactly/fail-fast/cxx/exactly.suite b/test/exactly/fail-fast/cxx/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/fail-fast/cxx/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/fail-fast/exactly.suite b/test/exactly/fail-fast/exactly.suite new file mode 100644 index 00000000..b56d011a --- /dev/null +++ b/test/exactly/fail-fast/exactly.suite @@ -0,0 +1,4 @@ +[suites] + +c +cxx diff --git a/test/exactly/help/c.suite b/test/exactly/help/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/help/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/help/c/exactly.suite b/test/exactly/help/c/exactly.suite new file mode 100644 index 00000000..b93e2dc4 --- /dev/null +++ b/test/exactly/help/c/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/help/c/help.case b/test/exactly/help/c/help.case new file mode 100644 index 00000000..a84d8633 --- /dev/null +++ b/test/exactly/help/c/help.case @@ -0,0 +1,7 @@ +[act] + +% simple.c.bin --help + +[assert] + +stderr is_help_text diff --git a/test/exactly/help/common.xly b/test/exactly/help/common.xly new file mode 100644 index 00000000..c686ad17 --- /dev/null +++ b/test/exactly/help/common.xly @@ -0,0 +1,24 @@ +[conf] + +including ../common.xly + +home = . + +[before-assert] + +def text-transformer replace_executable = + + replace -at ( contents matches 'usage:' ) + '[^ ]*simple\.cc?\.bin(\.exe)?' + + +def text-matcher is_help_text = + + -transformed-by ( replace_version | replace_executable ) + equals -contents-of expected/help-text.txt + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/help/cxx.suite b/test/exactly/help/cxx.suite new file mode 100644 index 00000000..874a9fbb --- /dev/null +++ b/test/exactly/help/cxx.suite @@ -0,0 +1,3 @@ +[suites] + +cxx diff --git a/test/exactly/help/cxx/exactly.suite b/test/exactly/help/cxx/exactly.suite new file mode 100644 index 00000000..b93e2dc4 --- /dev/null +++ b/test/exactly/help/cxx/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/help/cxx/help.case b/test/exactly/help/cxx/help.case new file mode 100644 index 00000000..06279f8c --- /dev/null +++ b/test/exactly/help/cxx/help.case @@ -0,0 +1,7 @@ +[act] + +% simple.cc.bin --help + +[assert] + +stderr is_help_text diff --git a/test/exactly/help/exactly.suite b/test/exactly/help/exactly.suite new file mode 100644 index 00000000..b56d011a --- /dev/null +++ b/test/exactly/help/exactly.suite @@ -0,0 +1,4 @@ +[suites] + +c +cxx diff --git a/test/exactly/help/expected/help-text.txt b/test/exactly/help/expected/help-text.txt new file mode 100644 index 00000000..83fc3e6b --- /dev/null +++ b/test/exactly/help/expected/help-text.txt @@ -0,0 +1,27 @@ +Tests compiled with Criterion + +usage: OPTIONS +options: + -h or --help: prints this message + -q or --quiet: disables all logging + -v or --version: prints the version of criterion these tests have been linked against + -l or --list: prints all the tests in a list + -jN or --jobs N: use N concurrent jobs + -f or --fail-fast: exit after the first failure + --color=: colorize the output + --encoding=: use the specified encoding for the output (default: locale-deduced) + --ascii: don't use fancy unicode symbols or colors in the output + -S or --short-filename: only display the base name of the source file on a failure + --filter [PATTERN]: run tests matching the given pattern + --timeout [TIMEOUT]: set a timeout (in seconds) for all tests + --tap[=FILE]: writes TAP report in FILE (no file or "-" means stderr) + --xml[=FILE]: writes XML report in FILE (no file or "-" means stderr) + --json[=FILE]: writes JSON report in FILE (no file or "-" means stderr) + --always-succeed: always exit with 0 + --verbose[=level]: sets verbosity to level (1 by default) + --crash: crash failing assertions rather than aborting (for debugging purposes) + --debug[=TYPE]: run tests with a debugging server, listening on localhost:1234 by default. TYPE may be gdb, lldb, or wingbd. + --debug-transport=VAL: the transport to use by the debugging server. `tcp:1234` by default + --full-stats: Tests must fully report statistics (causes massive slowdown for large number of assertions but is more accurate). + --ignore-warnings: Ignore warnings, do not exit with a non-zero exit status. + -OP:F or --output=PROVIDER=FILE: write test report to FILE using the specified provider diff --git a/test/exactly/log/c.suite b/test/exactly/log/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/log/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/log/c/exactly.suite b/test/exactly/log/c/exactly.suite new file mode 100644 index 00000000..b93e2dc4 --- /dev/null +++ b/test/exactly/log/c/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/log/c/expected/severity.txt b/test/exactly/log/c/expected/severity.txt new file mode 100644 index 00000000..aeddc3a0 --- /dev/null +++ b/test/exactly/log/c/expected/severity.txt @@ -0,0 +1,3 @@ +[WARN] This is a warning. They indicate some possible malfunction or misconfiguration in the test. +[ERR ] This is an error. They indicate serious problems and are usually shown before the test is aborted. +[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0 diff --git a/test/exactly/log/c/expected/verbose.txt b/test/exactly/log/c/expected/verbose.txt new file mode 100644 index 00000000..083c0ef2 --- /dev/null +++ b/test/exactly/log/c/expected/verbose.txt @@ -0,0 +1,11 @@ +[----] Criterion +[====] Running 2 tests from logging: +[RUN ] logging::format +[----] Log messages are printf-formatted strings. +[PASS] logging::format +[RUN ] logging::simple +[----] This is an informational message. They are not displayed by default. +[WARN] This is a warning. They indicate some possible malfunction or misconfiguration in the test. +[ERR ] This is an error. They indicate serious problems and are usually shown before the test is aborted. +[PASS] logging::simple +[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0 diff --git a/test/exactly/log/c/severity.case b/test/exactly/log/c/severity.case new file mode 100644 index 00000000..81de57d0 --- /dev/null +++ b/test/exactly/log/c/severity.case @@ -0,0 +1,7 @@ +[act] + +% log.c.bin + +[assert] + +stderr equals -contents-of expected/severity.txt diff --git a/test/exactly/log/c/verbose.case b/test/exactly/log/c/verbose.case new file mode 100644 index 00000000..54a9247f --- /dev/null +++ b/test/exactly/log/c/verbose.case @@ -0,0 +1,9 @@ +[act] + +% log.c.bin --verbose + +[assert] + +stderr + -transformed-by replace_version + equals -contents-of expected/verbose.txt diff --git a/test/exactly/log/common.xly b/test/exactly/log/common.xly new file mode 100644 index 00000000..ea797dbe --- /dev/null +++ b/test/exactly/log/common.xly @@ -0,0 +1,9 @@ +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/log/cxx.suite b/test/exactly/log/cxx.suite new file mode 100644 index 00000000..874a9fbb --- /dev/null +++ b/test/exactly/log/cxx.suite @@ -0,0 +1,3 @@ +[suites] + +cxx diff --git a/test/exactly/log/cxx/exactly.suite b/test/exactly/log/cxx/exactly.suite new file mode 100644 index 00000000..b93e2dc4 --- /dev/null +++ b/test/exactly/log/cxx/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stdout is-empty diff --git a/test/exactly/log/cxx/expected/severity.txt b/test/exactly/log/cxx/expected/severity.txt new file mode 100644 index 00000000..9c6e1363 --- /dev/null +++ b/test/exactly/log/cxx/expected/severity.txt @@ -0,0 +1,3 @@ +[WARN] This is a warning. They indicate some possible malfunction or misconfiguration in the test. +[ERR ] This is an error. They indicate serious problems and are usually shown before the test is aborted. +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 diff --git a/test/exactly/log/cxx/expected/verbose.txt b/test/exactly/log/cxx/expected/verbose.txt new file mode 100644 index 00000000..39ef3e0b --- /dev/null +++ b/test/exactly/log/cxx/expected/verbose.txt @@ -0,0 +1,8 @@ +[----] Criterion +[====] Running 1 test from logging: +[RUN ] logging::stream +[----] This is an informational message. They are not displayed by default. +[WARN] This is a warning. They indicate some possible malfunction or misconfiguration in the test. +[ERR ] This is an error. They indicate serious problems and are usually shown before the test is aborted. +[PASS] logging::stream +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 diff --git a/test/exactly/log/cxx/severity.case b/test/exactly/log/cxx/severity.case new file mode 100644 index 00000000..300e74e4 --- /dev/null +++ b/test/exactly/log/cxx/severity.case @@ -0,0 +1,9 @@ +[act] + +% log.cc.bin + +[assert] + +stderr + -transformed-by replace_version + equals -contents-of expected/severity.txt diff --git a/test/exactly/log/cxx/verbose.case b/test/exactly/log/cxx/verbose.case new file mode 100644 index 00000000..a377f6f5 --- /dev/null +++ b/test/exactly/log/cxx/verbose.case @@ -0,0 +1,9 @@ +[act] + +% log.cc.bin --verbose + +[assert] + +stderr + -transformed-by replace_version + equals -contents-of expected/verbose.txt diff --git a/test/exactly/log/exactly.suite b/test/exactly/log/exactly.suite new file mode 100644 index 00000000..b56d011a --- /dev/null +++ b/test/exactly/log/exactly.suite @@ -0,0 +1,4 @@ +[suites] + +c +cxx diff --git a/test/exactly/meson.build b/test/exactly/meson.build new file mode 100644 index 00000000..082bfe97 --- /dev/null +++ b/test/exactly/meson.build @@ -0,0 +1,19 @@ +exactly = find_program('exactly', required: false) +if exactly.found() + env = environment() + env.prepend('PATH', + join_paths(meson.current_build_dir(), '..', '..', 'samples'), + join_paths(meson.current_build_dir(), '..', 'full')) + + suite_file = 'c.suite' + if has_cxx + suite_file = 'exactly.suite' + endif + suite_path = join_paths(meson.current_source_dir(), suite_file) + + test('exactly tests', exactly, + timeout: 240, + args: ['suite', suite_path], + env: env) +endif + diff --git a/test/exactly/parametrized/c.suite b/test/exactly/parametrized/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/parametrized/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/parametrized/c/exactly.suite b/test/exactly/parametrized/c/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/parametrized/c/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/parametrized/c/expected/list.txt b/test/exactly/parametrized/c/expected/list.txt new file mode 100644 index 00000000..88785042 --- /dev/null +++ b/test/exactly/parametrized/c/expected/list.txt @@ -0,0 +1,5 @@ +params: 4 tests +|-- cleanup +|-- multiple +|-- simple +`-- string diff --git a/test/exactly/parametrized/c/expected/plain.txt b/test/exactly/parametrized/c/expected/plain.txt new file mode 100644 index 00000000..a469fc7c --- /dev/null +++ b/test/exactly/parametrized/c/expected/plain.txt @@ -0,0 +1,56 @@ +[----] parameterized.c:79: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::cleanup +[----] parameterized.c:79: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::cleanup +[----] parameterized.c:79: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::cleanup +[----] parameterized.c:35: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::multiple +[----] parameterized.c:35: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::multiple +[----] parameterized.c:35: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::multiple +[----] parameterized.c:14: Assertion Failed +[----] +[----] Parameter: 1 +[----] +[FAIL] params::simple +[----] parameterized.c:14: Assertion Failed +[----] +[----] Parameter: 2 +[----] +[FAIL] params::simple +[----] parameterized.c:14: Assertion Failed +[----] +[----] Parameter: 3 +[----] +[FAIL] params::simple +[----] parameterized.c:113: Assertion Failed +[----] +[----] string: Hello +[----] +[FAIL] params::string +[----] parameterized.c:113: Assertion Failed +[----] +[----] string: World +[----] +[FAIL] params::string +[====] Synthesis: Tested: 11 | Passing: 0 | Failing: 11 | Crashing: 0 diff --git a/test/exactly/parametrized/c/expected/verbose.txt b/test/exactly/parametrized/c/expected/verbose.txt new file mode 100644 index 00000000..9a03368a --- /dev/null +++ b/test/exactly/parametrized/c/expected/verbose.txt @@ -0,0 +1,69 @@ +[----] Criterion +[====] Running 4 tests from params: +[RUN ] params::cleanup +[----] parameterized.c:79: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::cleanup +[RUN ] params::cleanup +[----] parameterized.c:79: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::cleanup +[RUN ] params::cleanup +[----] parameterized.c:79: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::cleanup +[RUN ] params::multiple +[----] parameterized.c:35: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::multiple +[RUN ] params::multiple +[----] parameterized.c:35: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::multiple +[RUN ] params::multiple +[----] parameterized.c:35: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::multiple +[RUN ] params::simple +[----] parameterized.c:14: Assertion Failed +[----] +[----] Parameter: 1 +[----] +[FAIL] params::simple +[RUN ] params::simple +[----] parameterized.c:14: Assertion Failed +[----] +[----] Parameter: 2 +[----] +[FAIL] params::simple +[RUN ] params::simple +[----] parameterized.c:14: Assertion Failed +[----] +[----] Parameter: 3 +[----] +[FAIL] params::simple +[RUN ] params::string +[----] parameterized.c:113: Assertion Failed +[----] +[----] string: Hello +[----] +[FAIL] params::string +[RUN ] params::string +[----] parameterized.c:113: Assertion Failed +[----] +[----] string: World +[----] +[FAIL] params::string +[====] Synthesis: Tested: 11 | Passing: 0 | Failing: 11 | Crashing: 0 diff --git a/test/exactly/parametrized/c/list.case b/test/exactly/parametrized/c/list.case new file mode 100644 index 00000000..0486497c --- /dev/null +++ b/test/exactly/parametrized/c/list.case @@ -0,0 +1,11 @@ +[act] + +% parameterized.c.bin --list + +[assert] + +`Parameterized tests should be listed as one test` + +stdout equals -contents-of expected/list.txt + +stderr is-empty diff --git a/test/exactly/parametrized/c/plain.case b/test/exactly/parametrized/c/plain.case new file mode 100644 index 00000000..f32eb86d --- /dev/null +++ b/test/exactly/parametrized/c/plain.case @@ -0,0 +1,9 @@ +[act] + +% parameterized.c.bin + +[assert] + +stdout is-empty + +stderr equals -contents-of expected/plain.txt diff --git a/test/exactly/parametrized/c/verbose.case b/test/exactly/parametrized/c/verbose.case new file mode 100644 index 00000000..e4faeb6d --- /dev/null +++ b/test/exactly/parametrized/c/verbose.case @@ -0,0 +1,11 @@ +[act] + +% parameterized.c.bin --verbose + +[assert] + +stdout is-empty + +stderr + -transformed-by replace_version + equals -contents-of expected/verbose.txt diff --git a/test/exactly/parametrized/common.xly b/test/exactly/parametrized/common.xly new file mode 100644 index 00000000..c87e398a --- /dev/null +++ b/test/exactly/parametrized/common.xly @@ -0,0 +1,7 @@ +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 diff --git a/test/exactly/parametrized/cxx.suite b/test/exactly/parametrized/cxx.suite new file mode 100644 index 00000000..874a9fbb --- /dev/null +++ b/test/exactly/parametrized/cxx.suite @@ -0,0 +1,3 @@ +[suites] + +cxx diff --git a/test/exactly/parametrized/cxx/exactly.suite b/test/exactly/parametrized/cxx/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/parametrized/cxx/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/parametrized/cxx/expected/list.txt b/test/exactly/parametrized/cxx/expected/list.txt new file mode 100644 index 00000000..88785042 --- /dev/null +++ b/test/exactly/parametrized/cxx/expected/list.txt @@ -0,0 +1,5 @@ +params: 4 tests +|-- cleanup +|-- multiple +|-- simple +`-- string diff --git a/test/exactly/parametrized/cxx/expected/plain.txt b/test/exactly/parametrized/cxx/expected/plain.txt new file mode 100644 index 00000000..519813a0 --- /dev/null +++ b/test/exactly/parametrized/cxx/expected/plain.txt @@ -0,0 +1,56 @@ +[----] parameterized.cc:72: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::cleanup +[----] parameterized.cc:72: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::cleanup +[----] parameterized.cc:72: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::cleanup +[----] parameterized.cc:34: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::multiple +[----] parameterized.cc:34: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::multiple +[----] parameterized.cc:34: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::multiple +[----] parameterized.cc:13: Assertion Failed +[----] +[----] Parameter: 1 +[----] +[FAIL] params::simple +[----] parameterized.cc:13: Assertion Failed +[----] +[----] Parameter: 2 +[----] +[FAIL] params::simple +[----] parameterized.cc:13: Assertion Failed +[----] +[----] Parameter: 3 +[----] +[FAIL] params::simple +[----] parameterized.cc:92: Assertion Failed +[----] +[----] string: Hello +[----] +[FAIL] params::string +[----] parameterized.cc:92: Assertion Failed +[----] +[----] string: World +[----] +[FAIL] params::string +[====] Synthesis: Tested: 11 | Passing: 0 | Failing: 11 | Crashing: 0 diff --git a/test/exactly/parametrized/cxx/expected/verbose.txt b/test/exactly/parametrized/cxx/expected/verbose.txt new file mode 100644 index 00000000..bd11bbda --- /dev/null +++ b/test/exactly/parametrized/cxx/expected/verbose.txt @@ -0,0 +1,69 @@ +[----] Criterion +[====] Running 4 tests from params: +[RUN ] params::cleanup +[----] parameterized.cc:72: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::cleanup +[RUN ] params::cleanup +[----] parameterized.cc:72: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::cleanup +[RUN ] params::cleanup +[----] parameterized.cc:72: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::cleanup +[RUN ] params::multiple +[----] parameterized.cc:34: Assertion Failed +[----] +[----] Parameters: (1, 2.000000) +[----] +[FAIL] params::multiple +[RUN ] params::multiple +[----] parameterized.cc:34: Assertion Failed +[----] +[----] Parameters: (3, 4.000000) +[----] +[FAIL] params::multiple +[RUN ] params::multiple +[----] parameterized.cc:34: Assertion Failed +[----] +[----] Parameters: (5, 6.000000) +[----] +[FAIL] params::multiple +[RUN ] params::simple +[----] parameterized.cc:13: Assertion Failed +[----] +[----] Parameter: 1 +[----] +[FAIL] params::simple +[RUN ] params::simple +[----] parameterized.cc:13: Assertion Failed +[----] +[----] Parameter: 2 +[----] +[FAIL] params::simple +[RUN ] params::simple +[----] parameterized.cc:13: Assertion Failed +[----] +[----] Parameter: 3 +[----] +[FAIL] params::simple +[RUN ] params::string +[----] parameterized.cc:92: Assertion Failed +[----] +[----] string: Hello +[----] +[FAIL] params::string +[RUN ] params::string +[----] parameterized.cc:92: Assertion Failed +[----] +[----] string: World +[----] +[FAIL] params::string +[====] Synthesis: Tested: 11 | Passing: 0 | Failing: 11 | Crashing: 0 diff --git a/test/exactly/parametrized/cxx/list.case b/test/exactly/parametrized/cxx/list.case new file mode 100644 index 00000000..39eee58d --- /dev/null +++ b/test/exactly/parametrized/cxx/list.case @@ -0,0 +1,11 @@ +[act] + +% parameterized.cc.bin --list + +[assert] + +`Parameterized tests should be listed as one test` + +stdout equals -contents-of expected/list.txt + +stderr is-empty diff --git a/test/exactly/parametrized/cxx/plain.case b/test/exactly/parametrized/cxx/plain.case new file mode 100644 index 00000000..64e4e740 --- /dev/null +++ b/test/exactly/parametrized/cxx/plain.case @@ -0,0 +1,9 @@ +[act] + +% parameterized.cc.bin + +[assert] + +stdout is-empty + +stderr equals -contents-of expected/plain.txt diff --git a/test/exactly/parametrized/cxx/verbose.case b/test/exactly/parametrized/cxx/verbose.case new file mode 100644 index 00000000..0bd5f0a5 --- /dev/null +++ b/test/exactly/parametrized/cxx/verbose.case @@ -0,0 +1,11 @@ +[act] + +% parameterized.cc.bin --verbose + +[assert] + +stdout is-empty + +stderr + -transformed-by replace_version + equals -contents-of expected/verbose.txt diff --git a/test/exactly/parametrized/exactly.suite b/test/exactly/parametrized/exactly.suite new file mode 100644 index 00000000..b56d011a --- /dev/null +++ b/test/exactly/parametrized/exactly.suite @@ -0,0 +1,4 @@ +[suites] + +c +cxx diff --git a/test/exactly/pattern/character-classes/1.case b/test/exactly/pattern/character-classes/1.case new file mode 100644 index 00000000..031c4d96 --- /dev/null +++ b/test/exactly/pattern/character-classes/1.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='misc/[pf]assing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/character-classes/2.case b/test/exactly/pattern/character-classes/2.case new file mode 100644 index 00000000..d5addbca --- /dev/null +++ b/test/exactly/pattern/character-classes/2.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='misc/[p-s]assing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/character-classes/3.case b/test/exactly/pattern/character-classes/3.case new file mode 100644 index 00000000..0e445c05 --- /dev/null +++ b/test/exactly/pattern/character-classes/3.case @@ -0,0 +1,16 @@ +[act] + +% simple.c.bin --filter='misc/[!f]ailing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[SKIP] misc::passing: Test is disabled +[====] Synthesis: Tested: 0 | Passing: 0 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/character-classes/4.case b/test/exactly/pattern/character-classes/4.case new file mode 100644 index 00000000..5cb957e8 --- /dev/null +++ b/test/exactly/pattern/character-classes/4.case @@ -0,0 +1,21 @@ +[act] + +% simple.c.bin --filter='misc/[p-f]ailing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[SKIP] misc::passing: Test is disabled +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/character-classes/5.case b/test/exactly/pattern/character-classes/5.case new file mode 100644 index 00000000..a0fa895d --- /dev/null +++ b/test/exactly/pattern/character-classes/5.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='misc/[!azerty]assing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/character-classes/exactly.suite b/test/exactly/pattern/character-classes/exactly.suite new file mode 100644 index 00000000..f27f63f2 --- /dev/null +++ b/test/exactly/pattern/character-classes/exactly.suite @@ -0,0 +1,11 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 diff --git a/test/exactly/pattern/common.xly b/test/exactly/pattern/common.xly new file mode 100644 index 00000000..aea12dff --- /dev/null +++ b/test/exactly/pattern/common.xly @@ -0,0 +1,7 @@ +[setup] + +including ../common.xly + +[assert] + +stdout is-empty diff --git a/test/exactly/pattern/exactly.suite b/test/exactly/pattern/exactly.suite new file mode 100644 index 00000000..742dde57 --- /dev/null +++ b/test/exactly/pattern/exactly.suite @@ -0,0 +1,8 @@ +[suites] + +normal-globs +character-classes +extended-globs +nested-extglob-patterns +malformed-pattern +special-char-escaping diff --git a/test/exactly/pattern/extended-globs/1.case b/test/exactly/pattern/extended-globs/1.case new file mode 100644 index 00000000..b210d4f3 --- /dev/null +++ b/test/exactly/pattern/extended-globs/1.case @@ -0,0 +1,21 @@ +[act] + +% simple.c.bin --filter='!(*/passing)' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[SKIP] misc::passing: Test is disabled +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/2.case b/test/exactly/pattern/extended-globs/2.case new file mode 100644 index 00000000..8a57031f --- /dev/null +++ b/test/exactly/pattern/extended-globs/2.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='@(misc)/passing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/3.case b/test/exactly/pattern/extended-globs/3.case new file mode 100644 index 00000000..3713ce8f --- /dev/null +++ b/test/exactly/pattern/extended-globs/3.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='*(misc)/pa+(s)ing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/4.case b/test/exactly/pattern/extended-globs/4.case new file mode 100644 index 00000000..c25347eb --- /dev/null +++ b/test/exactly/pattern/extended-globs/4.case @@ -0,0 +1,22 @@ +[act] + +% simple.c.bin --filter='misc/[pf]a@(ss|il)ing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/5.case b/test/exactly/pattern/extended-globs/5.case new file mode 100644 index 00000000..a00595e3 --- /dev/null +++ b/test/exactly/pattern/extended-globs/5.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='!(*/failing|*/fail)' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/6.case b/test/exactly/pattern/extended-globs/6.case new file mode 100644 index 00000000..bbcca0bb --- /dev/null +++ b/test/exactly/pattern/extended-globs/6.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='misc/?(passing)' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/7.case b/test/exactly/pattern/extended-globs/7.case new file mode 100644 index 00000000..cd585f64 --- /dev/null +++ b/test/exactly/pattern/extended-globs/7.case @@ -0,0 +1,22 @@ +[act] + +% simple.c.bin --filter='misc/?(passing|failing)' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/8.case b/test/exactly/pattern/extended-globs/8.case new file mode 100644 index 00000000..cd6fed80 --- /dev/null +++ b/test/exactly/pattern/extended-globs/8.case @@ -0,0 +1,22 @@ +[act] + +% simple.c.bin --filter='misc/*!(passing)' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/extended-globs/exactly.suite b/test/exactly/pattern/extended-globs/exactly.suite new file mode 100644 index 00000000..f27f63f2 --- /dev/null +++ b/test/exactly/pattern/extended-globs/exactly.suite @@ -0,0 +1,11 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 diff --git a/test/exactly/pattern/malformed-pattern/1.case b/test/exactly/pattern/malformed-pattern/1.case new file mode 100644 index 00000000..b77950f8 --- /dev/null +++ b/test/exactly/pattern/malformed-pattern/1.case @@ -0,0 +1,10 @@ +[act] + +% simple.c.bin --filter='?(malformed' --verbose + +[assert] + +stderr equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[SKIP] misc::passing: Test is disabled +[====] Synthesis: Tested: 0 | Passing: 0 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/nested-extglob-patterns/2.case b/test/exactly/pattern/nested-extglob-patterns/2.case new file mode 100644 index 00000000..ba6d9074 --- /dev/null +++ b/test/exactly/pattern/nested-extglob-patterns/2.case @@ -0,0 +1,16 @@ +[act] + +% simple.c.bin --filter='?(*(a|b))' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[SKIP] misc::passing: Test is disabled +[====] Synthesis: Tested: 0 | Passing: 0 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/nested-extglob-patterns/3.case b/test/exactly/pattern/nested-extglob-patterns/3.case new file mode 100644 index 00000000..a1562d5e --- /dev/null +++ b/test/exactly/pattern/nested-extglob-patterns/3.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='misc/pa@(s|*(s))ing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/nested-extglob-patterns/exactly.suite b/test/exactly/pattern/nested-extglob-patterns/exactly.suite new file mode 100644 index 00000000..f27f63f2 --- /dev/null +++ b/test/exactly/pattern/nested-extglob-patterns/exactly.suite @@ -0,0 +1,11 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 diff --git a/test/exactly/pattern/normal-globs/1.case b/test/exactly/pattern/normal-globs/1.case new file mode 100644 index 00000000..d40418c5 --- /dev/null +++ b/test/exactly/pattern/normal-globs/1.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='*/passing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/normal-globs/2.case b/test/exactly/pattern/normal-globs/2.case new file mode 100644 index 00000000..bd04d686 --- /dev/null +++ b/test/exactly/pattern/normal-globs/2.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='*/pa?sing' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/normal-globs/3.case b/test/exactly/pattern/normal-globs/3.case new file mode 100644 index 00000000..a5ae9920 --- /dev/null +++ b/test/exactly/pattern/normal-globs/3.case @@ -0,0 +1,17 @@ +[act] + +% simple.c.bin --filter='misc/passing?' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/normal-globs/4.case b/test/exactly/pattern/normal-globs/4.case new file mode 100644 index 00000000..1bb9fb39 --- /dev/null +++ b/test/exactly/pattern/normal-globs/4.case @@ -0,0 +1,22 @@ +[act] + +% simple.c.bin --filter='misc/*' --verbose + +[assert] + +stderr + -transformed-by replace_version + equals +< +[====] Running 2 tests from misc: +[RUN ] misc::failing +[----] simple.c:4: Assertion Failed +[----] +[----] The expression 0 is false. +[----] +[FAIL] misc::failing +[RUN ] misc::passing +[PASS] misc::passing +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/normal-globs/exactly.suite b/test/exactly/pattern/normal-globs/exactly.suite new file mode 100644 index 00000000..f27f63f2 --- /dev/null +++ b/test/exactly/pattern/normal-globs/exactly.suite @@ -0,0 +1,11 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 diff --git a/test/exactly/pattern/special-char-escaping/1.case b/test/exactly/pattern/special-char-escaping/1.case new file mode 100644 index 00000000..961e4abc --- /dev/null +++ b/test/exactly/pattern/special-char-escaping/1.case @@ -0,0 +1,12 @@ +[act] + +% simple.c.bin --filter='\!(escaped)' --verbose + +[assert] + +exit-code == 3 + +stderr equals +< +[====] Running 2 tests from misc: +[SKIP] misc::failing: Test is disabled +[SKIP] misc::passing: Test is disabled +[====] Synthesis: Tested: 0 | Passing: 0 | Failing: 0 | Crashing: 0 +EOF diff --git a/test/exactly/pattern/special-char-escaping/exactly.suite b/test/exactly/pattern/special-char-escaping/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/pattern/special-char-escaping/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/prj-utils/extract-expected.awk b/test/exactly/prj-utils/extract-expected.awk new file mode 100644 index 00000000..5f90bffa --- /dev/null +++ b/test/exactly/prj-utils/extract-expected.awk @@ -0,0 +1,17 @@ +BEGIN { + found = 0 +} + +/^<"); + maybe_write_test(); + next; +} + +END { + print("EOF"); + maybe_write_test(); +} + +$0 ~ /^ / { + body_lines[body_line_idx++] = substr($0,3); + next; +} + +function maybe_write_test() { + if (atc != "") { + printf("WRITING test %3s/ %2d %s\n", output_dir, case_num, atc); + print " Description: " description + file_name = output_dir "/" case_num".case" + print test_case() > file_name + close(file_name) + atc = "" + } +} + +function test_case_code() { + delete lines; + lines[0] = "[act]" + lines[1] = "" + lines[2] = "@[" bin_dir_symbol "]@/" atc + lines[3] = "" + lines[4] = "[assert]" + lines[5] = "" + lines[6] = assertion_header() + lines[7] = "<:' + +stdout + -transformed-by ( replace_version | replace_source_file ) + equals -contents-of expected/description.txt diff --git a/test/exactly/reporting/json/c/exactly.suite b/test/exactly/reporting/json/c/exactly.suite new file mode 100644 index 00000000..34f1e68e --- /dev/null +++ b/test/exactly/reporting/json/c/exactly.suite @@ -0,0 +1,10 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +def text-transformer replace_simple_source_file = + replace '"simple.c:' '":' diff --git a/test/exactly/reporting/json/c/long-messages--json-option.case b/test/exactly/reporting/json/c/long-messages--json-option.case new file mode 100644 index 00000000..278f5e48 --- /dev/null +++ b/test/exactly/reporting/json/c/long-messages--json-option.case @@ -0,0 +1,12 @@ +[act] + +% long-messages.c.bin --json + +[assert] + +def text-transformer replace_source_file = + replace '"long-messages.c:' '":' + +stdout + -transformed-by ( replace_version | replace_source_file ) + equals -contents-of expected/long-messages.txt diff --git a/test/exactly/reporting/json/c/more-suites--json-option.case b/test/exactly/reporting/json/c/more-suites--json-option.case new file mode 100644 index 00000000..b68b94c3 --- /dev/null +++ b/test/exactly/reporting/json/c/more-suites--json-option.case @@ -0,0 +1,9 @@ +[act] + +% more-suites.c.bin --json + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/more-suites.txt diff --git a/test/exactly/reporting/json/c/signal--json-option.case b/test/exactly/reporting/json/c/signal--json-option.case new file mode 100644 index 00000000..db2e4842 --- /dev/null +++ b/test/exactly/reporting/json/c/signal--json-option.case @@ -0,0 +1,9 @@ +[act] + +% signal.c.bin --json + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/signal.txt diff --git a/test/exactly/reporting/json/c/simple--criterion_outputs.case b/test/exactly/reporting/json/c/simple--criterion_outputs.case new file mode 100644 index 00000000..a1aedc80 --- /dev/null +++ b/test/exactly/reporting/json/c/simple--criterion_outputs.case @@ -0,0 +1,13 @@ +[setup] + +env -of act CRITERION_OUTPUTS = json:- + +[act] + +% simple.c.bin + +[assert] + +stdout + -transformed-by ( replace_version | replace_simple_source_file ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/json/c/simple--json-option.case b/test/exactly/reporting/json/c/simple--json-option.case new file mode 100644 index 00000000..bc44bd18 --- /dev/null +++ b/test/exactly/reporting/json/c/simple--json-option.case @@ -0,0 +1,9 @@ +[act] + +% simple.c.bin --json + +[assert] + +stdout + -transformed-by ( replace_version | replace_simple_source_file ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/json/c/simple--output-option.case b/test/exactly/reporting/json/c/simple--output-option.case new file mode 100644 index 00000000..9910d7d4 --- /dev/null +++ b/test/exactly/reporting/json/c/simple--output-option.case @@ -0,0 +1,9 @@ +[act] + +% simple.c.bin --output=json:- + +[assert] + +stdout + -transformed-by ( replace_version | replace_simple_source_file ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/json/c/skip--json-option.case b/test/exactly/reporting/json/c/skip--json-option.case new file mode 100644 index 00000000..2f00a01a --- /dev/null +++ b/test/exactly/reporting/json/c/skip--json-option.case @@ -0,0 +1,9 @@ +[act] + +% skip.c.bin --json + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/skip.txt diff --git a/test/exactly/reporting/json/common.xly b/test/exactly/reporting/json/common.xly new file mode 100644 index 00000000..773f095e --- /dev/null +++ b/test/exactly/reporting/json/common.xly @@ -0,0 +1,13 @@ +[conf] + +home = . + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stderr is-empty diff --git a/test/exactly/reporting/json/cxx.suite b/test/exactly/reporting/json/cxx.suite new file mode 100644 index 00000000..874a9fbb --- /dev/null +++ b/test/exactly/reporting/json/cxx.suite @@ -0,0 +1,3 @@ +[suites] + +cxx diff --git a/test/exactly/reporting/json/cxx/asserts--json-option.case b/test/exactly/reporting/json/cxx/asserts--json-option.case new file mode 100644 index 00000000..ce058b85 --- /dev/null +++ b/test/exactly/reporting/json/cxx/asserts--json-option.case @@ -0,0 +1,9 @@ +[act] + +% asserts.cc.bin --json + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/asserts-cxx.txt diff --git a/test/exactly/reporting/json/cxx/description--json-option.case b/test/exactly/reporting/json/cxx/description--json-option.case new file mode 100644 index 00000000..9b53dcd9 --- /dev/null +++ b/test/exactly/reporting/json/cxx/description--json-option.case @@ -0,0 +1,12 @@ +[act] + +% description.cc.bin --json + +[assert] + +def text-transformer replace_source_file = + replace '"description.cc:' '":' + +stdout + -transformed-by ( replace_version | replace_source_file ) + equals -contents-of expected/description.txt diff --git a/test/exactly/reporting/json/cxx/exactly.suite b/test/exactly/reporting/json/cxx/exactly.suite new file mode 100644 index 00000000..3d1336a7 --- /dev/null +++ b/test/exactly/reporting/json/cxx/exactly.suite @@ -0,0 +1,10 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +def text-transformer replace_simple_source_file = + replace '"simple.cc:' '":' diff --git a/test/exactly/reporting/json/cxx/long-messages--json-option.case b/test/exactly/reporting/json/cxx/long-messages--json-option.case new file mode 100644 index 00000000..be52cfc5 --- /dev/null +++ b/test/exactly/reporting/json/cxx/long-messages--json-option.case @@ -0,0 +1,12 @@ +[act] + +% long-messages.cc.bin --json + +[assert] + +def text-transformer replace_source_file = + replace '"long-messages.cc:' '":' + +stdout + -transformed-by ( replace_version | replace_source_file ) + equals -contents-of expected/long-messages.txt diff --git a/test/exactly/reporting/json/cxx/more-suites--json-option.case b/test/exactly/reporting/json/cxx/more-suites--json-option.case new file mode 100644 index 00000000..2cc0b6b0 --- /dev/null +++ b/test/exactly/reporting/json/cxx/more-suites--json-option.case @@ -0,0 +1,9 @@ +[act] + +% more-suites.cc.bin --json + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/more-suites.txt diff --git a/test/exactly/reporting/json/cxx/signal--json-option.case b/test/exactly/reporting/json/cxx/signal--json-option.case new file mode 100644 index 00000000..56c7089b --- /dev/null +++ b/test/exactly/reporting/json/cxx/signal--json-option.case @@ -0,0 +1,9 @@ +[act] + +% signal.cc.bin --json + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/signal.txt diff --git a/test/exactly/reporting/json/cxx/simple--criterion_outputs.case b/test/exactly/reporting/json/cxx/simple--criterion_outputs.case new file mode 100644 index 00000000..3aebd675 --- /dev/null +++ b/test/exactly/reporting/json/cxx/simple--criterion_outputs.case @@ -0,0 +1,13 @@ +[setup] + +env -of act CRITERION_OUTPUTS = json:- + +[act] + +% simple.cc.bin + +[assert] + +stdout + -transformed-by ( replace_version | replace_simple_source_file ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/json/cxx/simple--json-option.case b/test/exactly/reporting/json/cxx/simple--json-option.case new file mode 100644 index 00000000..32c4d83b --- /dev/null +++ b/test/exactly/reporting/json/cxx/simple--json-option.case @@ -0,0 +1,9 @@ +[act] + +% simple.cc.bin --json + +[assert] + +stdout + -transformed-by ( replace_version | replace_simple_source_file ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/json/cxx/simple--output-option.case b/test/exactly/reporting/json/cxx/simple--output-option.case new file mode 100644 index 00000000..2c0609f7 --- /dev/null +++ b/test/exactly/reporting/json/cxx/simple--output-option.case @@ -0,0 +1,9 @@ +[act] + +% simple.cc.bin --output=json:- + +[assert] + +stdout + -transformed-by ( replace_version | replace_simple_source_file ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/json/cxx/skip--json-option.case b/test/exactly/reporting/json/cxx/skip--json-option.case new file mode 100644 index 00000000..0e50a6d9 --- /dev/null +++ b/test/exactly/reporting/json/cxx/skip--json-option.case @@ -0,0 +1,9 @@ +[act] + +% skip.cc.bin --json + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/skip.txt diff --git a/test/exactly/reporting/json/exactly.suite b/test/exactly/reporting/json/exactly.suite new file mode 100644 index 00000000..b56d011a --- /dev/null +++ b/test/exactly/reporting/json/exactly.suite @@ -0,0 +1,4 @@ +[suites] + +c +cxx diff --git a/test/exactly/reporting/json/expected/asserts-c.txt b/test/exactly/reporting/json/expected/asserts-c.txt new file mode 100644 index 00000000..b3211b0c --- /dev/null +++ b/test/exactly/reporting/json/expected/asserts-c.txt @@ -0,0 +1,66 @@ +{ + "id": "Criterion ", + "passed": 6, + "failed": 2, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "asserts", + "passed": 6, + "failed": 2, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "wstring", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "string", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "stream", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "old_school", + "assertions": 2, + "status": "FAILED", + "messages": [ + "asserts.c:19: (null)", + "asserts.c:18: You can fail an assertion with a message from anywhere" + ] + }, + { + "name": "native", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "float", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "base", + "assertions": 2, + "status": "FAILED", + "messages": [ + "asserts.c:13: This assert runs", + "asserts.c:12: assert is fatal, expect isn't" + ] + }, + { + "name": "array", + "assertions": 0, + "status": "PASSED" + } + ] + } + ] +} diff --git a/test/exactly/reporting/json/expected/asserts-cxx.txt b/test/exactly/reporting/json/expected/asserts-cxx.txt new file mode 100644 index 00000000..a00e89e6 --- /dev/null +++ b/test/exactly/reporting/json/expected/asserts-cxx.txt @@ -0,0 +1,75 @@ +{ + "id": "Criterion ", + "passed": 6, + "failed": 3, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "asserts", + "passed": 6, + "failed": 3, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "wstring", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "string", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "stream", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "old_school", + "assertions": 2, + "status": "FAILED", + "messages": [ + "asserts.cc:22: (null)", + "asserts.cc:21: You can fail an assertion with a message from anywhere" + ] + }, + { + "name": "native", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "float", + "assertions": 0, + "status": "PASSED" + }, + { + "name": "exception", + "assertions": 2, + "status": "FAILED", + "messages": [ + "asserts.cc:169: (null)", + "asserts.cc:167: (null)" + ] + }, + { + "name": "base", + "assertions": 2, + "status": "FAILED", + "messages": [ + "asserts.cc:16: This assert runs", + "asserts.cc:15: assert is fatal, expect isn't" + ] + }, + { + "name": "array", + "assertions": 0, + "status": "PASSED" + } + ] + } + ] +} diff --git a/test/exactly/reporting/json/expected/description.txt b/test/exactly/reporting/json/expected/description.txt new file mode 100644 index 00000000..d2efcd4d --- /dev/null +++ b/test/exactly/reporting/json/expected/description.txt @@ -0,0 +1,32 @@ +{ + "id": "Criterion ", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 1, + "test_suites": [ + { + "name": "misc", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 1, + "tests": [ + { + "name": "skipped", + "assertions": 0, + "status": "SKIPPED", + "messages": ["The test was skipped."] + }, + { + "name": "failing", + "assertions": 1, + "status": "FAILED", + "messages": [ + ":4: The expression 0 is false." + ] + } + ] + } + ] +} diff --git a/test/exactly/reporting/json/expected/long-messages.txt b/test/exactly/reporting/json/expected/long-messages.txt new file mode 100644 index 00000000..de106293 --- /dev/null +++ b/test/exactly/reporting/json/expected/long-messages.txt @@ -0,0 +1,29 @@ +{ + "id": "Criterion ", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "sample", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "long_msg", + "assertions": 1, + "status": "FAILED", + "messages": [ + ":4: This is", + " A long message", + " Spawning multiple lines.", + " Formatting is respected." + ] + } + ] + } + ] +} diff --git a/test/exactly/reporting/json/expected/more-suites.txt b/test/exactly/reporting/json/expected/more-suites.txt new file mode 100644 index 00000000..a5933aa1 --- /dev/null +++ b/test/exactly/reporting/json/expected/more-suites.txt @@ -0,0 +1,52 @@ +{ + "id": "Criterion ", + "passed": 2, + "failed": 0, + "errored": 0, + "skipped": 1, + "test_suites": [ + { + "name": "suite2", + "passed": 1, + "failed": 0, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "test", + "assertions": 1, + "status": "PASSED" + } + ] + }, + { + "name": "suite1", + "passed": 1, + "failed": 0, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "test", + "assertions": 1, + "status": "PASSED" + } + ] + }, + { + "name": "disabled", + "passed": 0, + "failed": 0, + "errored": 0, + "skipped": 1, + "tests": [ + { + "name": "test", + "assertions": 0, + "status": "SKIPPED", + "messages": ["The test was skipped."] + } + ] + } + ] +} diff --git a/test/exactly/reporting/json/expected/signal.txt b/test/exactly/reporting/json/expected/signal.txt new file mode 100644 index 00000000..e48cc04a --- /dev/null +++ b/test/exactly/reporting/json/expected/signal.txt @@ -0,0 +1,37 @@ +{ + "id": "Criterion ", + "passed": 1, + "failed": 2, + "errored": 1, + "skipped": 0, + "test_suites": [ + { + "name": "simple", + "passed": 1, + "failed": 2, + "errored": 1, + "skipped": 0, + "tests": [ + { + "name": "wrong_signal", + "assertions": 0, + "status": "FAILED", + "messages": [ + + ] + }, + { + "name": "uncaught", + "assertions": 0, + "status": "ERRORED", + "messages": ["The test crashed."] + }, + { + "name": "caught", + "assertions": 0, + "status": "PASSED" + } + ] + } + ] +} diff --git a/test/exactly/reporting/json/expected/simple.txt b/test/exactly/reporting/json/expected/simple.txt new file mode 100644 index 00000000..0bbde8b5 --- /dev/null +++ b/test/exactly/reporting/json/expected/simple.txt @@ -0,0 +1,31 @@ +{ + "id": "Criterion ", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "misc", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "passing", + "assertions": 1, + "status": "PASSED" + }, + { + "name": "failing", + "assertions": 1, + "status": "FAILED", + "messages": [ + ":4: The expression 0 is false." + ] + } + ] + } + ] +} diff --git a/test/exactly/reporting/json/expected/skip.txt b/test/exactly/reporting/json/expected/skip.txt new file mode 100644 index 00000000..d2926f9c --- /dev/null +++ b/test/exactly/reporting/json/expected/skip.txt @@ -0,0 +1,30 @@ +{ + "id": "Criterion ", + "passed": 0, + "failed": 0, + "errored": 0, + "skipped": 2, + "test_suites": [ + { + "name": "misc", + "passed": 0, + "failed": 0, + "errored": 0, + "skipped": 2, + "tests": [ + { + "name": "skipping", + "assertions": 0, + "status": "SKIPPED", + "messages": ["The test was skipped."] + }, + { + "name": "message", + "assertions": 0, + "status": "SKIPPED", + "messages": ["Skips may take printf-like messages"] + } + ] + } + ] +} diff --git a/test/exactly/reporting/tap/c.suite b/test/exactly/reporting/tap/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/reporting/tap/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/reporting/tap/c/asserts.case b/test/exactly/reporting/tap/c/asserts.case new file mode 100644 index 00000000..e3f47776 --- /dev/null +++ b/test/exactly/reporting/tap/c/asserts.case @@ -0,0 +1,9 @@ +[act] + +% asserts.c.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/asserts.txt diff --git a/test/exactly/reporting/tap/c/description.case b/test/exactly/reporting/tap/c/description.case new file mode 100644 index 00000000..35fd53b1 --- /dev/null +++ b/test/exactly/reporting/tap/c/description.case @@ -0,0 +1,9 @@ +[act] + +% description.c.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/description.txt diff --git a/test/exactly/reporting/tap/c/exactly.suite b/test/exactly/reporting/tap/c/exactly.suite new file mode 100644 index 00000000..c415324f --- /dev/null +++ b/test/exactly/reporting/tap/c/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stderr is-empty diff --git a/test/exactly/reporting/tap/c/expected/asserts.txt b/test/exactly/reporting/tap/c/expected/asserts.txt new file mode 100644 index 00000000..b99ebbcd --- /dev/null +++ b/test/exactly/reporting/tap/c/expected/asserts.txt @@ -0,0 +1,29 @@ +TAP version 13 +1..8 +# Criterion +# +# Running 8 tests from asserts +ok - asserts::wstring +ok - asserts::string +ok - asserts::stream +not ok - asserts::old_school + --- + assertions: 2 + failures: + - asserts.c:19: |+ + Assertion failed: (null) + - asserts.c:18: |+ + Assertion failed: You can fail an assertion with a message from anywhere + ... +ok - asserts::native +ok - asserts::float +not ok - asserts::base + --- + assertions: 2 + failures: + - asserts.c:13: |+ + Assertion failed: This assert runs + - asserts.c:12: |+ + Assertion failed: assert is fatal, expect isn't + ... +ok - asserts::array diff --git a/test/exactly/reporting/tap/c/expected/description.txt b/test/exactly/reporting/tap/c/expected/description.txt new file mode 100644 index 00000000..b9e3239f --- /dev/null +++ b/test/exactly/reporting/tap/c/expected/description.txt @@ -0,0 +1,13 @@ +TAP version 13 +1..2 +# Criterion +# +# Running 2 tests from misc +ok - misc::skipped This one is skipped # SKIP test was skipped +not ok - misc::failing Just a failing test + --- + assertions: 1 + failures: + - description.c:4: |+ + Assertion failed: The expression 0 is false. + ... diff --git a/test/exactly/reporting/tap/c/expected/long-messages.txt b/test/exactly/reporting/tap/c/expected/long-messages.txt new file mode 100644 index 00000000..e598995f --- /dev/null +++ b/test/exactly/reporting/tap/c/expected/long-messages.txt @@ -0,0 +1,15 @@ +TAP version 13 +1..1 +# Criterion +# +# Running 1 tests from sample +not ok - sample::long_msg + --- + assertions: 1 + failures: + - long-messages.c:4: |+ + Assertion failed: This is + A long message + Spawning multiple lines. + Formatting is respected. + ... diff --git a/test/exactly/reporting/tap/c/expected/more-suites.txt b/test/exactly/reporting/tap/c/expected/more-suites.txt new file mode 100644 index 00000000..de04bad3 --- /dev/null +++ b/test/exactly/reporting/tap/c/expected/more-suites.txt @@ -0,0 +1,12 @@ +TAP version 13 +1..3 +# Criterion +# +# Running 1 tests from suite2 +ok - suite2::test +# +# Running 1 tests from suite1 +ok - suite1::test +# +# Running 1 tests from disabled +ok - disabled::test # SKIP test was skipped diff --git a/test/exactly/reporting/tap/c/expected/signal.txt b/test/exactly/reporting/tap/c/expected/signal.txt new file mode 100644 index 00000000..ab2c4038 --- /dev/null +++ b/test/exactly/reporting/tap/c/expected/signal.txt @@ -0,0 +1,12 @@ +TAP version 13 +1..3 +# Criterion +# +# Running 3 tests from simple +not ok - simple::wrong_signal + --- + assertions: 0 + failures: + ... +not ok - simple::uncaught unexpected signal after signal.c:14 +ok - simple::caught diff --git a/test/exactly/reporting/tap/c/expected/simple.txt b/test/exactly/reporting/tap/c/expected/simple.txt new file mode 100644 index 00000000..4e225113 --- /dev/null +++ b/test/exactly/reporting/tap/c/expected/simple.txt @@ -0,0 +1,13 @@ +TAP version 13 +1..2 +# Criterion +# +# Running 2 tests from misc +ok - misc::passing +not ok - misc::failing + --- + assertions: 1 + failures: + - simple.c:4: |+ + Assertion failed: The expression 0 is false. + ... diff --git a/test/exactly/reporting/tap/c/expected/skip.txt b/test/exactly/reporting/tap/c/expected/skip.txt new file mode 100644 index 00000000..94aac004 --- /dev/null +++ b/test/exactly/reporting/tap/c/expected/skip.txt @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +# Criterion +# +# Running 2 tests from misc +ok - misc::skipping # SKIP test was skipped +ok - misc::message # SKIP Skips may take printf-like messages diff --git a/test/exactly/reporting/tap/c/long-messages.case b/test/exactly/reporting/tap/c/long-messages.case new file mode 100644 index 00000000..ca5414ee --- /dev/null +++ b/test/exactly/reporting/tap/c/long-messages.case @@ -0,0 +1,9 @@ +[act] + +% long-messages.c.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/long-messages.txt diff --git a/test/exactly/reporting/tap/c/more-suites.case b/test/exactly/reporting/tap/c/more-suites.case new file mode 100644 index 00000000..6b2b3cb6 --- /dev/null +++ b/test/exactly/reporting/tap/c/more-suites.case @@ -0,0 +1,9 @@ +[act] + +% more-suites.c.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/more-suites.txt diff --git a/test/exactly/reporting/tap/c/signal.case b/test/exactly/reporting/tap/c/signal.case new file mode 100644 index 00000000..3d8e395c --- /dev/null +++ b/test/exactly/reporting/tap/c/signal.case @@ -0,0 +1,9 @@ +[act] + +% signal.c.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/signal.txt diff --git a/test/exactly/reporting/tap/c/simple--criterion_outputs.case b/test/exactly/reporting/tap/c/simple--criterion_outputs.case new file mode 100644 index 00000000..becb6e47 --- /dev/null +++ b/test/exactly/reporting/tap/c/simple--criterion_outputs.case @@ -0,0 +1,13 @@ +[setup] + +env -of act CRITERION_OUTPUTS = tap:- + +[act] + +% simple.c.bin + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/tap/c/simple--output-option.case b/test/exactly/reporting/tap/c/simple--output-option.case new file mode 100644 index 00000000..05763bda --- /dev/null +++ b/test/exactly/reporting/tap/c/simple--output-option.case @@ -0,0 +1,11 @@ +# Testing --output=tap + +[act] + +% simple.c.bin --output=tap:- + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/tap/c/simple.case b/test/exactly/reporting/tap/c/simple.case new file mode 100644 index 00000000..9ddf44e5 --- /dev/null +++ b/test/exactly/reporting/tap/c/simple.case @@ -0,0 +1,9 @@ +[act] + +% simple.c.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/tap/c/skip.case b/test/exactly/reporting/tap/c/skip.case new file mode 100644 index 00000000..c5bdbf84 --- /dev/null +++ b/test/exactly/reporting/tap/c/skip.case @@ -0,0 +1,9 @@ +[act] + +% skip.c.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/skip.txt diff --git a/test/exactly/reporting/tap/common.xly b/test/exactly/reporting/tap/common.xly new file mode 100644 index 00000000..1b84bbb3 --- /dev/null +++ b/test/exactly/reporting/tap/common.xly @@ -0,0 +1,3 @@ +[setup] + +including ../common.xly diff --git a/test/exactly/reporting/tap/cxx.suite b/test/exactly/reporting/tap/cxx.suite new file mode 100644 index 00000000..874a9fbb --- /dev/null +++ b/test/exactly/reporting/tap/cxx.suite @@ -0,0 +1,3 @@ +[suites] + +cxx diff --git a/test/exactly/reporting/tap/cxx/asserts.case b/test/exactly/reporting/tap/cxx/asserts.case new file mode 100644 index 00000000..c9cde03e --- /dev/null +++ b/test/exactly/reporting/tap/cxx/asserts.case @@ -0,0 +1,9 @@ +[act] + +% asserts.cc.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/asserts.txt diff --git a/test/exactly/reporting/tap/cxx/description.case b/test/exactly/reporting/tap/cxx/description.case new file mode 100644 index 00000000..9fc48bda --- /dev/null +++ b/test/exactly/reporting/tap/cxx/description.case @@ -0,0 +1,9 @@ +[act] + +% description.cc.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/description.txt diff --git a/test/exactly/reporting/tap/cxx/exactly.suite b/test/exactly/reporting/tap/cxx/exactly.suite new file mode 100644 index 00000000..c415324f --- /dev/null +++ b/test/exactly/reporting/tap/cxx/exactly.suite @@ -0,0 +1,13 @@ +[cases] + +*.case + +[setup] + +including ../common.xly + +[assert] + +exit-code == 0 + +stderr is-empty diff --git a/test/exactly/reporting/tap/cxx/expected/asserts.txt b/test/exactly/reporting/tap/cxx/expected/asserts.txt new file mode 100644 index 00000000..e975a72e --- /dev/null +++ b/test/exactly/reporting/tap/cxx/expected/asserts.txt @@ -0,0 +1,38 @@ +TAP version 13 +1..9 +# Criterion +# +# Running 9 tests from asserts +ok - asserts::wstring +ok - asserts::string +ok - asserts::stream +not ok - asserts::old_school + --- + assertions: 2 + failures: + - asserts.cc:22: |+ + Assertion failed: (null) + - asserts.cc:21: |+ + Assertion failed: You can fail an assertion with a message from anywhere + ... +ok - asserts::native +ok - asserts::float +not ok - asserts::exception + --- + assertions: 2 + failures: + - asserts.cc:169: |+ + Assertion failed: (null) + - asserts.cc:167: |+ + Assertion failed: (null) + ... +not ok - asserts::base + --- + assertions: 2 + failures: + - asserts.cc:16: |+ + Assertion failed: This assert runs + - asserts.cc:15: |+ + Assertion failed: assert is fatal, expect isn't + ... +ok - asserts::array diff --git a/test/exactly/reporting/tap/cxx/expected/description.txt b/test/exactly/reporting/tap/cxx/expected/description.txt new file mode 100644 index 00000000..fa07a0ef --- /dev/null +++ b/test/exactly/reporting/tap/cxx/expected/description.txt @@ -0,0 +1,13 @@ +TAP version 13 +1..2 +# Criterion +# +# Running 2 tests from misc +ok - misc::skipped This one is skipped # SKIP test was skipped +not ok - misc::failing Just a failing test + --- + assertions: 1 + failures: + - description.cc:4: |+ + Assertion failed: The expression 0 is false. + ... diff --git a/test/exactly/reporting/tap/cxx/expected/long-messages.txt b/test/exactly/reporting/tap/cxx/expected/long-messages.txt new file mode 100644 index 00000000..68825e45 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/expected/long-messages.txt @@ -0,0 +1,15 @@ +TAP version 13 +1..1 +# Criterion +# +# Running 1 tests from sample +not ok - sample::long_msg + --- + assertions: 1 + failures: + - long-messages.cc:4: |+ + Assertion failed: This is + A long message + Spawning multiple lines. + Formatting is respected. + ... diff --git a/test/exactly/reporting/tap/cxx/expected/more-suites.txt b/test/exactly/reporting/tap/cxx/expected/more-suites.txt new file mode 100644 index 00000000..de04bad3 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/expected/more-suites.txt @@ -0,0 +1,12 @@ +TAP version 13 +1..3 +# Criterion +# +# Running 1 tests from suite2 +ok - suite2::test +# +# Running 1 tests from suite1 +ok - suite1::test +# +# Running 1 tests from disabled +ok - disabled::test # SKIP test was skipped diff --git a/test/exactly/reporting/tap/cxx/expected/signal.txt b/test/exactly/reporting/tap/cxx/expected/signal.txt new file mode 100644 index 00000000..cea3b43c --- /dev/null +++ b/test/exactly/reporting/tap/cxx/expected/signal.txt @@ -0,0 +1,12 @@ +TAP version 13 +1..3 +# Criterion +# +# Running 3 tests from simple +not ok - simple::wrong_signal + --- + assertions: 0 + failures: + ... +not ok - simple::uncaught unexpected signal after signal.cc:14 +ok - simple::caught diff --git a/test/exactly/reporting/tap/cxx/expected/simple.txt b/test/exactly/reporting/tap/cxx/expected/simple.txt new file mode 100644 index 00000000..0a2a2070 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/expected/simple.txt @@ -0,0 +1,13 @@ +TAP version 13 +1..2 +# Criterion +# +# Running 2 tests from misc +ok - misc::passing +not ok - misc::failing + --- + assertions: 1 + failures: + - simple.cc:4: |+ + Assertion failed: The expression 0 is false. + ... diff --git a/test/exactly/reporting/tap/cxx/expected/skip.txt b/test/exactly/reporting/tap/cxx/expected/skip.txt new file mode 100644 index 00000000..94aac004 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/expected/skip.txt @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +# Criterion +# +# Running 2 tests from misc +ok - misc::skipping # SKIP test was skipped +ok - misc::message # SKIP Skips may take printf-like messages diff --git a/test/exactly/reporting/tap/cxx/long-messages.case b/test/exactly/reporting/tap/cxx/long-messages.case new file mode 100644 index 00000000..8b57bd6a --- /dev/null +++ b/test/exactly/reporting/tap/cxx/long-messages.case @@ -0,0 +1,9 @@ +[act] + +% long-messages.cc.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/long-messages.txt diff --git a/test/exactly/reporting/tap/cxx/more-suites.case b/test/exactly/reporting/tap/cxx/more-suites.case new file mode 100644 index 00000000..82316121 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/more-suites.case @@ -0,0 +1,9 @@ +[act] + +% more-suites.cc.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/more-suites.txt diff --git a/test/exactly/reporting/tap/cxx/signal.case b/test/exactly/reporting/tap/cxx/signal.case new file mode 100644 index 00000000..3ebeb661 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/signal.case @@ -0,0 +1,9 @@ +[act] + +% signal.cc.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/signal.txt diff --git a/test/exactly/reporting/tap/cxx/simple--criterion_outputs.case b/test/exactly/reporting/tap/cxx/simple--criterion_outputs.case new file mode 100644 index 00000000..a602bb56 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/simple--criterion_outputs.case @@ -0,0 +1,13 @@ +[setup] + +env -of act CRITERION_OUTPUTS = tap:- + +[act] + +% simple.cc.bin + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/tap/cxx/simple--output-option.case b/test/exactly/reporting/tap/cxx/simple--output-option.case new file mode 100644 index 00000000..69d01ce1 --- /dev/null +++ b/test/exactly/reporting/tap/cxx/simple--output-option.case @@ -0,0 +1,11 @@ +# Testing --output=tap + +[act] + +% simple.cc.bin --output=tap:- + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/tap/cxx/simple.case b/test/exactly/reporting/tap/cxx/simple.case new file mode 100644 index 00000000..6136d37f --- /dev/null +++ b/test/exactly/reporting/tap/cxx/simple.case @@ -0,0 +1,9 @@ +[act] + +% simple.cc.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/tap/cxx/skip.case b/test/exactly/reporting/tap/cxx/skip.case new file mode 100644 index 00000000..42757f8c --- /dev/null +++ b/test/exactly/reporting/tap/cxx/skip.case @@ -0,0 +1,9 @@ +[act] + +% skip.cc.bin --tap + +[assert] + +stdout + -transformed-by replace_version + equals -contents-of expected/skip.txt diff --git a/test/exactly/reporting/tap/exactly.suite b/test/exactly/reporting/tap/exactly.suite new file mode 100644 index 00000000..b56d011a --- /dev/null +++ b/test/exactly/reporting/tap/exactly.suite @@ -0,0 +1,4 @@ +[suites] + +c +cxx diff --git a/test/exactly/reporting/xml/c.suite b/test/exactly/reporting/xml/c.suite new file mode 100644 index 00000000..13f621bd --- /dev/null +++ b/test/exactly/reporting/xml/c.suite @@ -0,0 +1,3 @@ +[suites] + +c diff --git a/test/exactly/reporting/xml/c/asserts.case b/test/exactly/reporting/xml/c/asserts.case new file mode 100644 index 00000000..7adbaa5b --- /dev/null +++ b/test/exactly/reporting/xml/c/asserts.case @@ -0,0 +1,9 @@ +[act] + +% asserts.c.bin --xml + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/asserts.txt diff --git a/test/exactly/reporting/xml/c/description.case b/test/exactly/reporting/xml/c/description.case new file mode 100644 index 00000000..a1b64ff0 --- /dev/null +++ b/test/exactly/reporting/xml/c/description.case @@ -0,0 +1,9 @@ +[act] + +% description.c.bin --xml + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/description.txt diff --git a/test/exactly/reporting/xml/c/exactly.suite b/test/exactly/reporting/xml/c/exactly.suite new file mode 100644 index 00000000..edf2c3e2 --- /dev/null +++ b/test/exactly/reporting/xml/c/exactly.suite @@ -0,0 +1,7 @@ +[cases] + +*.case + +[setup] + +including ../common.xly diff --git a/test/exactly/reporting/xml/c/expected/asserts.txt b/test/exactly/reporting/xml/c/expected/asserts.txt new file mode 100644 index 00000000..b03ee8e8 --- /dev/null +++ b/test/exactly/reporting/xml/c/expected/asserts.txt @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/exactly/reporting/xml/c/expected/description.txt b/test/exactly/reporting/xml/c/expected/description.txt new file mode 100644 index 00000000..c3273f92 --- /dev/null +++ b/test/exactly/reporting/xml/c/expected/description.txt @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/exactly/reporting/xml/c/expected/long-messages.txt b/test/exactly/reporting/xml/c/expected/long-messages.txt new file mode 100644 index 00000000..548d13e9 --- /dev/null +++ b/test/exactly/reporting/xml/c/expected/long-messages.txt @@ -0,0 +1,9 @@ + + + + + + A long message Spawning multiple lines. Formatting is respected. + + + diff --git a/test/exactly/reporting/xml/c/expected/more-suites.txt b/test/exactly/reporting/xml/c/expected/more-suites.txt new file mode 100644 index 00000000..19aca1cb --- /dev/null +++ b/test/exactly/reporting/xml/c/expected/more-suites.txt @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/test/exactly/reporting/xml/c/expected/signal.txt b/test/exactly/reporting/xml/c/expected/signal.txt new file mode 100644 index 00000000..751886ab --- /dev/null +++ b/test/exactly/reporting/xml/c/expected/signal.txt @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/test/exactly/reporting/xml/c/expected/simple.txt b/test/exactly/reporting/xml/c/expected/simple.txt new file mode 100644 index 00000000..31d55af0 --- /dev/null +++ b/test/exactly/reporting/xml/c/expected/simple.txt @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/test/exactly/reporting/xml/c/expected/skip.txt b/test/exactly/reporting/xml/c/expected/skip.txt new file mode 100644 index 00000000..909b390c --- /dev/null +++ b/test/exactly/reporting/xml/c/expected/skip.txt @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/exactly/reporting/xml/c/long-messages.case b/test/exactly/reporting/xml/c/long-messages.case new file mode 100644 index 00000000..2c47b65f --- /dev/null +++ b/test/exactly/reporting/xml/c/long-messages.case @@ -0,0 +1,9 @@ +[act] + +% long-messages.c.bin --xml + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/long-messages.txt diff --git a/test/exactly/reporting/xml/c/more-suites.case b/test/exactly/reporting/xml/c/more-suites.case new file mode 100644 index 00000000..f405fd57 --- /dev/null +++ b/test/exactly/reporting/xml/c/more-suites.case @@ -0,0 +1,9 @@ +[act] + +% more-suites.c.bin --xml + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/more-suites.txt diff --git a/test/exactly/reporting/xml/c/signal.case b/test/exactly/reporting/xml/c/signal.case new file mode 100644 index 00000000..7aec1045 --- /dev/null +++ b/test/exactly/reporting/xml/c/signal.case @@ -0,0 +1,9 @@ +[act] + +% signal.c.bin --xml + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/signal.txt diff --git a/test/exactly/reporting/xml/c/simple--criterion_outputs.case b/test/exactly/reporting/xml/c/simple--criterion_outputs.case new file mode 100644 index 00000000..6ee04d37 --- /dev/null +++ b/test/exactly/reporting/xml/c/simple--criterion_outputs.case @@ -0,0 +1,13 @@ +[setup] + +env -of act CRITERION_OUTPUTS = xml:- + +[act] + +% simple.c.bin + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/xml/c/simple--output-option.case b/test/exactly/reporting/xml/c/simple--output-option.case new file mode 100644 index 00000000..ff329a7b --- /dev/null +++ b/test/exactly/reporting/xml/c/simple--output-option.case @@ -0,0 +1,9 @@ +[act] + +% simple.c.bin --output=xml:- + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/xml/c/simple.case b/test/exactly/reporting/xml/c/simple.case new file mode 100644 index 00000000..a36aba2b --- /dev/null +++ b/test/exactly/reporting/xml/c/simple.case @@ -0,0 +1,9 @@ +[act] + +% simple.c.bin --xml + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/simple.txt diff --git a/test/exactly/reporting/xml/c/skip.case b/test/exactly/reporting/xml/c/skip.case new file mode 100644 index 00000000..8713c0e4 --- /dev/null +++ b/test/exactly/reporting/xml/c/skip.case @@ -0,0 +1,9 @@ +[act] + +% skip.c.bin --xml + +[assert] + +stdout + -transformed-by ( replace_version | replace_times ) + equals -contents-of expected/skip.txt diff --git a/test/exactly/reporting/xml/common.xly b/test/exactly/reporting/xml/common.xly new file mode 100644 index 00000000..a85b41e0 --- /dev/null +++ b/test/exactly/reporting/xml/common.xly @@ -0,0 +1,15 @@ +[setup] + +including ../common.xly + +[before-assert] + +def text-transformer replace_times = + + replace 'time="\d\.\d\d\d"' 'time="