From 07d41a7cd7f3ef8ca1e9b70768975695d3536272 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sun, 13 Jul 2025 20:38:12 +0200 Subject: [PATCH 01/10] Correctly handle `--no-run` rustdoc test option --- src/librustdoc/doctest.rs | 2 +- src/librustdoc/doctest/runner.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 9499258f983a1..a1e917df75ab9 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -358,7 +358,7 @@ pub(crate) fn run_tests( ); for (doctest, scraped_test) in &doctests { - tests_runner.add_test(doctest, scraped_test, &target_str); + tests_runner.add_test(doctest, scraped_test, &target_str, rustdoc_options); } let (duration, ret) = tests_runner.run_merged_tests( rustdoc_test_options, diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index fcfa424968e48..5493d56456872 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -39,6 +39,7 @@ impl DocTestRunner { doctest: &DocTestBuilder, scraped_test: &ScrapedDocTest, target_str: &str, + opts: &RustdocOptions, ) { let ignore = match scraped_test.langstr.ignore { Ignore::All => true, @@ -62,6 +63,7 @@ impl DocTestRunner { self.nb_tests, &mut self.output, &mut self.output_merged_tests, + opts, ), )); self.supports_color &= doctest.supports_color; @@ -223,6 +225,7 @@ fn generate_mergeable_doctest( id: usize, output: &mut String, output_merged_tests: &mut String, + opts: &RustdocOptions, ) -> String { let test_id = format!("__doctest_{id}"); @@ -256,7 +259,7 @@ fn main() {returns_result} {{ ) .unwrap(); } - let not_running = ignore || scraped_test.langstr.no_run; + let not_running = ignore || scraped_test.no_run(opts); writeln!( output_merged_tests, " @@ -270,7 +273,7 @@ test::StaticTestFn( test_name = scraped_test.name, file = scraped_test.path(), line = scraped_test.line, - no_run = scraped_test.langstr.no_run, + no_run = scraped_test.no_run(opts), should_panic = !scraped_test.langstr.no_run && scraped_test.langstr.should_panic, // Setting `no_run` to `true` in `TestDesc` still makes the test run, so we simply // don't give it the function to run. From 796c4efe44a57ddbaf071937e26f1279e8595302 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 7 Jul 2025 11:53:18 +0200 Subject: [PATCH 02/10] Correctly handle `should_panic` doctest attribute --- src/librustdoc/doctest.rs | 4 ++-- src/librustdoc/doctest/runner.rs | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index a1e917df75ab9..df457536b70e8 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -836,7 +836,7 @@ fn run_test( match result { Err(e) => return (duration, Err(TestFailure::ExecutionError(e))), Ok(out) => { - if langstr.should_panic && out.status.success() { + if langstr.should_panic && out.status.code() != Some(101) { return (duration, Err(TestFailure::UnexpectedRunPass)); } else if !langstr.should_panic && !out.status.success() { return (duration, Err(TestFailure::ExecutionFailure(out))); @@ -1146,7 +1146,7 @@ fn doctest_run_fn( eprint!("Test compiled successfully, but it's marked `compile_fail`."); } TestFailure::UnexpectedRunPass => { - eprint!("Test executable succeeded, but it's marked `should_panic`."); + eprint!("Test didn't panic, but it's marked `should_panic`."); } TestFailure::MissingErrorCodes(codes) => { eprint!("Some expected error codes were not found: {codes:?}"); diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index 5493d56456872..d02b32faa319e 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -136,13 +136,20 @@ mod __doctest_mod {{ }} #[allow(unused)] - pub fn doctest_runner(bin: &std::path::Path, test_nb: usize) -> ExitCode {{ + pub fn doctest_runner(bin: &std::path::Path, test_nb: usize, should_panic: bool) -> ExitCode {{ let out = std::process::Command::new(bin) .env(self::RUN_OPTION, test_nb.to_string()) .args(std::env::args().skip(1).collect::>()) .output() .expect(\"failed to run command\"); - if !out.status.success() {{ + if should_panic {{ + if out.status.code() != Some(101) {{ + eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); + ExitCode::FAILURE + }} else {{ + ExitCode::SUCCESS + }} + }} else if !out.status.success() {{ if let Some(code) = out.status.code() {{ eprintln!(\"Test executable failed (exit status: {{code}}).\"); }} else {{ @@ -265,7 +272,7 @@ fn main() {returns_result} {{ " mod {test_id} {{ pub const TEST: test::TestDescAndFn = test::TestDescAndFn::new_doctest( -{test_name:?}, {ignore}, {file:?}, {line}, {no_run}, {should_panic}, +{test_name:?}, {ignore}, {file:?}, {line}, {no_run}, false, test::StaticTestFn( || {{{runner}}}, )); @@ -274,7 +281,6 @@ test::StaticTestFn( file = scraped_test.path(), line = scraped_test.line, no_run = scraped_test.no_run(opts), - should_panic = !scraped_test.langstr.no_run && scraped_test.langstr.should_panic, // Setting `no_run` to `true` in `TestDesc` still makes the test run, so we simply // don't give it the function to run. runner = if not_running { @@ -283,11 +289,12 @@ test::StaticTestFn( format!( " if let Some(bin_path) = crate::__doctest_mod::doctest_path() {{ - test::assert_test_result(crate::__doctest_mod::doctest_runner(bin_path, {id})) + test::assert_test_result(crate::__doctest_mod::doctest_runner(bin_path, {id}, {should_panic})) }} else {{ test::assert_test_result(doctest_bundle::{test_id}::__main_fn()) }} ", + should_panic = scraped_test.langstr.should_panic, ) }, ) From 11b7070577e52c1d26b35021b8a07475e794e93c Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 4 Jul 2025 21:42:19 +0200 Subject: [PATCH 03/10] Add regression test for #143009 --- tests/run-make/rustdoc-should-panic/rmake.rs | 36 ++++++++++++++++++++ tests/run-make/rustdoc-should-panic/test.rs | 14 ++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/run-make/rustdoc-should-panic/rmake.rs create mode 100644 tests/run-make/rustdoc-should-panic/test.rs diff --git a/tests/run-make/rustdoc-should-panic/rmake.rs b/tests/run-make/rustdoc-should-panic/rmake.rs new file mode 100644 index 0000000000000..100a62f5db313 --- /dev/null +++ b/tests/run-make/rustdoc-should-panic/rmake.rs @@ -0,0 +1,36 @@ +// Ensure that `should_panic` doctests only succeed if the test actually panicked. +// Regression test for . + +//@ needs-target-std + +use run_make_support::rustdoc; + +fn check_output(output: String, edition: &str) { + let should_contain = &[ + "test test.rs - bad_exit_code (line 1) ... FAILED", + "test test.rs - did_not_panic (line 6) ... FAILED", + "test test.rs - did_panic (line 11) ... ok", + "---- test.rs - bad_exit_code (line 1) stdout ---- +Test executable failed (exit status: 1).", + "---- test.rs - did_not_panic (line 6) stdout ---- +Test didn't panic, but it's marked `should_panic`.", + "test result: FAILED. 1 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out;", + ]; + for text in should_contain { + assert!( + output.contains(text), + "output doesn't contains (edition: {edition}) {:?}\nfull output: {output}", + text + ); + } +} + +fn main() { + check_output(rustdoc().input("test.rs").arg("--test").run_fail().stdout_utf8(), "2015"); + + // Same check with the merged doctest feature (enabled with the 2024 edition). + check_output( + rustdoc().input("test.rs").arg("--test").edition("2024").run_fail().stdout_utf8(), + "2024", + ); +} diff --git a/tests/run-make/rustdoc-should-panic/test.rs b/tests/run-make/rustdoc-should-panic/test.rs new file mode 100644 index 0000000000000..1eea8e1e1958c --- /dev/null +++ b/tests/run-make/rustdoc-should-panic/test.rs @@ -0,0 +1,14 @@ +/// ``` +/// std::process::exit(1); +/// ``` +fn bad_exit_code() {} + +/// ```should_panic +/// std::process::exit(1); +/// ``` +fn did_not_panic() {} + +/// ```should_panic +/// panic!("yeay"); +/// ``` +fn did_panic() {} From 21a4d9dda7b2234d4454c2413956f96d8884cd65 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 7 Jul 2025 13:35:02 +0200 Subject: [PATCH 04/10] Update std doctests --- library/std/src/error.rs | 16 ++++++++++++---- .../failed-doctest-should-panic-2021.stdout | 2 +- .../doctest/failed-doctest-should-panic.stdout | 5 +++-- tests/rustdoc-ui/doctest/wrong-ast-2024.stdout | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/library/std/src/error.rs b/library/std/src/error.rs index def5f984c88e4..09bfc83ebca6c 100644 --- a/library/std/src/error.rs +++ b/library/std/src/error.rs @@ -123,7 +123,7 @@ use crate::fmt::{self, Write}; /// the `Debug` output means `Report` is an ideal starting place for formatting errors returned /// from `main`. /// -/// ```should_panic +/// ``` /// #![feature(error_reporter)] /// use std::error::Report; /// # use std::error::Error; @@ -154,10 +154,14 @@ use crate::fmt::{self, Write}; /// # Err(SuperError { source: SuperErrorSideKick }) /// # } /// -/// fn main() -> Result<(), Report> { +/// fn run() -> Result<(), Report> { /// get_super_error()?; /// Ok(()) /// } +/// +/// fn main() { +/// assert!(run().is_err()); +/// } /// ``` /// /// This example produces the following output: @@ -170,7 +174,7 @@ use crate::fmt::{self, Write}; /// output format. If you want to make sure your `Report`s are pretty printed and include backtrace /// you will need to manually convert and enable those flags. /// -/// ```should_panic +/// ``` /// #![feature(error_reporter)] /// use std::error::Report; /// # use std::error::Error; @@ -201,12 +205,16 @@ use crate::fmt::{self, Write}; /// # Err(SuperError { source: SuperErrorSideKick }) /// # } /// -/// fn main() -> Result<(), Report> { +/// fn run() -> Result<(), Report> { /// get_super_error() /// .map_err(Report::from) /// .map_err(|r| r.pretty(true).show_backtrace(true))?; /// Ok(()) /// } +/// +/// fn main() { +/// assert!(run().is_err()); +/// } /// ``` /// /// This example produces the following output: diff --git a/tests/rustdoc-ui/doctest/failed-doctest-should-panic-2021.stdout b/tests/rustdoc-ui/doctest/failed-doctest-should-panic-2021.stdout index 9f4d60e6f4de5..f8413756e3d6d 100644 --- a/tests/rustdoc-ui/doctest/failed-doctest-should-panic-2021.stdout +++ b/tests/rustdoc-ui/doctest/failed-doctest-should-panic-2021.stdout @@ -5,7 +5,7 @@ test $DIR/failed-doctest-should-panic-2021.rs - Foo (line 10) ... FAILED failures: ---- $DIR/failed-doctest-should-panic-2021.rs - Foo (line 10) stdout ---- -Test executable succeeded, but it's marked `should_panic`. +Test didn't panic, but it's marked `should_panic`. failures: $DIR/failed-doctest-should-panic-2021.rs - Foo (line 10) diff --git a/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout b/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout index 9047fe0dcdd93..61099e6424ae7 100644 --- a/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout +++ b/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout @@ -1,11 +1,12 @@ running 1 test -test $DIR/failed-doctest-should-panic.rs - Foo (line 12) - should panic ... FAILED +test $DIR/failed-doctest-should-panic.rs - Foo (line 12) ... FAILED failures: ---- $DIR/failed-doctest-should-panic.rs - Foo (line 12) stdout ---- -note: test did not panic as expected at $DIR/failed-doctest-should-panic.rs:12:0 +Test didn't panic, but it's marked `should_panic`. + failures: $DIR/failed-doctest-should-panic.rs - Foo (line 12) diff --git a/tests/rustdoc-ui/doctest/wrong-ast-2024.stdout b/tests/rustdoc-ui/doctest/wrong-ast-2024.stdout index 13567b41e51f5..27f9a0157a6cc 100644 --- a/tests/rustdoc-ui/doctest/wrong-ast-2024.stdout +++ b/tests/rustdoc-ui/doctest/wrong-ast-2024.stdout @@ -1,6 +1,6 @@ running 1 test -test $DIR/wrong-ast-2024.rs - three (line 20) - should panic ... ok +test $DIR/wrong-ast-2024.rs - three (line 20) ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME From 5f2ae4fb078569486b3a285b3bc5378e3bf7c9fb Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sun, 13 Jul 2025 20:49:29 +0200 Subject: [PATCH 05/10] Add regression test for #143858 --- tests/run-make/rustdoc-should-panic/rmake.rs | 2 +- .../doctest/no-run.edition2021.stdout | 12 +++++ .../doctest/no-run.edition2024.stdout | 18 ++++++++ tests/rustdoc-ui/doctest/no-run.rs | 44 +++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/rustdoc-ui/doctest/no-run.edition2021.stdout create mode 100644 tests/rustdoc-ui/doctest/no-run.edition2024.stdout create mode 100644 tests/rustdoc-ui/doctest/no-run.rs diff --git a/tests/run-make/rustdoc-should-panic/rmake.rs b/tests/run-make/rustdoc-should-panic/rmake.rs index 100a62f5db313..ffe3c9ef6eb60 100644 --- a/tests/run-make/rustdoc-should-panic/rmake.rs +++ b/tests/run-make/rustdoc-should-panic/rmake.rs @@ -1,7 +1,7 @@ // Ensure that `should_panic` doctests only succeed if the test actually panicked. // Regression test for . -//@ needs-target-std +//@ ignore-cross-compile use run_make_support::rustdoc; diff --git a/tests/rustdoc-ui/doctest/no-run.edition2021.stdout b/tests/rustdoc-ui/doctest/no-run.edition2021.stdout new file mode 100644 index 0000000000000..937cd76bfb462 --- /dev/null +++ b/tests/rustdoc-ui/doctest/no-run.edition2021.stdout @@ -0,0 +1,12 @@ + +running 7 tests +test $DIR/no-run.rs - f (line 14) - compile ... ok +test $DIR/no-run.rs - f (line 17) - compile ... ok +test $DIR/no-run.rs - f (line 20) ... ignored +test $DIR/no-run.rs - f (line 23) - compile ... ok +test $DIR/no-run.rs - f (line 29) - compile fail ... ok +test $DIR/no-run.rs - f (line 34) - compile ... ok +test $DIR/no-run.rs - f (line 38) - compile ... ok + +test result: ok. 6 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in $TIME + diff --git a/tests/rustdoc-ui/doctest/no-run.edition2024.stdout b/tests/rustdoc-ui/doctest/no-run.edition2024.stdout new file mode 100644 index 0000000000000..921e059979b1f --- /dev/null +++ b/tests/rustdoc-ui/doctest/no-run.edition2024.stdout @@ -0,0 +1,18 @@ + +running 5 tests +test $DIR/no-run.rs - f (line 14) - compile ... ok +test $DIR/no-run.rs - f (line 17) - compile ... ok +test $DIR/no-run.rs - f (line 23) - compile ... ok +test $DIR/no-run.rs - f (line 34) - compile ... ok +test $DIR/no-run.rs - f (line 38) - compile ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME + + +running 2 tests +test $DIR/no-run.rs - f (line 20) ... ignored +test $DIR/no-run.rs - f (line 29) - compile fail ... ok + +test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in $TIME + +all doctests ran in $TIME; merged doctests compilation took $TIME diff --git a/tests/rustdoc-ui/doctest/no-run.rs b/tests/rustdoc-ui/doctest/no-run.rs new file mode 100644 index 0000000000000..78198badd43b5 --- /dev/null +++ b/tests/rustdoc-ui/doctest/no-run.rs @@ -0,0 +1,44 @@ +// This test ensures that the `--no-run` flag works the same between normal and merged doctests. +// Regression test for . + +//@ check-pass +//@ revisions: edition2021 edition2024 +//@ [edition2021]edition:2021 +//@ [edition2024]edition:2024 +//@ compile-flags:-Z unstable-options --test --no-run --test-args=--test-threads=1 +//@ normalize-stdout: "tests/rustdoc-ui/doctest" -> "$$DIR" +//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME" +//@ normalize-stdout: "ran in \d+\.\d+s" -> "ran in $$TIME" +//@ normalize-stdout: "compilation took \d+\.\d+s" -> "compilation took $$TIME" + +/// ``` +/// let a = true; +/// ``` +/// ```should_panic +/// panic!() +/// ``` +/// ```ignore (incomplete-code) +/// fn foo() { +/// ``` +/// ```no_run +/// loop { +/// println!("Hello, world"); +/// } +/// ``` +/// fails to compile +/// ```compile_fail +/// let x = 5; +/// x += 2; // shouldn't compile! +/// ``` +/// Ok the test does not run +/// ``` +/// panic!() +/// ``` +/// Ok the test does not run +/// ```should_panic +/// loop { +/// println!("Hello, world"); +/// panic!() +/// } +/// ``` +pub fn f() {} From b001ba67dda247037b052cfa76b154d2df3908ab Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 1 Aug 2025 11:28:17 +0200 Subject: [PATCH 06/10] Add FIXME comments to use `test::ERROR_EXIT_CODE` once public and fix typo --- src/librustdoc/doctest.rs | 1 + src/librustdoc/doctest/runner.rs | 1 + tests/run-make/rustdoc-should-panic/rmake.rs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index df457536b70e8..51972f8b149c7 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -836,6 +836,7 @@ fn run_test( match result { Err(e) => return (duration, Err(TestFailure::ExecutionError(e))), Ok(out) => { + // FIXME: use test::ERROR_EXIT_CODE once public if langstr.should_panic && out.status.code() != Some(101) { return (duration, Err(TestFailure::UnexpectedRunPass)); } else if !langstr.should_panic && !out.status.success() { diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index d02b32faa319e..d241d44441d53 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -143,6 +143,7 @@ mod __doctest_mod {{ .output() .expect(\"failed to run command\"); if should_panic {{ + // FIXME: use test::ERROR_EXIT_CODE once public if out.status.code() != Some(101) {{ eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); ExitCode::FAILURE diff --git a/tests/run-make/rustdoc-should-panic/rmake.rs b/tests/run-make/rustdoc-should-panic/rmake.rs index ffe3c9ef6eb60..4d6e2c98ae771 100644 --- a/tests/run-make/rustdoc-should-panic/rmake.rs +++ b/tests/run-make/rustdoc-should-panic/rmake.rs @@ -19,7 +19,7 @@ Test didn't panic, but it's marked `should_panic`.", for text in should_contain { assert!( output.contains(text), - "output doesn't contains (edition: {edition}) {:?}\nfull output: {output}", + "output (edition: {edition}) doesn't contain {:?}\nfull output: {output}", text ); } From 030b66441986be7b84d355d5377cb68783b4b564 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 4 Oct 2025 13:39:31 +0200 Subject: [PATCH 07/10] Use libtest `ERROR_EXIT_CODE` constant --- src/librustdoc/doctest.rs | 3 +-- src/librustdoc/doctest/runner.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 51972f8b149c7..0eb698ca544fe 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -836,8 +836,7 @@ fn run_test( match result { Err(e) => return (duration, Err(TestFailure::ExecutionError(e))), Ok(out) => { - // FIXME: use test::ERROR_EXIT_CODE once public - if langstr.should_panic && out.status.code() != Some(101) { + if langstr.should_panic && out.status.code() != Some(test::ERROR_EXIT_CODE) { return (duration, Err(TestFailure::UnexpectedRunPass)); } else if !langstr.should_panic && !out.status.success() { return (duration, Err(TestFailure::ExecutionFailure(out))); diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index d241d44441d53..66ffb8aae23e6 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -143,8 +143,7 @@ mod __doctest_mod {{ .output() .expect(\"failed to run command\"); if should_panic {{ - // FIXME: use test::ERROR_EXIT_CODE once public - if out.status.code() != Some(101) {{ + if out.status.code() != Some(test::ERROR_EXIT_CODE) {{ eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); ExitCode::FAILURE }} else {{ From 560d4505591c160f0cfbfcc7653f281e8325cc0b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sun, 5 Oct 2025 20:43:05 +0200 Subject: [PATCH 08/10] Correctly handle `should_panic` on targets not supporting it --- src/librustdoc/doctest.rs | 16 ++++++++++++++++ src/librustdoc/doctest/runner.rs | 11 ++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 0eb698ca544fe..c3db6b712b316 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -803,6 +803,22 @@ fn run_test( let duration = instant.elapsed(); if doctest.no_run { return (duration, Ok(())); + } else if doctest.langstr.should_panic + // Equivalent of: + // + // ``` + // (cfg!(target_family = "wasm") || cfg!(target_os = "zkvm")) + // && !cfg!(target_os = "emscripten") + // ``` + && let TargetTuple::TargetTuple(ref s) = rustdoc_options.target + && let mut iter = s.split('-') + && let Some(arch) = iter.next() + && iter.next().is_some() + && let os = iter.next() + && (arch.starts_with("wasm") || os == Some("zkvm")) && os != Some("emscripten") + { + // We cannot correctly handle `should_panic` in some wasm targets so we exit early. + return (duration, Ok(())); } // Run the code! diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index 66ffb8aae23e6..d7fd586ebac1c 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -129,6 +129,9 @@ mod __doctest_mod {{ pub static BINARY_PATH: OnceLock = OnceLock::new(); pub const RUN_OPTION: &str = \"RUSTDOC_DOCTEST_RUN_NB_TEST\"; + pub const SHOULD_PANIC_DISABLED: bool = ( + cfg!(target_family = \"wasm\") || cfg!(target_os = \"zkvm\") + ) && !cfg!(target_os = \"emscripten\"); #[allow(unused)] pub fn doctest_path() -> Option<&'static PathBuf> {{ @@ -266,13 +269,14 @@ fn main() {returns_result} {{ ) .unwrap(); } + let should_panic = scraped_test.langstr.should_panic; let not_running = ignore || scraped_test.no_run(opts); writeln!( output_merged_tests, " mod {test_id} {{ pub const TEST: test::TestDescAndFn = test::TestDescAndFn::new_doctest( -{test_name:?}, {ignore}, {file:?}, {line}, {no_run}, false, +{test_name:?}, {ignore} || ({should_panic} && crate::__doctest_mod::SHOULD_PANIC_DISABLED), {file:?}, {line}, {no_run}, false, test::StaticTestFn( || {{{runner}}}, )); @@ -288,13 +292,14 @@ test::StaticTestFn( } else { format!( " -if let Some(bin_path) = crate::__doctest_mod::doctest_path() {{ +if {should_panic} && crate::__doctest_mod::SHOULD_PANIC_DISABLED {{ + test::assert_test_result(Ok::<(), String>(())) +}} else if let Some(bin_path) = crate::__doctest_mod::doctest_path() {{ test::assert_test_result(crate::__doctest_mod::doctest_runner(bin_path, {id}, {should_panic})) }} else {{ test::assert_test_result(doctest_bundle::{test_id}::__main_fn()) }} ", - should_panic = scraped_test.langstr.should_panic, ) }, ) From b70e20ab8829a61c9ade751ae4d8295ff56b4789 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 7 Oct 2025 16:27:43 +0200 Subject: [PATCH 09/10] Correctly handle `-C panic=abort` in doctests --- src/librustdoc/doctest.rs | 35 ++++++++++++++++-- src/librustdoc/doctest/runner.rs | 38 ++++++++++++++++---- tests/run-make/rustdoc-should-panic/rmake.rs | 19 ++++++---- 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index c3db6b712b316..28453f990aed2 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -7,6 +7,8 @@ mod rust; use std::fs::File; use std::hash::{Hash, Hasher}; use std::io::{self, Write}; +#[cfg(unix)] +use std::os::unix::process::ExitStatusExt; use std::path::{Path, PathBuf}; use std::process::{self, Command, Stdio}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -849,12 +851,39 @@ fn run_test( } else { cmd.output() }; + + // FIXME: Make `test::get_result_from_exit_code` public and use this code instead of this. + // + // On Zircon (the Fuchsia kernel), an abort from userspace calls the + // LLVM implementation of __builtin_trap(), e.g., ud2 on x86, which + // raises a kernel exception. If a userspace process does not + // otherwise arrange exception handling, the kernel kills the process + // with this return code. + #[cfg(target_os = "fuchsia")] + const ZX_TASK_RETCODE_EXCEPTION_KILL: i32 = -1028; + // On Windows we use __fastfail to abort, which is documented to use this + // exception code. + #[cfg(windows)] + const STATUS_FAIL_FAST_EXCEPTION: i32 = 0xC0000409u32 as i32; + #[cfg(unix)] + const SIGABRT: std::ffi::c_int = 6; match result { Err(e) => return (duration, Err(TestFailure::ExecutionError(e))), Ok(out) => { - if langstr.should_panic && out.status.code() != Some(test::ERROR_EXIT_CODE) { - return (duration, Err(TestFailure::UnexpectedRunPass)); - } else if !langstr.should_panic && !out.status.success() { + if langstr.should_panic { + match out.status.code() { + Some(test::ERROR_EXIT_CODE) => {} + #[cfg(windows)] + Some(STATUS_FAIL_FAST_EXCEPTION) => {} + #[cfg(unix)] + None if out.status.signal() == Some(SIGABRT) => {} + // Upon an abort, Fuchsia returns the status code + // `ZX_TASK_RETCODE_EXCEPTION_KILL`. + #[cfg(target_os = "fuchsia")] + Some(ZX_TASK_RETCODE_EXCEPTION_KILL) => {} + _ => return (duration, Err(TestFailure::UnexpectedRunPass)), + } + } else if !out.status.success() { return (duration, Err(TestFailure::ExecutionFailure(out))); } } diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index d7fd586ebac1c..afcab202a8e3e 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -123,9 +123,11 @@ impl DocTestRunner { {output} mod __doctest_mod {{ - use std::sync::OnceLock; + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt; use std::path::PathBuf; use std::process::ExitCode; + use std::sync::OnceLock; pub static BINARY_PATH: OnceLock = OnceLock::new(); pub const RUN_OPTION: &str = \"RUSTDOC_DOCTEST_RUN_NB_TEST\"; @@ -146,11 +148,35 @@ mod __doctest_mod {{ .output() .expect(\"failed to run command\"); if should_panic {{ - if out.status.code() != Some(test::ERROR_EXIT_CODE) {{ - eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); - ExitCode::FAILURE - }} else {{ - ExitCode::SUCCESS + // FIXME: Make `test::get_result_from_exit_code` public and use this code instead of this. + // + // On Zircon (the Fuchsia kernel), an abort from userspace calls the + // LLVM implementation of __builtin_trap(), e.g., ud2 on x86, which + // raises a kernel exception. If a userspace process does not + // otherwise arrange exception handling, the kernel kills the process + // with this return code. + #[cfg(target_os = \"fuchsia\")] + const ZX_TASK_RETCODE_EXCEPTION_KILL: i32 = -1028; + // On Windows we use __fastfail to abort, which is documented to use this + // exception code. + #[cfg(windows)] + const STATUS_FAIL_FAST_EXCEPTION: i32 = 0xC0000409u32 as i32; + #[cfg(unix)] + const SIGABRT: std::ffi::c_int = 6; + + match out.status.code() {{ + Some(test::ERROR_EXIT_CODE) => ExitCode::SUCCESS, + #[cfg(windows)] + Some(STATUS_FAIL_FAST_EXCEPTION) => ExitCode::SUCCESS, + #[cfg(unix)] + None if out.status.signal() == Some(SIGABRT) => ExitCode::SUCCESS, + // Upon an abort, Fuchsia returns the status code ZX_TASK_RETCODE_EXCEPTION_KILL. + #[cfg(target_os = \"fuchsia\")] + Some(ZX_TASK_RETCODE_EXCEPTION_KILL) => ExitCode::SUCCESS, + _ => {{ + eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); + ExitCode::FAILURE + }} }} }} else if !out.status.success() {{ if let Some(code) = out.status.code() {{ diff --git a/tests/run-make/rustdoc-should-panic/rmake.rs b/tests/run-make/rustdoc-should-panic/rmake.rs index 4d6e2c98ae771..438608ff8b4ed 100644 --- a/tests/run-make/rustdoc-should-panic/rmake.rs +++ b/tests/run-make/rustdoc-should-panic/rmake.rs @@ -5,7 +5,13 @@ use run_make_support::rustdoc; -fn check_output(output: String, edition: &str) { +fn check_output(edition: &str, panic_abort: bool) { + let mut rustdoc_cmd = rustdoc(); + rustdoc_cmd.input("test.rs").arg("--test").edition(edition); + if panic_abort { + rustdoc_cmd.args(["-C", "panic=abort"]); + } + let output = rustdoc_cmd.run_fail().stdout_utf8(); let should_contain = &[ "test test.rs - bad_exit_code (line 1) ... FAILED", "test test.rs - did_not_panic (line 6) ... FAILED", @@ -26,11 +32,12 @@ Test didn't panic, but it's marked `should_panic`.", } fn main() { - check_output(rustdoc().input("test.rs").arg("--test").run_fail().stdout_utf8(), "2015"); + check_output("2015", false); // Same check with the merged doctest feature (enabled with the 2024 edition). - check_output( - rustdoc().input("test.rs").arg("--test").edition("2024").run_fail().stdout_utf8(), - "2024", - ); + check_output("2024", false); + + // Checking that `-C panic=abort` is working too. + check_output("2015", true); + check_output("2024", true); } From 6060bccd26b9a3a6bc14fd26e47d35fccdf590a3 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 8 Oct 2025 11:57:11 +0200 Subject: [PATCH 10/10] Improve error messages for failing `should_panic` doctests Added missing FIXME comments --- src/librustdoc/doctest.rs | 48 ++++++++++++++++--- src/librustdoc/doctest/runner.rs | 25 ++++++++-- tests/run-make/rustdoc-should-panic/rmake.rs | 2 +- .../doctest/failed-doctest-should-panic.rs | 13 +++-- .../failed-doctest-should-panic.stdout | 15 ++++-- 5 files changed, 83 insertions(+), 20 deletions(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 28453f990aed2..561a23a22a755 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -465,8 +465,8 @@ enum TestFailure { /// /// This typically means an assertion in the test failed or another form of panic occurred. ExecutionFailure(process::Output), - /// The test is marked `should_panic` but the test binary executed successfully. - UnexpectedRunPass, + /// The test is marked `should_panic` but the test binary didn't panic. + NoPanic(Option), } enum DirState { @@ -812,6 +812,9 @@ fn run_test( // (cfg!(target_family = "wasm") || cfg!(target_os = "zkvm")) // && !cfg!(target_os = "emscripten") // ``` + // + // FIXME: All this code is terrible and doesn't take into account `TargetTuple::TargetJson`. + // If `libtest` doesn't allow to handle this case, we'll need to use a rustc's API instead. && let TargetTuple::TargetTuple(ref s) = rustdoc_options.target && let mut iter = s.split('-') && let Some(arch) = iter.next() @@ -876,12 +879,41 @@ fn run_test( #[cfg(windows)] Some(STATUS_FAIL_FAST_EXCEPTION) => {} #[cfg(unix)] - None if out.status.signal() == Some(SIGABRT) => {} + None => match out.status.signal() { + Some(SIGABRT) => {} + Some(signal) => { + return ( + duration, + Err(TestFailure::NoPanic(Some(format!( + "Test didn't panic, but it's marked `should_panic` (exit signal: {signal}).", + )))), + ); + } + None => { + return ( + duration, + Err(TestFailure::NoPanic(Some(format!( + "Test didn't panic, but it's marked `should_panic` and exited with no error code and no signal.", + )))), + ); + } + }, + #[cfg(not(unix))] + None => return (duration, Err(TestFailure::NoPanic(None))), // Upon an abort, Fuchsia returns the status code // `ZX_TASK_RETCODE_EXCEPTION_KILL`. #[cfg(target_os = "fuchsia")] Some(ZX_TASK_RETCODE_EXCEPTION_KILL) => {} - _ => return (duration, Err(TestFailure::UnexpectedRunPass)), + Some(exit_code) => { + let err_msg = if !out.status.success() { + Some(format!( + "Test didn't panic, but it's marked `should_panic` (exit status: {exit_code}).", + )) + } else { + None + }; + return (duration, Err(TestFailure::NoPanic(err_msg))); + } } } else if !out.status.success() { return (duration, Err(TestFailure::ExecutionFailure(out))); @@ -1190,8 +1222,12 @@ fn doctest_run_fn( TestFailure::UnexpectedCompilePass => { eprint!("Test compiled successfully, but it's marked `compile_fail`."); } - TestFailure::UnexpectedRunPass => { - eprint!("Test didn't panic, but it's marked `should_panic`."); + TestFailure::NoPanic(msg) => { + if let Some(msg) = msg { + eprint!("{msg}"); + } else { + eprint!("Test didn't panic, but it's marked `should_panic`."); + } } TestFailure::MissingErrorCodes(codes) => { eprint!("Some expected error codes were not found: {codes:?}"); diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index afcab202a8e3e..1327b0717a7f5 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -169,12 +169,31 @@ mod __doctest_mod {{ #[cfg(windows)] Some(STATUS_FAIL_FAST_EXCEPTION) => ExitCode::SUCCESS, #[cfg(unix)] - None if out.status.signal() == Some(SIGABRT) => ExitCode::SUCCESS, + None => match out.status.signal() {{ + Some(SIGABRT) => ExitCode::SUCCESS, + Some(signal) => {{ + eprintln!(\"Test didn't panic, but it's marked `should_panic` (exit signal: {{signal}}).\"); + ExitCode::FAILURE + }} + None => {{ + eprintln!(\"Test didn't panic, but it's marked `should_panic` and exited with no error code and no signal.\"); + ExitCode::FAILURE + }} + }}, + #[cfg(not(unix))] + None => {{ + eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); + ExitCode::FAILURE + }} // Upon an abort, Fuchsia returns the status code ZX_TASK_RETCODE_EXCEPTION_KILL. #[cfg(target_os = \"fuchsia\")] Some(ZX_TASK_RETCODE_EXCEPTION_KILL) => ExitCode::SUCCESS, - _ => {{ - eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); + Some(exit_code) => {{ + if !out.status.success() {{ + eprintln!(\"Test didn't panic, but it's marked `should_panic` (exit status: {{exit_code}}).\"); + }} else {{ + eprintln!(\"Test didn't panic, but it's marked `should_panic`.\"); + }} ExitCode::FAILURE }} }} diff --git a/tests/run-make/rustdoc-should-panic/rmake.rs b/tests/run-make/rustdoc-should-panic/rmake.rs index 438608ff8b4ed..0e93440af5ace 100644 --- a/tests/run-make/rustdoc-should-panic/rmake.rs +++ b/tests/run-make/rustdoc-should-panic/rmake.rs @@ -19,7 +19,7 @@ fn check_output(edition: &str, panic_abort: bool) { "---- test.rs - bad_exit_code (line 1) stdout ---- Test executable failed (exit status: 1).", "---- test.rs - did_not_panic (line 6) stdout ---- -Test didn't panic, but it's marked `should_panic`.", +Test didn't panic, but it's marked `should_panic` (exit status: 1).", "test result: FAILED. 1 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out;", ]; for text in should_contain { diff --git a/tests/rustdoc-ui/doctest/failed-doctest-should-panic.rs b/tests/rustdoc-ui/doctest/failed-doctest-should-panic.rs index 0504c3dc73033..b95e23715175f 100644 --- a/tests/rustdoc-ui/doctest/failed-doctest-should-panic.rs +++ b/tests/rustdoc-ui/doctest/failed-doctest-should-panic.rs @@ -2,14 +2,17 @@ // adapted to use that, and that normalize line can go away //@ edition: 2024 -//@ compile-flags:--test +//@ compile-flags:--test --test-args=--test-threads=1 //@ normalize-stdout: "tests/rustdoc-ui/doctest" -> "$$DIR" //@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME" //@ normalize-stdout: "ran in \d+\.\d+s" -> "ran in $$TIME" //@ normalize-stdout: "compilation took \d+\.\d+s" -> "compilation took $$TIME" //@ failure-status: 101 -/// ```should_panic -/// println!("Hello, world!"); -/// ``` -pub struct Foo; +//! ```should_panic +//! println!("Hello, world!"); +//! ``` +//! +//! ```should_panic +//! std::process::exit(2); +//! ``` diff --git a/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout b/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout index 61099e6424ae7..a8e27fcdda2c9 100644 --- a/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout +++ b/tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout @@ -1,16 +1,21 @@ -running 1 test -test $DIR/failed-doctest-should-panic.rs - Foo (line 12) ... FAILED +running 2 tests +test $DIR/failed-doctest-should-panic.rs - (line 12) ... FAILED +test $DIR/failed-doctest-should-panic.rs - (line 16) ... FAILED failures: ----- $DIR/failed-doctest-should-panic.rs - Foo (line 12) stdout ---- +---- $DIR/failed-doctest-should-panic.rs - (line 12) stdout ---- Test didn't panic, but it's marked `should_panic`. +---- $DIR/failed-doctest-should-panic.rs - (line 16) stdout ---- +Test didn't panic, but it's marked `should_panic` (exit status: 2). + failures: - $DIR/failed-doctest-should-panic.rs - Foo (line 12) + $DIR/failed-doctest-should-panic.rs - (line 12) + $DIR/failed-doctest-should-panic.rs - (line 16) -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME +test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME all doctests ran in $TIME; merged doctests compilation took $TIME