wgpu_test/
expectations.rs

1use core::fmt;
2
3/// Conditions under which a test should fail or be skipped.
4///
5/// By passing a `FailureCase` to [`TestParameters::expect_fail`][expect_fail], you can
6/// mark a test as expected to fail under the indicated conditions. By
7/// passing it to [`TestParameters::skip`][skip], you can request that the
8/// test be skipped altogether.
9///
10/// If a field is `None`, then that field does not restrict matches. For
11/// example:
12///
13/// ```
14/// # use wgpu_test::*;
15/// FailureCase {
16///     backends: Some(wgpu::Backends::DX12),
17///     vendor: None,
18///     adapter: Some("RTX"),
19///     driver: None,
20///     reasons: vec![FailureReason::validation_error().with_message("Some error substring")],
21///     behavior: FailureBehavior::AssertFailure,
22/// }
23/// # ;
24/// ```
25///
26/// This applies to all cards with `"RTX'` in their name on either
27/// Direct3D backend, no matter the vendor ID or driver name.
28///
29/// The strings given here need only appear as a substring in the
30/// corresponding [`AdapterInfo`] fields. The comparison is
31/// case-insensitive.
32///
33/// The default value of `FailureCase` applies to any test case. That
34/// is, there are no criteria to constrain the match.
35///
36/// [skip]: super::TestParameters::skip
37/// [expect_fail]: super::TestParameters::expect_fail
38/// [`AdapterInfo`]: wgpu::AdapterInfo
39#[derive(Default, Clone, PartialEq)]
40pub struct FailureCase {
41    /// Backends expected to fail, or `None` for any backend.
42    ///
43    /// If this is `None`, or if the test is using one of the backends
44    /// in `backends`, then this `FailureCase` applies.
45    pub backends: Option<wgpu::Backends>,
46
47    /// Vendor expected to fail, or `None` for any vendor.
48    ///
49    /// If `Some`, this must match [`AdapterInfo::device`], which is
50    /// usually the PCI device id. Otherwise, this `FailureCase`
51    /// applies regardless of vendor.
52    ///
53    /// [`AdapterInfo::device`]: wgpu::AdapterInfo::device
54    pub vendor: Option<u32>,
55
56    /// Name of adapter expected to fail, or `None` for any adapter name.
57    ///
58    /// If this is `Some(s)` and `s` is a substring of
59    /// [`AdapterInfo::name`], then this `FailureCase` applies. If
60    /// this is `None`, the adapter name isn't considered.
61    ///
62    /// [`AdapterInfo::name`]: wgpu::AdapterInfo::name
63    pub adapter: Option<&'static str>,
64
65    /// Name of driver expected to fail, or `None` for any driver name.
66    ///
67    /// If this is `Some(s)` and `s` is a substring of
68    /// [`AdapterInfo::driver`], then this `FailureCase` applies. If
69    /// this is `None`, the driver name isn't considered.
70    ///
71    /// [`AdapterInfo::driver`]: wgpu::AdapterInfo::driver
72    pub driver: Option<&'static str>,
73
74    /// Reason why the test is expected to fail.
75    ///
76    /// If this does not match, the failure will not match this case.
77    ///
78    /// If no reasons are pushed, will match any failure.
79    pub reasons: Vec<FailureReason>,
80
81    /// Behavior after this case matches a failure.
82    pub behavior: FailureBehavior,
83}
84
85impl FailureCase {
86    /// Create a new failure case.
87    pub fn new() -> Self {
88        Self::default()
89    }
90
91    /// This case applies to all tests.
92    pub fn always() -> Self {
93        FailureCase::default()
94    }
95
96    /// This case applies to no tests.
97    pub fn never() -> Self {
98        FailureCase {
99            backends: Some(wgpu::Backends::empty()),
100            ..FailureCase::default()
101        }
102    }
103
104    /// Tests running on any of the given backends.
105    pub fn backend(backends: wgpu::Backends) -> Self {
106        FailureCase {
107            backends: Some(backends),
108            ..FailureCase::default()
109        }
110    }
111
112    /// Tests running on `adapter`.
113    ///
114    /// For this case to apply, the `adapter` string must appear as a substring
115    /// of the adapter's [`AdapterInfo::name`]. The comparison is
116    /// case-insensitive.
117    ///
118    /// [`AdapterInfo::name`]: wgpu::AdapterInfo::name
119    pub fn adapter(adapter: &'static str) -> Self {
120        FailureCase {
121            adapter: Some(adapter),
122            ..FailureCase::default()
123        }
124    }
125
126    /// Tests running on `backend` and `adapter`.
127    ///
128    /// For this case to apply, the test must be using an adapter for one of the
129    /// given `backend` bits, and `adapter` string must appear as a substring of
130    /// the adapter's [`AdapterInfo::name`]. The string comparison is
131    /// case-insensitive.
132    ///
133    /// [`AdapterInfo::name`]: wgpu::AdapterInfo::name
134    pub fn backend_adapter(backends: wgpu::Backends, adapter: &'static str) -> Self {
135        FailureCase {
136            backends: Some(backends),
137            adapter: Some(adapter),
138            ..FailureCase::default()
139        }
140    }
141
142    /// Tests running under WebGL.
143    pub fn webgl2() -> Self {
144        #[cfg(target_arch = "wasm32")]
145        let case = FailureCase::backend(wgpu::Backends::GL);
146        #[cfg(not(target_arch = "wasm32"))]
147        let case = FailureCase::never();
148        case
149    }
150
151    /// Tests running on the MoltenVK Vulkan driver on macOS.
152    pub fn molten_vk() -> Self {
153        FailureCase {
154            backends: Some(wgpu::Backends::VULKAN),
155            driver: Some("MoltenVK"),
156            ..FailureCase::default()
157        }
158    }
159
160    /// Tests running on the KosmicKrisp Vulkan driver on macOS.
161    pub fn kosmic_krisp() -> Self {
162        FailureCase {
163            backends: Some(wgpu::Backends::VULKAN),
164            driver: Some("KosmicKrisp"),
165            ..FailureCase::default()
166        }
167    }
168
169    /// Tests running on either Vulkan driver on macOS.
170    pub fn mac_vulkan(f: impl Fn(FailureCase) -> FailureCase) -> Vec<Self> {
171        vec![f(FailureCase::molten_vk()), f(FailureCase::kosmic_krisp())]
172    }
173
174    /// Return the reasons why this case should fail.
175    pub fn reasons(&self) -> &[FailureReason] {
176        if self.reasons.is_empty() {
177            std::array::from_ref(&FailureReason::ANY)
178        } else {
179            &self.reasons
180        }
181    }
182
183    /// Matches this failure case against the given validation error substring.
184    ///
185    /// Substrings are matched case-insensitively.
186    ///
187    /// If multiple reasons are pushed, will match any of them.
188    pub fn validation_error(mut self, msg: &'static str) -> Self {
189        self.reasons
190            .push(FailureReason::validation_error().with_message(msg));
191        self
192    }
193
194    /// Matches this failure case against the given panic substring.
195    ///
196    /// Substrings are matched case-insensitively.
197    ///
198    /// If multiple reasons are pushed, will match any of them.
199    pub fn panic(mut self, msg: &'static str) -> Self {
200        self.reasons.push(FailureReason::panic().with_message(msg));
201        self
202    }
203
204    /// Matches this failure case against an unexpected driver error.
205    ///
206    /// Depending on build configuration, the error may surface as either a
207    /// panic raised by the `internal_error_panic` feature (with the supplied
208    /// message as a substring), or as a device-loss. Either behavior is
209    /// accepted. In the device loss case, the original message has been
210    /// discarded, and there is some risk this accepts a different error than
211    /// intended (but it must be a device loss due to an unexpected driver
212    /// error, which should be rare).
213    pub fn unexpected_error(mut self, msg: &'static str) -> Self {
214        self.reasons.push(FailureReason::panic().with_message(msg));
215        self.reasons.push(FailureReason::panic().with_message(
216            "Device lost: Unexpected error variant (driver implementation is at fault)",
217        ));
218        self
219    }
220
221    /// Test is flaky with the given configuration. Do not assert failure.
222    ///
223    /// Use this _very_ sparyingly, and match as tightly as you can, including giving a specific failure message.
224    pub fn flaky(self) -> Self {
225        FailureCase {
226            behavior: FailureBehavior::Ignore,
227            ..self
228        }
229    }
230
231    /// Test whether `self` applies to `info`.
232    ///
233    /// If it does, return a `FailureReasons` whose set bits indicate
234    /// why. If it doesn't, return `None`.
235    ///
236    /// The caller is responsible for converting the string-valued
237    /// fields of `info` to lower case, to ensure case-insensitive
238    /// matching.
239    pub(crate) fn applies_to_adapter(
240        &self,
241        info: &wgpu::AdapterInfo,
242    ) -> Option<FailureApplicationReasons> {
243        let mut reasons = FailureApplicationReasons::empty();
244
245        if let Some(backends) = self.backends {
246            if !backends.contains(wgpu::Backends::from(info.backend)) {
247                return None;
248            }
249            reasons.set(FailureApplicationReasons::BACKEND, true);
250        }
251        if let Some(vendor) = self.vendor {
252            if vendor != info.vendor {
253                return None;
254            }
255            reasons.set(FailureApplicationReasons::VENDOR, true);
256        }
257        if let Some(adapter) = self.adapter {
258            let adapter = adapter.to_lowercase();
259            if !info.name.contains(&adapter) {
260                return None;
261            }
262            reasons.set(FailureApplicationReasons::ADAPTER, true);
263        }
264        if let Some(driver) = self.driver {
265            let driver = driver.to_lowercase();
266            if !info.driver.contains(&driver) {
267                return None;
268            }
269            reasons.set(FailureApplicationReasons::DRIVER, true);
270        }
271
272        // If we got this far but no specific reasons were triggered, then this
273        // must be a wildcard.
274        if reasons.is_empty() {
275            Some(FailureApplicationReasons::ALWAYS)
276        } else {
277            Some(reasons)
278        }
279    }
280
281    /// Returns true if the given failure "satisfies" this failure case.
282    pub(crate) fn matches_failure(&self, failure: &FailureResult) -> bool {
283        for reason in self.reasons() {
284            let kind_matched = reason.kind.is_none_or(|kind| kind == failure.kind);
285
286            let message_matched =
287                reason
288                    .message
289                    .is_none_or(|message| matches!(&failure.message, Some(actual) if actual.to_lowercase().contains(&message.to_lowercase())));
290
291            if kind_matched && message_matched {
292                let message = failure.message.as_deref().unwrap_or("*no message*");
293                log::error!("Matched {} {message}", failure.kind);
294                return true;
295            }
296        }
297
298        false
299    }
300}
301
302bitflags::bitflags! {
303    /// Reason why a test matches a given failure case.
304    #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
305    pub struct FailureApplicationReasons: u8 {
306        const BACKEND = 1 << 0;
307        const VENDOR = 1 << 1;
308        const ADAPTER = 1 << 2;
309        const DRIVER = 1 << 3;
310        const ALWAYS = 1 << 4;
311    }
312}
313
314/// Reason why a test is expected to fail.
315///
316/// If the test fails for a different reason, the given FailureCase will be ignored.
317#[derive(Default, Debug, Clone, PartialEq)]
318pub struct FailureReason {
319    /// Match a particular kind of failure result.
320    ///
321    /// If `None`, match any result kind.
322    kind: Option<FailureResultKind>,
323    /// Match a particular message of a failure result.
324    ///
325    /// If `None`, matches any message. If `Some`, a case-insensitive sub-string
326    /// test is performed. Allowing `"error occurred"` to match a message like
327    /// `"An unexpected Error occurred!"`.
328    message: Option<&'static str>,
329}
330
331impl FailureReason {
332    /// Match any failure reason.
333    const ANY: Self = Self {
334        kind: None,
335        message: None,
336    };
337
338    /// Match a validation error.
339    #[allow(dead_code, reason = "Not constructed on wasm")]
340    pub fn validation_error() -> Self {
341        Self {
342            kind: Some(FailureResultKind::ValidationError),
343            message: None,
344        }
345    }
346
347    /// Match a panic.
348    pub fn panic() -> Self {
349        Self {
350            kind: Some(FailureResultKind::Panic),
351            message: None,
352        }
353    }
354
355    /// Match an error with a message.
356    ///
357    /// If specified, a case-insensitive sub-string test is performed. Allowing
358    /// `"error occurred"` to match a message like `"An unexpected Error
359    /// occurred!"`.
360    pub fn with_message(self, message: &'static str) -> Self {
361        Self {
362            message: Some(message),
363            ..self
364        }
365    }
366}
367
368#[derive(Default, Clone, PartialEq)]
369pub enum FailureBehavior {
370    /// Assert that the test fails for the given reason.
371    ///
372    /// If the test passes, the test harness will panic.
373    #[default]
374    AssertFailure,
375    /// Ignore the matching failure.
376    ///
377    /// This is useful for tests that flake in a very specific way,
378    /// but sometimes succeed, so we can't assert that they always fail.
379    Ignore,
380}
381
382#[derive(Debug, Clone, Copy, PartialEq)]
383pub(crate) enum FailureResultKind {
384    #[allow(dead_code, reason = "Not constructed on wasm")]
385    ValidationError,
386    Panic,
387}
388
389impl fmt::Display for FailureResultKind {
390    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
391        match self {
392            FailureResultKind::ValidationError => write!(f, "Validation Error"),
393            FailureResultKind::Panic => write!(f, "Panic"),
394        }
395    }
396}
397
398#[derive(Debug)]
399pub(crate) struct FailureResult {
400    kind: FailureResultKind,
401    message: Option<String>,
402}
403
404impl FailureResult {
405    /// Failure result is a panic.
406    pub(super) fn panic() -> Self {
407        Self {
408            kind: FailureResultKind::Panic,
409            message: None,
410        }
411    }
412
413    /// Failure result is a validation error.
414    #[allow(dead_code, reason = "Not constructed on wasm")]
415    pub(super) fn validation_error() -> Self {
416        Self {
417            kind: FailureResultKind::ValidationError,
418            message: None,
419        }
420    }
421
422    /// Message associated with a failure result.
423    pub(super) fn with_message(self, message: impl fmt::Display) -> Self {
424        Self {
425            kind: self.kind,
426            message: Some(message.to_string()),
427        }
428    }
429}
430
431#[derive(PartialEq, Clone, Copy, Debug)]
432pub(crate) enum ExpectationMatchResult {
433    Panic,
434    Complete,
435}
436
437/// Compares if the actual failures match the expected failures.
438pub(crate) fn expectations_match_failures(
439    expectations: &[FailureCase],
440    mut actual: Vec<FailureResult>,
441) -> ExpectationMatchResult {
442    // Start with the assumption that we will pass.
443    let mut result = ExpectationMatchResult::Complete;
444
445    // Run through all expected failures.
446    for expected_failure in expectations {
447        // If any of the failures match.
448        let mut matched = false;
449
450        // Iterate through the failures.
451        //
452        // In reverse, to be able to use swap_remove.
453        actual.retain(|failure| {
454            // If the failure matches, remove it from the list of failures, as we expected it.
455            let matches = expected_failure.matches_failure(failure);
456
457            if matches {
458                matched = true;
459            }
460
461            // Retain removes on false, so flip the bool so we remove on failure.
462            !matches
463        });
464
465        // If we didn't match our expected failure against any of the actual failures,
466        // and this failure is not flaky, then we need to panic, as we got an unexpected success.
467        if !matched && matches!(expected_failure.behavior, FailureBehavior::AssertFailure) {
468            result = ExpectationMatchResult::Panic;
469            log::error!(
470                "Expected to fail due to {:?}, but did not fail",
471                expected_failure.reasons()
472            );
473        }
474    }
475
476    // If we have any failures left, then we got an unexpected failure
477    // and we need to panic.
478    if !actual.is_empty() {
479        result = ExpectationMatchResult::Panic;
480        for failure in actual {
481            let message = failure.message.as_deref().unwrap_or("*no message*");
482            log::error!("{}: {message}", failure.kind);
483        }
484    }
485
486    result
487}
488
489#[cfg(test)]
490mod test {
491    use crate::{
492        expectations::{ExpectationMatchResult, FailureResult},
493        init::init_logger,
494        FailureCase,
495    };
496
497    fn validation_err(msg: &'static str) -> FailureResult {
498        FailureResult::validation_error().with_message(msg)
499    }
500
501    fn panic(msg: &'static str) -> FailureResult {
502        FailureResult::panic().with_message(msg)
503    }
504
505    #[test]
506    fn simple_match() {
507        init_logger();
508
509        // -- Unexpected failure --
510
511        let expectation = vec![];
512        let actual = vec![FailureResult::validation_error()];
513
514        assert_eq!(
515            super::expectations_match_failures(&expectation, actual),
516            ExpectationMatchResult::Panic
517        );
518
519        // -- Missing expected failure --
520
521        let expectation = vec![FailureCase::always()];
522        let actual = vec![];
523
524        assert_eq!(
525            super::expectations_match_failures(&expectation, actual),
526            ExpectationMatchResult::Panic
527        );
528
529        // -- Expected failure (validation) --
530
531        let expectation = vec![FailureCase::always()];
532        let actual = vec![FailureResult::validation_error()];
533
534        assert_eq!(
535            super::expectations_match_failures(&expectation, actual),
536            ExpectationMatchResult::Complete
537        );
538
539        // -- Expected failure (panic) --
540
541        let expectation = vec![FailureCase::always()];
542        let actual = vec![FailureResult::panic()];
543
544        assert_eq!(
545            super::expectations_match_failures(&expectation, actual),
546            ExpectationMatchResult::Complete
547        );
548    }
549
550    #[test]
551    fn substring_match() {
552        init_logger();
553
554        // -- Matching Substring --
555
556        let expectation: Vec<FailureCase> =
557            vec![FailureCase::always().validation_error("Some StrIng")];
558        let actual = vec![FailureResult::validation_error().with_message(
559            "a very long string that contains sOmE sTrInG of different capitalization",
560        )];
561
562        assert_eq!(
563            super::expectations_match_failures(&expectation, actual),
564            ExpectationMatchResult::Complete
565        );
566
567        // -- Non-Matching Substring --
568
569        let expectation = vec![FailureCase::always().validation_error("Some String")];
570        let actual = vec![validation_err("a very long string that doesn't contain it")];
571
572        assert_eq!(
573            super::expectations_match_failures(&expectation, actual),
574            ExpectationMatchResult::Panic
575        );
576    }
577
578    #[test]
579    fn ignore_flaky() {
580        init_logger();
581
582        let expectation = vec![FailureCase::always().validation_error("blah").flaky()];
583        let actual = vec![validation_err("some blah")];
584
585        assert_eq!(
586            super::expectations_match_failures(&expectation, actual),
587            ExpectationMatchResult::Complete
588        );
589
590        let expectation = vec![FailureCase::always().validation_error("blah").flaky()];
591        let actual = vec![];
592
593        assert_eq!(
594            super::expectations_match_failures(&expectation, actual),
595            ExpectationMatchResult::Complete
596        );
597    }
598
599    #[test]
600    fn matches_multiple_errors() {
601        init_logger();
602
603        // -- matches all matching errors --
604
605        let expectation = vec![FailureCase::always().validation_error("blah")];
606        let actual = vec![
607            validation_err("some blah"),
608            validation_err("some other blah"),
609        ];
610
611        assert_eq!(
612            super::expectations_match_failures(&expectation, actual),
613            ExpectationMatchResult::Complete
614        );
615
616        // -- but not all errors --
617
618        let expectation = vec![FailureCase::always().validation_error("blah")];
619        let actual = vec![
620            validation_err("some blah"),
621            validation_err("some other blah"),
622            validation_err("something else"),
623        ];
624
625        assert_eq!(
626            super::expectations_match_failures(&expectation, actual),
627            ExpectationMatchResult::Panic
628        );
629    }
630
631    #[test]
632    fn multi_reason_error() {
633        init_logger();
634
635        let expectation = vec![FailureCase::default()
636            .validation_error("blah")
637            .panic("panik")];
638        let actual = vec![
639            validation_err("my blah blah validation error"),
640            panic("my panik"),
641        ];
642
643        assert_eq!(
644            super::expectations_match_failures(&expectation, actual),
645            ExpectationMatchResult::Complete
646        );
647    }
648}
wgpu_test/expectations.rs

wgpu_test/
expectations.rs
