wgpu_test/
expectations.rs

1use core::fmt;
2
3/// Conditions under which a test should fail or be skipped.
4///
5/// By passing a `FailureCase` to [`TestParameters::expect_fail`][expect_fail], you can
6/// mark a test as expected to fail under the indicated conditions. By
7/// passing it to [`TestParameters::skip`][skip], you can request that the
8/// test be skipped altogether.
9///
10/// If a field is `None`, then that field does not restrict matches. For
11/// example:
12///
13/// ```
14/// # use wgpu_test::*;
15/// FailureCase {
16///     backends: Some(wgpu::Backends::DX12),
17///     vendor: None,
18///     adapter: Some("RTX"),
19///     driver: None,
20///     reasons: vec![FailureReason::validation_error().with_message("Some error substring")],
21///     behavior: FailureBehavior::AssertFailure,
22/// }
23/// # ;
24/// ```
25///
26/// This applies to all cards with `"RTX'` in their name on either
27/// Direct3D backend, no matter the vendor ID or driver name.
28///
29/// The strings given here need only appear as a substring in the
30/// corresponding [`AdapterInfo`] fields. The comparison is
31/// case-insensitive.
32///
33/// The default value of `FailureCase` applies to any test case. That
34/// is, there are no criteria to constrain the match.
35///
36/// [skip]: super::TestParameters::skip
37/// [expect_fail]: super::TestParameters::expect_fail
38/// [`AdapterInfo`]: wgpu::AdapterInfo
39#[derive(Default, Clone, PartialEq)]
40pub struct FailureCase {
41    /// Backends expected to fail, or `None` for any backend.
42    ///
43    /// If this is `None`, or if the test is using one of the backends
44    /// in `backends`, then this `FailureCase` applies.
45    pub backends: Option<wgpu::Backends>,
46
47    /// Vendor expected to fail, or `None` for any vendor.
48    ///
49    /// If `Some`, this must match [`AdapterInfo::device`], which is
50    /// usually the PCI device id. Otherwise, this `FailureCase`
51    /// applies regardless of vendor.
52    ///
53    /// [`AdapterInfo::device`]: wgpu::AdapterInfo::device
54    pub vendor: Option<u32>,
55
56    /// Name of adapter expected to fail, or `None` for any adapter name.
57    ///
58    /// If this is `Some(s)` and `s` is a substring of
59    /// [`AdapterInfo::name`], then this `FailureCase` applies. If
60    /// this is `None`, the adapter name isn't considered.
61    ///
62    /// [`AdapterInfo::name`]: wgpu::AdapterInfo::name
63    pub adapter: Option<&'static str>,
64
65    /// Name of driver expected to fail, or `None` for any driver name.
66    ///
67    /// If this is `Some(s)` and `s` is a substring of
68    /// [`AdapterInfo::driver`], then this `FailureCase` applies. If
69    /// this is `None`, the driver name isn't considered.
70    ///
71    /// [`AdapterInfo::driver`]: wgpu::AdapterInfo::driver
72    pub driver: Option<&'static str>,
73
74    /// Reason why the test is expected to fail.
75    ///
76    /// If this does not match, the failure will not match this case.
77    ///
78    /// If no reasons are pushed, will match any failure.
79    pub reasons: Vec<FailureReason>,
80
81    /// Behavior after this case matches a failure.
82    pub behavior: FailureBehavior,
83}
84
85impl FailureCase {
86    /// Create a new failure case.
87    pub fn new() -> Self {
88        Self::default()
89    }
90
91    /// This case applies to all tests.
92    pub fn always() -> Self {
93        FailureCase::default()
94    }
95
96    /// This case applies to no tests.
97    pub fn never() -> Self {
98        FailureCase {
99            backends: Some(wgpu::Backends::empty()),
100            ..FailureCase::default()
101        }
102    }
103
104    /// Tests running on any of the given backends.
105    pub fn backend(backends: wgpu::Backends) -> Self {
106        FailureCase {
107            backends: Some(backends),
108            ..FailureCase::default()
109        }
110    }
111
112    /// Tests running on `adapter`.
113    ///
114    /// For this case to apply, the `adapter` string must appear as a substring
115    /// of the adapter's [`AdapterInfo::name`]. The comparison is
116    /// case-insensitive.
117    ///
118    /// [`AdapterInfo::name`]: wgpu::AdapterInfo::name
119    pub fn adapter(adapter: &'static str) -> Self {
120        FailureCase {
121            adapter: Some(adapter),
122            ..FailureCase::default()
123        }
124    }
125
126    /// Tests running on `backend` and `adapter`.
127    ///
128    /// For this case to apply, the test must be using an adapter for one of the
129    /// given `backend` bits, and `adapter` string must appear as a substring of
130    /// the adapter's [`AdapterInfo::name`]. The string comparison is
131    /// case-insensitive.
132    ///
133    /// [`AdapterInfo::name`]: wgpu::AdapterInfo::name
134    pub fn backend_adapter(backends: wgpu::Backends, adapter: &'static str) -> Self {
135        FailureCase {
136            backends: Some(backends),
137            adapter: Some(adapter),
138            ..FailureCase::default()
139        }
140    }
141
142    /// Tests running under WebGL.
143    pub fn webgl2() -> Self {
144        #[cfg(target_arch = "wasm32")]
145        let case = FailureCase::backend(wgpu::Backends::GL);
146        #[cfg(not(target_arch = "wasm32"))]
147        let case = FailureCase::never();
148        case
149    }
150
151    /// Tests running on the MoltenVK Vulkan driver on macOS.
152    pub fn molten_vk() -> Self {
153        FailureCase {
154            backends: Some(wgpu::Backends::VULKAN),
155            driver: Some("MoltenVK"),
156            ..FailureCase::default()
157        }
158    }
159
160    /// Return the reasons why this case should fail.
161    pub fn reasons(&self) -> &[FailureReason] {
162        if self.reasons.is_empty() {
163            std::array::from_ref(&FailureReason::ANY)
164        } else {
165            &self.reasons
166        }
167    }
168
169    /// Matches this failure case against the given validation error substring.
170    ///
171    /// Substrings are matched case-insensitively.
172    ///
173    /// If multiple reasons are pushed, will match any of them.
174    pub fn validation_error(mut self, msg: &'static str) -> Self {
175        self.reasons
176            .push(FailureReason::validation_error().with_message(msg));
177        self
178    }
179
180    /// Matches this failure case against the given panic substring.
181    ///
182    /// Substrings are matched case-insensitively.
183    ///
184    /// If multiple reasons are pushed, will match any of them.
185    pub fn panic(mut self, msg: &'static str) -> Self {
186        self.reasons.push(FailureReason::panic().with_message(msg));
187        self
188    }
189
190    /// Test is flaky with the given configuration. Do not assert failure.
191    ///
192    /// Use this _very_ sparyingly, and match as tightly as you can, including giving a specific failure message.
193    pub fn flaky(self) -> Self {
194        FailureCase {
195            behavior: FailureBehavior::Ignore,
196            ..self
197        }
198    }
199
200    /// Test whether `self` applies to `info`.
201    ///
202    /// If it does, return a `FailureReasons` whose set bits indicate
203    /// why. If it doesn't, return `None`.
204    ///
205    /// The caller is responsible for converting the string-valued
206    /// fields of `info` to lower case, to ensure case-insensitive
207    /// matching.
208    pub(crate) fn applies_to_adapter(
209        &self,
210        info: &wgpu::AdapterInfo,
211    ) -> Option<FailureApplicationReasons> {
212        let mut reasons = FailureApplicationReasons::empty();
213
214        if let Some(backends) = self.backends {
215            if !backends.contains(wgpu::Backends::from(info.backend)) {
216                return None;
217            }
218            reasons.set(FailureApplicationReasons::BACKEND, true);
219        }
220        if let Some(vendor) = self.vendor {
221            if vendor != info.vendor {
222                return None;
223            }
224            reasons.set(FailureApplicationReasons::VENDOR, true);
225        }
226        if let Some(adapter) = self.adapter {
227            let adapter = adapter.to_lowercase();
228            if !info.name.contains(&adapter) {
229                return None;
230            }
231            reasons.set(FailureApplicationReasons::ADAPTER, true);
232        }
233        if let Some(driver) = self.driver {
234            let driver = driver.to_lowercase();
235            if !info.driver.contains(&driver) {
236                return None;
237            }
238            reasons.set(FailureApplicationReasons::DRIVER, true);
239        }
240
241        // If we got this far but no specific reasons were triggered, then this
242        // must be a wildcard.
243        if reasons.is_empty() {
244            Some(FailureApplicationReasons::ALWAYS)
245        } else {
246            Some(reasons)
247        }
248    }
249
250    /// Returns true if the given failure "satisfies" this failure case.
251    pub(crate) fn matches_failure(&self, failure: &FailureResult) -> bool {
252        for reason in self.reasons() {
253            let kind_matched = reason.kind.is_none_or(|kind| kind == failure.kind);
254
255            let message_matched =
256                reason
257                    .message
258                    .is_none_or(|message| matches!(&failure.message, Some(actual) if actual.to_lowercase().contains(&message.to_lowercase())));
259
260            if kind_matched && message_matched {
261                let message = failure.message.as_deref().unwrap_or("*no message*");
262                log::error!("Matched {} {message}", failure.kind);
263                return true;
264            }
265        }
266
267        false
268    }
269}
270
271bitflags::bitflags! {
272    /// Reason why a test matches a given failure case.
273    #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
274    pub struct FailureApplicationReasons: u8 {
275        const BACKEND = 1 << 0;
276        const VENDOR = 1 << 1;
277        const ADAPTER = 1 << 2;
278        const DRIVER = 1 << 3;
279        const ALWAYS = 1 << 4;
280    }
281}
282
283/// Reason why a test is expected to fail.
284///
285/// If the test fails for a different reason, the given FailureCase will be ignored.
286#[derive(Default, Debug, Clone, PartialEq)]
287pub struct FailureReason {
288    /// Match a particular kind of failure result.
289    ///
290    /// If `None`, match any result kind.
291    kind: Option<FailureResultKind>,
292    /// Match a particular message of a failure result.
293    ///
294    /// If `None`, matches any message. If `Some`, a case-insensitive sub-string
295    /// test is performed. Allowing `"error occurred"` to match a message like
296    /// `"An unexpected Error occurred!"`.
297    message: Option<&'static str>,
298}
299
300impl FailureReason {
301    /// Match any failure reason.
302    const ANY: Self = Self {
303        kind: None,
304        message: None,
305    };
306
307    /// Match a validation error.
308    #[allow(dead_code)] // Not constructed on wasm
309    pub fn validation_error() -> Self {
310        Self {
311            kind: Some(FailureResultKind::ValidationError),
312            message: None,
313        }
314    }
315
316    /// Match a panic.
317    pub fn panic() -> Self {
318        Self {
319            kind: Some(FailureResultKind::Panic),
320            message: None,
321        }
322    }
323
324    /// Match an error with a message.
325    ///
326    /// If specified, a case-insensitive sub-string test is performed. Allowing
327    /// `"error occurred"` to match a message like `"An unexpected Error
328    /// occurred!"`.
329    pub fn with_message(self, message: &'static str) -> Self {
330        Self {
331            message: Some(message),
332            ..self
333        }
334    }
335}
336
337#[derive(Default, Clone, PartialEq)]
338pub enum FailureBehavior {
339    /// Assert that the test fails for the given reason.
340    ///
341    /// If the test passes, the test harness will panic.
342    #[default]
343    AssertFailure,
344    /// Ignore the matching failure.
345    ///
346    /// This is useful for tests that flake in a very specific way,
347    /// but sometimes succeed, so we can't assert that they always fail.
348    Ignore,
349}
350
351#[derive(Debug, Clone, Copy, PartialEq)]
352pub(crate) enum FailureResultKind {
353    #[allow(dead_code)] // Not constructed on wasm
354    ValidationError,
355    Panic,
356}
357
358impl fmt::Display for FailureResultKind {
359    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
360        match self {
361            FailureResultKind::ValidationError => write!(f, "Validation Error"),
362            FailureResultKind::Panic => write!(f, "Panic"),
363        }
364    }
365}
366
367#[derive(Debug)]
368pub(crate) struct FailureResult {
369    kind: FailureResultKind,
370    message: Option<String>,
371}
372
373impl FailureResult {
374    /// Failure result is a panic.
375    pub(super) fn panic() -> Self {
376        Self {
377            kind: FailureResultKind::Panic,
378            message: None,
379        }
380    }
381
382    /// Failure result is a validation error.
383    #[allow(dead_code)] // Not constructed on wasm
384    pub(super) fn validation_error() -> Self {
385        Self {
386            kind: FailureResultKind::ValidationError,
387            message: None,
388        }
389    }
390
391    /// Message associated with a failure result.
392    pub(super) fn with_message(self, message: impl fmt::Display) -> Self {
393        Self {
394            kind: self.kind,
395            message: Some(message.to_string()),
396        }
397    }
398}
399
400#[derive(PartialEq, Clone, Copy, Debug)]
401pub(crate) enum ExpectationMatchResult {
402    Panic,
403    Complete,
404}
405
406/// Compares if the actual failures match the expected failures.
407pub(crate) fn expectations_match_failures(
408    expectations: &[FailureCase],
409    mut actual: Vec<FailureResult>,
410) -> ExpectationMatchResult {
411    // Start with the assumption that we will pass.
412    let mut result = ExpectationMatchResult::Complete;
413
414    // Run through all expected failures.
415    for expected_failure in expectations {
416        // If any of the failures match.
417        let mut matched = false;
418
419        // Iterate through the failures.
420        //
421        // In reverse, to be able to use swap_remove.
422        actual.retain(|failure| {
423            // If the failure matches, remove it from the list of failures, as we expected it.
424            let matches = expected_failure.matches_failure(failure);
425
426            if matches {
427                matched = true;
428            }
429
430            // Retain removes on false, so flip the bool so we remove on failure.
431            !matches
432        });
433
434        // If we didn't match our expected failure against any of the actual failures,
435        // and this failure is not flaky, then we need to panic, as we got an unexpected success.
436        if !matched && matches!(expected_failure.behavior, FailureBehavior::AssertFailure) {
437            result = ExpectationMatchResult::Panic;
438            log::error!(
439                "Expected to fail due to {:?}, but did not fail",
440                expected_failure.reasons()
441            );
442        }
443    }
444
445    // If we have any failures left, then we got an unexpected failure
446    // and we need to panic.
447    if !actual.is_empty() {
448        result = ExpectationMatchResult::Panic;
449        for failure in actual {
450            let message = failure.message.as_deref().unwrap_or("*no message*");
451            log::error!("{}: {message}", failure.kind);
452        }
453    }
454
455    result
456}
457
458#[cfg(test)]
459mod test {
460    use crate::{
461        expectations::{ExpectationMatchResult, FailureResult},
462        init::init_logger,
463        FailureCase,
464    };
465
466    fn validation_err(msg: &'static str) -> FailureResult {
467        FailureResult::validation_error().with_message(msg)
468    }
469
470    fn panic(msg: &'static str) -> FailureResult {
471        FailureResult::panic().with_message(msg)
472    }
473
474    #[test]
475    fn simple_match() {
476        init_logger();
477
478        // -- Unexpected failure --
479
480        let expectation = vec![];
481        let actual = vec![FailureResult::validation_error()];
482
483        assert_eq!(
484            super::expectations_match_failures(&expectation, actual),
485            ExpectationMatchResult::Panic
486        );
487
488        // -- Missing expected failure --
489
490        let expectation = vec![FailureCase::always()];
491        let actual = vec![];
492
493        assert_eq!(
494            super::expectations_match_failures(&expectation, actual),
495            ExpectationMatchResult::Panic
496        );
497
498        // -- Expected failure (validation) --
499
500        let expectation = vec![FailureCase::always()];
501        let actual = vec![FailureResult::validation_error()];
502
503        assert_eq!(
504            super::expectations_match_failures(&expectation, actual),
505            ExpectationMatchResult::Complete
506        );
507
508        // -- Expected failure (panic) --
509
510        let expectation = vec![FailureCase::always()];
511        let actual = vec![FailureResult::panic()];
512
513        assert_eq!(
514            super::expectations_match_failures(&expectation, actual),
515            ExpectationMatchResult::Complete
516        );
517    }
518
519    #[test]
520    fn substring_match() {
521        init_logger();
522
523        // -- Matching Substring --
524
525        let expectation: Vec<FailureCase> =
526            vec![FailureCase::always().validation_error("Some StrIng")];
527        let actual = vec![FailureResult::validation_error().with_message(
528            "a very long string that contains sOmE sTrInG of different capitalization",
529        )];
530
531        assert_eq!(
532            super::expectations_match_failures(&expectation, actual),
533            ExpectationMatchResult::Complete
534        );
535
536        // -- Non-Matching Substring --
537
538        let expectation = vec![FailureCase::always().validation_error("Some String")];
539        let actual = vec![validation_err("a very long string that doesn't contain it")];
540
541        assert_eq!(
542            super::expectations_match_failures(&expectation, actual),
543            ExpectationMatchResult::Panic
544        );
545    }
546
547    #[test]
548    fn ignore_flaky() {
549        init_logger();
550
551        let expectation = vec![FailureCase::always().validation_error("blah").flaky()];
552        let actual = vec![validation_err("some blah")];
553
554        assert_eq!(
555            super::expectations_match_failures(&expectation, actual),
556            ExpectationMatchResult::Complete
557        );
558
559        let expectation = vec![FailureCase::always().validation_error("blah").flaky()];
560        let actual = vec![];
561
562        assert_eq!(
563            super::expectations_match_failures(&expectation, actual),
564            ExpectationMatchResult::Complete
565        );
566    }
567
568    #[test]
569    fn matches_multiple_errors() {
570        init_logger();
571
572        // -- matches all matching errors --
573
574        let expectation = vec![FailureCase::always().validation_error("blah")];
575        let actual = vec![
576            validation_err("some blah"),
577            validation_err("some other blah"),
578        ];
579
580        assert_eq!(
581            super::expectations_match_failures(&expectation, actual),
582            ExpectationMatchResult::Complete
583        );
584
585        // -- but not all errors --
586
587        let expectation = vec![FailureCase::always().validation_error("blah")];
588        let actual = vec![
589            validation_err("some blah"),
590            validation_err("some other blah"),
591            validation_err("something else"),
592        ];
593
594        assert_eq!(
595            super::expectations_match_failures(&expectation, actual),
596            ExpectationMatchResult::Panic
597        );
598    }
599
600    #[test]
601    fn multi_reason_error() {
602        init_logger();
603
604        let expectation = vec![FailureCase::default()
605            .validation_error("blah")
606            .panic("panik")];
607        let actual = vec![
608            validation_err("my blah blah validation error"),
609            panic("my panik"),
610        ];
611
612        assert_eq!(
613            super::expectations_match_failures(&expectation, actual),
614            ExpectationMatchResult::Complete
615        );
616    }
617}