1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
use core::fmt;

/// Conditions under which a test should fail or be skipped.
///
/// By passing a `FailureCase` to [`TestParameters::expect_fail`][expect_fail], you can
/// mark a test as expected to fail under the indicated conditions. By
/// passing it to [`TestParameters::skip`][skip], you can request that the
/// test be skipped altogether.
///
/// If a field is `None`, then that field does not restrict matches. For
/// example:
///
/// ```
/// # use wgpu_test::*;
/// FailureCase {
///     backends: Some(wgpu::Backends::DX12),
///     vendor: None,
///     adapter: Some("RTX"),
///     driver: None,
///     reasons: vec![FailureReason::validation_error().with_message("Some error substring")],
///     behavior: FailureBehavior::AssertFailure,
/// }
/// # ;
/// ```
///
/// This applies to all cards with `"RTX'` in their name on either
/// Direct3D backend, no matter the vendor ID or driver name.
///
/// The strings given here need only appear as a substring in the
/// corresponding [`AdapterInfo`] fields. The comparison is
/// case-insensitive.
///
/// The default value of `FailureCase` applies to any test case. That
/// is, there are no criteria to constrain the match.
///
/// [skip]: super::TestParameters::skip
/// [expect_fail]: super::TestParameters::expect_fail
/// [`AdapterInfo`]: wgt::AdapterInfo
#[derive(Default, Clone)]
pub struct FailureCase {
    /// Backends expected to fail, or `None` for any backend.
    ///
    /// If this is `None`, or if the test is using one of the backends
    /// in `backends`, then this `FailureCase` applies.
    pub backends: Option<wgpu::Backends>,

    /// Vendor expected to fail, or `None` for any vendor.
    ///
    /// If `Some`, this must match [`AdapterInfo::device`], which is
    /// usually the PCI device id. Otherwise, this `FailureCase`
    /// applies regardless of vendor.
    ///
    /// [`AdapterInfo::device`]: wgt::AdapterInfo::device
    pub vendor: Option<u32>,

    /// Name of adapter expected to fail, or `None` for any adapter name.
    ///
    /// If this is `Some(s)` and `s` is a substring of
    /// [`AdapterInfo::name`], then this `FailureCase` applies. If
    /// this is `None`, the adapter name isn't considered.
    ///
    /// [`AdapterInfo::name`]: wgt::AdapterInfo::name
    pub adapter: Option<&'static str>,

    /// Name of driver expected to fail, or `None` for any driver name.
    ///
    /// If this is `Some(s)` and `s` is a substring of
    /// [`AdapterInfo::driver`], then this `FailureCase` applies. If
    /// this is `None`, the driver name isn't considered.
    ///
    /// [`AdapterInfo::driver`]: wgt::AdapterInfo::driver
    pub driver: Option<&'static str>,

    /// Reason why the test is expected to fail.
    ///
    /// If this does not match, the failure will not match this case.
    ///
    /// If no reasons are pushed, will match any failure.
    pub reasons: Vec<FailureReason>,

    /// Behavior after this case matches a failure.
    pub behavior: FailureBehavior,
}

impl FailureCase {
    /// Create a new failure case.
    pub fn new() -> Self {
        Self::default()
    }

    /// This case applies to all tests.
    pub fn always() -> Self {
        FailureCase::default()
    }

    /// This case applies to no tests.
    pub fn never() -> Self {
        FailureCase {
            backends: Some(wgpu::Backends::empty()),
            ..FailureCase::default()
        }
    }

    /// Tests running on any of the given backends.
    pub fn backend(backends: wgpu::Backends) -> Self {
        FailureCase {
            backends: Some(backends),
            ..FailureCase::default()
        }
    }

    /// Tests running on `adapter`.
    ///
    /// For this case to apply, the `adapter` string must appear as a substring
    /// of the adapter's [`AdapterInfo::name`]. The comparison is
    /// case-insensitive.
    ///
    /// [`AdapterInfo::name`]: wgt::AdapterInfo::name
    pub fn adapter(adapter: &'static str) -> Self {
        FailureCase {
            adapter: Some(adapter),
            ..FailureCase::default()
        }
    }

    /// Tests running on `backend` and `adapter`.
    ///
    /// For this case to apply, the test must be using an adapter for one of the
    /// given `backend` bits, and `adapter` string must appear as a substring of
    /// the adapter's [`AdapterInfo::name`]. The string comparison is
    /// case-insensitive.
    ///
    /// [`AdapterInfo::name`]: wgt::AdapterInfo::name
    pub fn backend_adapter(backends: wgpu::Backends, adapter: &'static str) -> Self {
        FailureCase {
            backends: Some(backends),
            adapter: Some(adapter),
            ..FailureCase::default()
        }
    }

    /// Tests running under WebGL.
    pub fn webgl2() -> Self {
        #[cfg(target_arch = "wasm32")]
        let case = FailureCase::backend(wgpu::Backends::GL);
        #[cfg(not(target_arch = "wasm32"))]
        let case = FailureCase::never();
        case
    }

    /// Tests running on the MoltenVK Vulkan driver on macOS.
    pub fn molten_vk() -> Self {
        FailureCase {
            backends: Some(wgpu::Backends::VULKAN),
            driver: Some("MoltenVK"),
            ..FailureCase::default()
        }
    }

    /// Return the reasons why this case should fail.
    pub fn reasons(&self) -> &[FailureReason] {
        if self.reasons.is_empty() {
            std::array::from_ref(&FailureReason::ANY)
        } else {
            &self.reasons
        }
    }

    /// Matches this failure case against the given validation error substring.
    ///
    /// Substrings are matched case-insensitively.
    ///
    /// If multiple reasons are pushed, will match any of them.
    pub fn validation_error(mut self, msg: &'static str) -> Self {
        self.reasons
            .push(FailureReason::validation_error().with_message(msg));
        self
    }

    /// Matches this failure case against the given panic substring.
    ///
    /// Substrings are matched case-insensitively.
    ///
    /// If multiple reasons are pushed, will match any of them.
    pub fn panic(mut self, msg: &'static str) -> Self {
        self.reasons.push(FailureReason::panic().with_message(msg));
        self
    }

    /// Test is flaky with the given configuration. Do not assert failure.
    ///
    /// Use this _very_ sparyingly, and match as tightly as you can, including giving a specific failure message.
    pub fn flaky(self) -> Self {
        FailureCase {
            behavior: FailureBehavior::Ignore,
            ..self
        }
    }

    /// Test whether `self` applies to `info`.
    ///
    /// If it does, return a `FailureReasons` whose set bits indicate
    /// why. If it doesn't, return `None`.
    ///
    /// The caller is responsible for converting the string-valued
    /// fields of `info` to lower case, to ensure case-insensitive
    /// matching.
    pub(crate) fn applies_to_adapter(
        &self,
        info: &wgt::AdapterInfo,
    ) -> Option<FailureApplicationReasons> {
        let mut reasons = FailureApplicationReasons::empty();

        if let Some(backends) = self.backends {
            if !backends.contains(wgpu::Backends::from(info.backend)) {
                return None;
            }
            reasons.set(FailureApplicationReasons::BACKEND, true);
        }
        if let Some(vendor) = self.vendor {
            if vendor != info.vendor {
                return None;
            }
            reasons.set(FailureApplicationReasons::VENDOR, true);
        }
        if let Some(adapter) = self.adapter {
            let adapter = adapter.to_lowercase();
            if !info.name.contains(&adapter) {
                return None;
            }
            reasons.set(FailureApplicationReasons::ADAPTER, true);
        }
        if let Some(driver) = self.driver {
            let driver = driver.to_lowercase();
            if !info.driver.contains(&driver) {
                return None;
            }
            reasons.set(FailureApplicationReasons::DRIVER, true);
        }

        // If we got this far but no specific reasons were triggered, then this
        // must be a wildcard.
        if reasons.is_empty() {
            Some(FailureApplicationReasons::ALWAYS)
        } else {
            Some(reasons)
        }
    }

    /// Returns true if the given failure "satisfies" this failure case.
    pub(crate) fn matches_failure(&self, failure: &FailureResult) -> bool {
        for reason in self.reasons() {
            let kind_matched = reason.kind.map_or(true, |kind| kind == failure.kind);

            let message_matched =
                reason
                    .message
                    .map_or(true, |message| matches!(&failure.message, Some(actual) if actual.to_lowercase().contains(&message.to_lowercase())));

            if kind_matched && message_matched {
                let message = failure.message.as_deref().unwrap_or("*no message*");
                log::error!("Matched {} {message}", failure.kind);
                return true;
            }
        }

        false
    }
}

bitflags::bitflags! {
    /// Reason why a test matches a given failure case.
    #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
    pub struct FailureApplicationReasons: u8 {
        const BACKEND = 1 << 0;
        const VENDOR = 1 << 1;
        const ADAPTER = 1 << 2;
        const DRIVER = 1 << 3;
        const ALWAYS = 1 << 4;
    }
}

/// Reason why a test is expected to fail.
///
/// If the test fails for a different reason, the given FailureCase will be ignored.
#[derive(Default, Debug, Clone, PartialEq)]
pub struct FailureReason {
    /// Match a particular kind of failure result.
    ///
    /// If `None`, match any result kind.
    kind: Option<FailureResultKind>,
    /// Match a particular message of a failure result.
    ///
    /// If `None`, matches any message. If `Some`, a case-insensitive sub-string
    /// test is performed. Allowing `"error occurred"` to match a message like
    /// `"An unexpected Error occurred!"`.
    message: Option<&'static str>,
}

impl FailureReason {
    /// Match any failure reason.
    const ANY: Self = Self {
        kind: None,
        message: None,
    };

    /// Match a validation error.
    #[allow(dead_code)] // Not constructed on wasm
    pub fn validation_error() -> Self {
        Self {
            kind: Some(FailureResultKind::ValidationError),
            message: None,
        }
    }

    /// Match a panic.
    pub fn panic() -> Self {
        Self {
            kind: Some(FailureResultKind::Panic),
            message: None,
        }
    }

    /// Match an error with a message.
    ///
    /// If specified, a case-insensitive sub-string test is performed. Allowing
    /// `"error occurred"` to match a message like `"An unexpected Error
    /// occurred!"`.
    pub fn with_message(self, message: &'static str) -> Self {
        Self {
            message: Some(message),
            ..self
        }
    }
}

#[derive(Default, Clone)]
pub enum FailureBehavior {
    /// Assert that the test fails for the given reason.
    ///
    /// If the test passes, the test harness will panic.
    #[default]
    AssertFailure,
    /// Ignore the matching failure.
    ///
    /// This is useful for tests that flake in a very specific way,
    /// but sometimes succeed, so we can't assert that they always fail.
    Ignore,
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) enum FailureResultKind {
    #[allow(dead_code)] // Not constructed on wasm
    ValidationError,
    Panic,
}

impl fmt::Display for FailureResultKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            FailureResultKind::ValidationError => write!(f, "Validation Error"),
            FailureResultKind::Panic => write!(f, "Panic"),
        }
    }
}

#[derive(Debug)]
pub(crate) struct FailureResult {
    kind: FailureResultKind,
    message: Option<String>,
}

impl FailureResult {
    /// Failure result is a panic.
    pub(super) fn panic() -> Self {
        Self {
            kind: FailureResultKind::Panic,
            message: None,
        }
    }

    /// Failure result is a validation error.
    #[allow(dead_code)] // Not constructed on wasm
    pub(super) fn validation_error() -> Self {
        Self {
            kind: FailureResultKind::ValidationError,
            message: None,
        }
    }

    /// Message associated with a failure result.
    pub(super) fn with_message(self, message: impl fmt::Display) -> Self {
        Self {
            kind: self.kind,
            message: Some(message.to_string()),
        }
    }
}

#[derive(PartialEq, Clone, Copy, Debug)]
pub(crate) enum ExpectationMatchResult {
    Panic,
    Complete,
}

/// Compares if the actual failures match the expected failures.
pub(crate) fn expectations_match_failures(
    expectations: &[FailureCase],
    mut actual: Vec<FailureResult>,
) -> ExpectationMatchResult {
    // Start with the assumption that we will pass.
    let mut result = ExpectationMatchResult::Complete;

    // Run through all expected failures.
    for expected_failure in expectations {
        // If any of the failures match.
        let mut matched = false;

        // Iterate through the failures.
        //
        // In reverse, to be able to use swap_remove.
        actual.retain(|failure| {
            // If the failure matches, remove it from the list of failures, as we expected it.
            let matches = expected_failure.matches_failure(failure);

            if matches {
                matched = true;
            }

            // Retain removes on false, so flip the bool so we remove on failure.
            !matches
        });

        // If we didn't match our expected failure against any of the actual failures,
        // and this failure is not flaky, then we need to panic, as we got an unexpected success.
        if !matched && matches!(expected_failure.behavior, FailureBehavior::AssertFailure) {
            result = ExpectationMatchResult::Panic;
            log::error!(
                "Expected to fail due to {:?}, but did not fail",
                expected_failure.reasons()
            );
        }
    }

    // If we have any failures left, then we got an unexpected failure
    // and we need to panic.
    if !actual.is_empty() {
        result = ExpectationMatchResult::Panic;
        for failure in actual {
            let message = failure.message.as_deref().unwrap_or("*no message*");
            log::error!("{}: {message}", failure.kind);
        }
    }

    result
}

#[cfg(test)]
mod test {
    use crate::{
        expectations::{ExpectationMatchResult, FailureResult},
        init::init_logger,
        FailureCase,
    };

    fn validation_err(msg: &'static str) -> FailureResult {
        FailureResult::validation_error().with_message(msg)
    }

    fn panic(msg: &'static str) -> FailureResult {
        FailureResult::panic().with_message(msg)
    }

    #[test]
    fn simple_match() {
        init_logger();

        // -- Unexpected failure --

        let expectation = vec![];
        let actual = vec![FailureResult::validation_error()];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Panic
        );

        // -- Missing expected failure --

        let expectation = vec![FailureCase::always()];
        let actual = vec![];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Panic
        );

        // -- Expected failure (validation) --

        let expectation = vec![FailureCase::always()];
        let actual = vec![FailureResult::validation_error()];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Complete
        );

        // -- Expected failure (panic) --

        let expectation = vec![FailureCase::always()];
        let actual = vec![FailureResult::panic()];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Complete
        );
    }

    #[test]
    fn substring_match() {
        init_logger();

        // -- Matching Substring --

        let expectation: Vec<FailureCase> =
            vec![FailureCase::always().validation_error("Some StrIng")];
        let actual = vec![FailureResult::validation_error().with_message(
            "a very long string that contains sOmE sTrInG of different capitalization",
        )];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Complete
        );

        // -- Non-Matching Substring --

        let expectation = vec![FailureCase::always().validation_error("Some String")];
        let actual = vec![validation_err("a very long string that doesn't contain it")];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Panic
        );
    }

    #[test]
    fn ignore_flaky() {
        init_logger();

        let expectation = vec![FailureCase::always().validation_error("blah").flaky()];
        let actual = vec![validation_err("some blah")];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Complete
        );

        let expectation = vec![FailureCase::always().validation_error("blah").flaky()];
        let actual = vec![];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Complete
        );
    }

    #[test]
    fn matches_multiple_errors() {
        init_logger();

        // -- matches all matching errors --

        let expectation = vec![FailureCase::always().validation_error("blah")];
        let actual = vec![
            validation_err("some blah"),
            validation_err("some other blah"),
        ];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Complete
        );

        // -- but not all errors --

        let expectation = vec![FailureCase::always().validation_error("blah")];
        let actual = vec![
            validation_err("some blah"),
            validation_err("some other blah"),
            validation_err("something else"),
        ];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Panic
        );
    }

    #[test]
    fn multi_reason_error() {
        init_logger();

        let expectation = vec![FailureCase::default()
            .validation_error("blah")
            .panic("panik")];
        let actual = vec![
            validation_err("my blah blah validation error"),
            panic("my panik"),
        ];

        assert_eq!(
            super::expectations_match_failures(&expectation, actual),
            ExpectationMatchResult::Complete
        );
    }
}