virtio_accel_mock/
fault.rs

1//! Deterministic fault injection and ownership auditing for accelerator backends.
2//!
3//! [`FaultAccelerator`] wraps any [`Accelerator`] with an explicit script keyed by method and
4//! one-based call occurrence. The wrapper records calls, release attempts, and provider ownership
5//! without requiring host synchronization. It is test infrastructure, not part of the normative
6//! accelerator ABI.
7
8use std::cell::RefCell;
9use std::rc::Rc;
10use std::vec::Vec;
11use virtio_accel_core::{
12    Accelerator, AllocatedBuffer, ArtifactRef, BackendError, BindingRef, BufferDesc, ByteSink,
13    ByteSource, ContextDesc, DeviceInfo, EventState, QueueDesc, ReleaseFailure, SubmitFailure,
14    Timeout,
15};
16
17const FAULT_POINT_COUNT: usize = 15;
18
19/// One fallible method in the [`Accelerator`] contract.
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub enum FaultPoint {
22    DeviceInfo,
23    CreateContext,
24    DestroyContext,
25    AllocateBuffer,
26    WriteBuffer,
27    ReadBuffer,
28    FreeBuffer,
29    LoadProgram,
30    UnloadProgram,
31    CreateQueue,
32    DestroyQueue,
33    Submit,
34    PollEvent,
35    CancelEvent,
36    DestroyEvent,
37}
38
39impl FaultPoint {
40    const fn index(self) -> usize {
41        match self {
42            Self::DeviceInfo => 0,
43            Self::CreateContext => 1,
44            Self::DestroyContext => 2,
45            Self::AllocateBuffer => 3,
46            Self::WriteBuffer => 4,
47            Self::ReadBuffer => 5,
48            Self::FreeBuffer => 6,
49            Self::LoadProgram => 7,
50            Self::UnloadProgram => 8,
51            Self::CreateQueue => 9,
52            Self::DestroyQueue => 10,
53            Self::Submit => 11,
54            Self::PollEvent => 12,
55            Self::CancelEvent => 13,
56            Self::DestroyEvent => 14,
57        }
58    }
59
60    const fn allows(self, action: FaultAction) -> bool {
61        match self {
62            Self::DestroyContext
63            | Self::FreeBuffer
64            | Self::UnloadProgram
65            | Self::DestroyQueue
66            | Self::DestroyEvent
67            | Self::Submit => matches!(
68                action,
69                FaultAction::Rejected(_) | FaultAction::Indeterminate(_)
70            ),
71            Self::PollEvent => match action {
72                FaultAction::ErrorBefore(_) | FaultAction::ErrorAfter(_) => true,
73                FaultAction::Completion(state) => !matches!(state, EventState::Pending),
74                FaultAction::Rejected(_) | FaultAction::Indeterminate(_) => false,
75            },
76            _ => matches!(
77                action,
78                FaultAction::ErrorBefore(_) | FaultAction::ErrorAfter(_)
79            ),
80        }
81    }
82}
83
84/// Fault behavior applied to one scripted method occurrence.
85#[derive(Clone, Copy, Debug, PartialEq, Eq)]
86pub enum FaultAction {
87    /// Return an error without invoking the wrapped method.
88    ErrorBefore(BackendError),
89    /// Invoke the wrapped method, then replace a successful result with an error.
90    ///
91    /// Resource creation methods release the newly created resource before returning this error.
92    ErrorAfter(BackendError),
93    /// Reject a release or submission before ownership changes.
94    Rejected(BackendError),
95    /// Make release or submission ownership indeterminate after the acceptance boundary.
96    Indeterminate(BackendError),
97    /// Publish and retain one synthetic terminal completion state.
98    Completion(EventState),
99}
100
101/// One fault keyed by method and one-based call occurrence.
102#[derive(Clone, Copy, Debug, PartialEq, Eq)]
103pub struct FaultStep {
104    pub point: FaultPoint,
105    pub occurrence: u32,
106    pub action: FaultAction,
107}
108
109impl FaultStep {
110    pub const fn new(point: FaultPoint, occurrence: u32, action: FaultAction) -> Self {
111        Self {
112            point,
113            occurrence,
114            action,
115        }
116    }
117}
118
119/// Invalid deterministic script.
120#[derive(Clone, Copy, Debug, PartialEq, Eq)]
121pub enum FaultScriptError {
122    ZeroOccurrence {
123        point: FaultPoint,
124    },
125    DuplicateStep {
126        point: FaultPoint,
127        occurrence: u32,
128    },
129    IncompatibleAction {
130        point: FaultPoint,
131        action: FaultAction,
132    },
133}
134
135/// Validated explicit fault schedule.
136#[derive(Clone, Debug, Default, PartialEq, Eq)]
137pub struct FaultScript {
138    steps: Vec<FaultStep>,
139}
140
141impl FaultScript {
142    pub fn new(steps: impl IntoIterator<Item = FaultStep>) -> Result<Self, FaultScriptError> {
143        let mut validated = Vec::new();
144        for step in steps {
145            if step.occurrence == 0 {
146                return Err(FaultScriptError::ZeroOccurrence { point: step.point });
147            }
148            if !step.point.allows(step.action) {
149                return Err(FaultScriptError::IncompatibleAction {
150                    point: step.point,
151                    action: step.action,
152                });
153            }
154            if validated.iter().any(|prior: &FaultStep| {
155                prior.point == step.point && prior.occurrence == step.occurrence
156            }) {
157                return Err(FaultScriptError::DuplicateStep {
158                    point: step.point,
159                    occurrence: step.occurrence,
160                });
161            }
162            validated.push(step);
163        }
164        Ok(Self { steps: validated })
165    }
166}
167
168/// Stable harness-owned identity for one provider resource.
169#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
170pub struct ResourceId(u64);
171
172impl ResourceId {
173    pub const fn get(self) -> u64 {
174        self.0
175    }
176}
177
178/// Provider resource category tracked by the harness.
179#[derive(Clone, Copy, Debug, PartialEq, Eq)]
180pub enum ResourceKind {
181    Context,
182    Buffer,
183    Program,
184    Queue,
185    Event,
186}
187
188/// Last known provider ownership state.
189#[derive(Clone, Copy, Debug, PartialEq, Eq)]
190pub enum ResourceState {
191    Live,
192    Released,
193    Indeterminate,
194    Discarded,
195}
196
197/// One recorded resource and its last known state.
198#[derive(Clone, Copy, Debug, PartialEq, Eq)]
199pub struct ResourceRecord {
200    pub id: ResourceId,
201    pub kind: ResourceKind,
202    pub state: ResourceState,
203}
204
205/// One backend method call.
206#[derive(Clone, Copy, Debug, PartialEq, Eq)]
207pub struct CallRecord {
208    pub sequence: u64,
209    pub point: FaultPoint,
210    pub occurrence: u32,
211    pub injected: Option<FaultAction>,
212}
213
214/// Result of one provider release attempt.
215#[derive(Clone, Copy, Debug, PartialEq, Eq)]
216pub enum ReleaseOutcome {
217    Success,
218    Rejected(BackendError),
219    Indeterminate(BackendError),
220}
221
222/// One release attempt, including internal rollback after an injected post-create fault.
223#[derive(Clone, Copy, Debug, PartialEq, Eq)]
224pub struct ReleaseRecord {
225    pub resource_id: ResourceId,
226    pub kind: ResourceKind,
227    pub attempt: u32,
228    pub outcome: ReleaseOutcome,
229    pub rollback: bool,
230}
231
232/// Ownership-contract violation detected by the harness.
233#[derive(Clone, Copy, Debug, PartialEq, Eq)]
234pub enum OwnershipViolation {
235    UnknownResource {
236        id: ResourceId,
237        expected: ResourceKind,
238    },
239    KindMismatch {
240        id: ResourceId,
241        expected: ResourceKind,
242        actual: ResourceKind,
243    },
244    ResourceNotLive {
245        id: ResourceId,
246        kind: ResourceKind,
247        state: ResourceState,
248    },
249    PrematureRelease {
250        id: ResourceId,
251        kind: ResourceKind,
252        retained_by: ResourceId,
253    },
254    RollbackFailed {
255        id: ResourceId,
256        kind: ResourceKind,
257        error: BackendError,
258    },
259}
260
261/// Resource counts for one ownership state.
262#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
263pub struct ResourceTally {
264    pub contexts: u32,
265    pub buffers: u32,
266    pub programs: u32,
267    pub queues: u32,
268    pub events: u32,
269}
270
271impl ResourceTally {
272    fn increment(&mut self, kind: ResourceKind) {
273        let count = match kind {
274            ResourceKind::Context => &mut self.contexts,
275            ResourceKind::Buffer => &mut self.buffers,
276            ResourceKind::Program => &mut self.programs,
277            ResourceKind::Queue => &mut self.queues,
278            ResourceKind::Event => &mut self.events,
279        };
280        *count = count.saturating_add(1);
281    }
282}
283
284/// Point-in-time copy of fault and ownership state.
285#[derive(Clone, Debug, PartialEq, Eq)]
286pub struct FaultSnapshot {
287    pub calls: Vec<CallRecord>,
288    pub releases: Vec<ReleaseRecord>,
289    pub resources: Vec<ResourceRecord>,
290    pub pending_faults: Vec<FaultStep>,
291    pub violations: Vec<OwnershipViolation>,
292}
293
294impl FaultSnapshot {
295    pub fn resources_in(&self, state: ResourceState) -> ResourceTally {
296        let mut tally = ResourceTally::default();
297        for resource in self.resources.iter().filter(|item| item.state == state) {
298            tally.increment(resource.kind);
299        }
300        tally
301    }
302
303    pub fn is_clean(&self) -> bool {
304        self.pending_faults.is_empty()
305            && self.violations.is_empty()
306            && self.resources_in(ResourceState::Live) == ResourceTally::default()
307            && self.resources_in(ResourceState::Indeterminate) == ResourceTally::default()
308    }
309}
310
311#[derive(Debug)]
312struct FaultState {
313    pending_faults: Vec<FaultStep>,
314    occurrences: [u32; FAULT_POINT_COUNT],
315    next_sequence: u64,
316    next_resource_id: u64,
317    calls: Vec<CallRecord>,
318    releases: Vec<ReleaseRecord>,
319    resources: Vec<ResourceRecord>,
320    parents: Vec<(ResourceId, ResourceId)>,
321    event_dependencies: Vec<(ResourceId, Vec<ResourceId>)>,
322    event_completions: Vec<(ResourceId, EventState)>,
323    violations: Vec<OwnershipViolation>,
324}
325
326impl FaultState {
327    fn new(script: FaultScript) -> Self {
328        Self {
329            pending_faults: script.steps,
330            occurrences: [0; FAULT_POINT_COUNT],
331            next_sequence: 1,
332            next_resource_id: 1,
333            calls: Vec::new(),
334            releases: Vec::new(),
335            resources: Vec::new(),
336            parents: Vec::new(),
337            event_dependencies: Vec::new(),
338            event_completions: Vec::new(),
339            violations: Vec::new(),
340        }
341    }
342}
343
344/// Cloneable control and audit handle for a [`FaultAccelerator`].
345#[derive(Clone, Debug)]
346pub struct FaultControl {
347    state: Rc<RefCell<FaultState>>,
348}
349
350#[derive(Clone, Copy, Debug, PartialEq, Eq)]
351enum ReleaseValidation {
352    Ready,
353    Invalid,
354    Premature,
355}
356
357impl FaultControl {
358    fn new(script: FaultScript) -> Self {
359        Self {
360            state: Rc::new(RefCell::new(FaultState::new(script))),
361        }
362    }
363
364    fn begin_call(&self, point: FaultPoint) -> Option<FaultAction> {
365        let mut state = self.state.borrow_mut();
366        let occurrence = state.occurrences[point.index()]
367            .checked_add(1)
368            .expect("fault call occurrence overflow");
369        state.occurrences[point.index()] = occurrence;
370        let injected = state
371            .pending_faults
372            .iter()
373            .position(|step| step.point == point && step.occurrence == occurrence)
374            .map(|index| state.pending_faults.remove(index).action);
375        let sequence = state.next_sequence;
376        state.next_sequence = state
377            .next_sequence
378            .checked_add(1)
379            .expect("fault call sequence overflow");
380        state.calls.push(CallRecord {
381            sequence,
382            point,
383            occurrence,
384            injected,
385        });
386        injected
387    }
388
389    fn register(&self, kind: ResourceKind) -> ResourceId {
390        let mut state = self.state.borrow_mut();
391        let id = ResourceId(state.next_resource_id);
392        state.next_resource_id = state
393            .next_resource_id
394            .checked_add(1)
395            .expect("fault resource identity overflow");
396        state.resources.push(ResourceRecord {
397            id,
398            kind,
399            state: ResourceState::Live,
400        });
401        id
402    }
403
404    fn register_child(&self, kind: ResourceKind, parent: ResourceId) -> ResourceId {
405        let id = self.register(kind);
406        self.state.borrow_mut().parents.push((id, parent));
407        id
408    }
409
410    fn register_event(&self, dependencies: Vec<ResourceId>) -> ResourceId {
411        let id = self.register(ResourceKind::Event);
412        self.state
413            .borrow_mut()
414            .event_dependencies
415            .push((id, dependencies));
416        id
417    }
418
419    fn set_completion(&self, id: ResourceId, completion: EventState) {
420        let mut state = self.state.borrow_mut();
421        if let Some((_, prior)) = state
422            .event_completions
423            .iter_mut()
424            .find(|(event, _)| *event == id)
425        {
426            *prior = completion;
427        } else {
428            state.event_completions.push((id, completion));
429        }
430    }
431
432    fn completion(&self, id: ResourceId) -> Option<EventState> {
433        self.state
434            .borrow()
435            .event_completions
436            .iter()
437            .find_map(|(event, state)| (*event == id).then_some(*state))
438    }
439
440    fn validate(&self, id: ResourceId, expected: ResourceKind) -> bool {
441        let mut state = self.state.borrow_mut();
442        let Some(index) = state.resources.iter().position(|item| item.id == id) else {
443            state
444                .violations
445                .push(OwnershipViolation::UnknownResource { id, expected });
446            return false;
447        };
448        let resource = state.resources[index];
449        if resource.kind != expected {
450            state.violations.push(OwnershipViolation::KindMismatch {
451                id,
452                expected,
453                actual: resource.kind,
454            });
455            return false;
456        }
457        if resource.state != ResourceState::Live {
458            state.violations.push(OwnershipViolation::ResourceNotLive {
459                id,
460                kind: expected,
461                state: resource.state,
462            });
463            return false;
464        }
465        true
466    }
467
468    fn validate_release(&self, id: ResourceId, expected: ResourceKind) -> ReleaseValidation {
469        if !self.validate(id, expected) {
470            return ReleaseValidation::Invalid;
471        }
472
473        let mut state = self.state.borrow_mut();
474        let retained_by = match expected {
475            ResourceKind::Context => state.parents.iter().find_map(|(child, parent)| {
476                (*parent == id
477                    && state.resources.iter().any(|resource| {
478                        resource.id == *child
479                            && matches!(
480                                resource.state,
481                                ResourceState::Live | ResourceState::Indeterminate
482                            )
483                    }))
484                .then_some(*child)
485            }),
486            ResourceKind::Buffer | ResourceKind::Program | ResourceKind::Queue => state
487                .event_dependencies
488                .iter()
489                .find_map(|(event, dependencies)| {
490                    (dependencies.contains(&id)
491                        && state.resources.iter().any(|resource| {
492                            resource.id == *event
493                                && matches!(
494                                    resource.state,
495                                    ResourceState::Live | ResourceState::Indeterminate
496                                )
497                        }))
498                    .then_some(*event)
499                }),
500            ResourceKind::Event => None,
501        };
502        match retained_by {
503            Some(retained_by) => {
504                state.violations.push(OwnershipViolation::PrematureRelease {
505                    id,
506                    kind: expected,
507                    retained_by,
508                });
509                ReleaseValidation::Premature
510            }
511            None => ReleaseValidation::Ready,
512        }
513    }
514
515    fn finish_release(
516        &self,
517        id: ResourceId,
518        kind: ResourceKind,
519        outcome: ReleaseOutcome,
520        rollback: bool,
521    ) {
522        let mut state = self.state.borrow_mut();
523        let prior_attempts = state
524            .releases
525            .iter()
526            .filter(|record| record.resource_id == id)
527            .count();
528        let attempt = u32::try_from(prior_attempts)
529            .expect("fault release attempt overflow")
530            .checked_add(1)
531            .expect("fault release attempt overflow");
532        state.releases.push(ReleaseRecord {
533            resource_id: id,
534            kind,
535            attempt,
536            outcome,
537            rollback,
538        });
539        if let Some(resource) = state.resources.iter_mut().find(|item| item.id == id) {
540            resource.state = match outcome {
541                ReleaseOutcome::Success => ResourceState::Released,
542                ReleaseOutcome::Rejected(_) => ResourceState::Live,
543                ReleaseOutcome::Indeterminate(_) => ResourceState::Indeterminate,
544            };
545        }
546    }
547
548    fn finish_rollback(&self, id: ResourceId, kind: ResourceKind, outcome: ReleaseOutcome) {
549        self.finish_release(id, kind, outcome, true);
550        if let ReleaseOutcome::Rejected(error) | ReleaseOutcome::Indeterminate(error) = outcome {
551            let mut state = self.state.borrow_mut();
552            if let Some(resource) = state.resources.iter_mut().find(|item| item.id == id) {
553                resource.state = ResourceState::Indeterminate;
554            }
555            state
556                .violations
557                .push(OwnershipViolation::RollbackFailed { id, kind, error });
558        }
559    }
560
561    /// Mark all provider state discarded after the caller discards or replaces the backend instance.
562    pub fn discard_all(&self) {
563        for resource in &mut self.state.borrow_mut().resources {
564            if matches!(
565                resource.state,
566                ResourceState::Live | ResourceState::Indeterminate
567            ) {
568                resource.state = ResourceState::Discarded;
569            }
570        }
571    }
572
573    pub fn snapshot(&self) -> FaultSnapshot {
574        let state = self.state.borrow();
575        FaultSnapshot {
576            calls: state.calls.clone(),
577            releases: state.releases.clone(),
578            resources: state.resources.clone(),
579            pending_faults: state.pending_faults.clone(),
580            violations: state.violations.clone(),
581        }
582    }
583}
584
585#[derive(Clone, Debug)]
586struct Tracked<T> {
587    id: ResourceId,
588    inner: T,
589}
590
591macro_rules! tracked_handle {
592    ($name:ident) => {
593        #[derive(Clone, Debug)]
594        pub struct $name<T>(Tracked<T>);
595
596        impl<T> $name<T> {
597            pub const fn id(&self) -> ResourceId {
598                self.0.id
599            }
600
601            pub const fn inner(&self) -> &T {
602                &self.0.inner
603            }
604
605            pub fn inner_mut(&mut self) -> &mut T {
606                &mut self.0.inner
607            }
608        }
609    };
610}
611
612tracked_handle!(FaultContext);
613tracked_handle!(FaultBuffer);
614tracked_handle!(FaultProgram);
615tracked_handle!(FaultQueue);
616tracked_handle!(FaultEvent);
617
618/// Accelerator wrapper driven by a validated deterministic fault script.
619pub struct FaultAccelerator<A> {
620    inner: A,
621    control: FaultControl,
622}
623
624impl<A> FaultAccelerator<A> {
625    pub fn new(inner: A, script: FaultScript) -> Self {
626        Self {
627            inner,
628            control: FaultControl::new(script),
629        }
630    }
631
632    pub const fn inner(&self) -> &A {
633        &self.inner
634    }
635
636    pub fn control(&self) -> FaultControl {
637        self.control.clone()
638    }
639}
640
641const fn before_error(action: Option<FaultAction>) -> Option<BackendError> {
642    match action {
643        Some(FaultAction::ErrorBefore(error)) => Some(error),
644        _ => None,
645    }
646}
647
648const fn after_error(action: Option<FaultAction>) -> Option<BackendError> {
649    match action {
650        Some(FaultAction::ErrorAfter(error)) => Some(error),
651        _ => None,
652    }
653}
654
655impl<A: Accelerator> FaultAccelerator<A> {
656    fn rollback_resource<R>(
657        &self,
658        id: ResourceId,
659        kind: ResourceKind,
660        resource: R,
661        release: impl FnOnce(&A, R) -> Result<(), ReleaseFailure<R>>,
662    ) {
663        let outcome = match release(&self.inner, resource) {
664            Ok(()) => ReleaseOutcome::Success,
665            Err(ReleaseFailure::Rejected { error, .. }) => ReleaseOutcome::Rejected(error),
666            Err(ReleaseFailure::Indeterminate { error }) => ReleaseOutcome::Indeterminate(error),
667        };
668        self.control.finish_rollback(id, kind, outcome);
669    }
670
671    fn release_resource<R, W>(
672        &self,
673        point: FaultPoint,
674        kind: ResourceKind,
675        tracked: Tracked<R>,
676        wrap: fn(Tracked<R>) -> W,
677        release: impl FnOnce(&A, R) -> Result<(), ReleaseFailure<R>>,
678    ) -> Result<(), ReleaseFailure<W>> {
679        let action = self.control.begin_call(point);
680        let id = tracked.id;
681        match self.control.validate_release(id, kind) {
682            ReleaseValidation::Ready => {}
683            ReleaseValidation::Invalid => {
684                return Err(ReleaseFailure::Indeterminate {
685                    error: BackendError::DeviceLost,
686                });
687            }
688            ReleaseValidation::Premature => {
689                self.control.finish_release(
690                    id,
691                    kind,
692                    ReleaseOutcome::Indeterminate(BackendError::DeviceLost),
693                    false,
694                );
695                return Err(ReleaseFailure::Indeterminate {
696                    error: BackendError::DeviceLost,
697                });
698            }
699        }
700        match action {
701            Some(FaultAction::Rejected(error)) => {
702                self.control
703                    .finish_release(id, kind, ReleaseOutcome::Rejected(error), false);
704                Err(ReleaseFailure::Rejected {
705                    error,
706                    resource: wrap(tracked),
707                })
708            }
709            Some(FaultAction::Indeterminate(error)) => {
710                self.control
711                    .finish_release(id, kind, ReleaseOutcome::Indeterminate(error), false);
712                Err(ReleaseFailure::Indeterminate { error })
713            }
714            None => {
715                let Tracked { id, inner } = tracked;
716                match release(&self.inner, inner) {
717                    Ok(()) => {
718                        self.control
719                            .finish_release(id, kind, ReleaseOutcome::Success, false);
720                        Ok(())
721                    }
722                    Err(ReleaseFailure::Rejected { error, resource }) => {
723                        self.control.finish_release(
724                            id,
725                            kind,
726                            ReleaseOutcome::Rejected(error),
727                            false,
728                        );
729                        Err(ReleaseFailure::Rejected {
730                            error,
731                            resource: wrap(Tracked {
732                                id,
733                                inner: resource,
734                            }),
735                        })
736                    }
737                    Err(ReleaseFailure::Indeterminate { error }) => {
738                        self.control.finish_release(
739                            id,
740                            kind,
741                            ReleaseOutcome::Indeterminate(error),
742                            false,
743                        );
744                        Err(ReleaseFailure::Indeterminate { error })
745                    }
746                }
747            }
748            Some(
749                FaultAction::ErrorBefore(_)
750                | FaultAction::ErrorAfter(_)
751                | FaultAction::Completion(_),
752            ) => {
753                unreachable!("validated release scripts contain only ownership outcomes")
754            }
755        }
756    }
757
758    fn validate_resource<T>(
759        &self,
760        resource: &Tracked<T>,
761        kind: ResourceKind,
762    ) -> Result<(), BackendError> {
763        if self.control.validate(resource.id, kind) {
764            Ok(())
765        } else {
766            Err(BackendError::DeviceLost)
767        }
768    }
769}
770
771impl<A: Accelerator> Accelerator for FaultAccelerator<A> {
772    type Context = FaultContext<A::Context>;
773    type Buffer = FaultBuffer<A::Buffer>;
774    type Program = FaultProgram<A::Program>;
775    type Queue = FaultQueue<A::Queue>;
776    type Event = FaultEvent<A::Event>;
777
778    fn device_info(&self) -> Result<DeviceInfo, BackendError> {
779        let action = self.control.begin_call(FaultPoint::DeviceInfo);
780        if let Some(error) = before_error(action) {
781            return Err(error);
782        }
783        let result = self.inner.device_info();
784        match (result, after_error(action)) {
785            (Ok(_), Some(error)) => Err(error),
786            (result, _) => result,
787        }
788    }
789
790    fn create_context(&self, desc: ContextDesc) -> Result<Self::Context, BackendError> {
791        let action = self.control.begin_call(FaultPoint::CreateContext);
792        if let Some(error) = before_error(action) {
793            return Err(error);
794        }
795        let context = self.inner.create_context(desc)?;
796        let id = self.control.register(ResourceKind::Context);
797        if let Some(error) = after_error(action) {
798            self.rollback_resource(id, ResourceKind::Context, context, A::destroy_context);
799            return Err(error);
800        }
801        Ok(FaultContext(Tracked { id, inner: context }))
802    }
803
804    fn destroy_context(&self, context: Self::Context) -> Result<(), ReleaseFailure<Self::Context>> {
805        self.release_resource(
806            FaultPoint::DestroyContext,
807            ResourceKind::Context,
808            context.0,
809            FaultContext,
810            A::destroy_context,
811        )
812    }
813
814    fn allocate_buffer(
815        &self,
816        context: &Self::Context,
817        desc: BufferDesc,
818    ) -> Result<AllocatedBuffer<Self::Buffer>, BackendError> {
819        let action = self.control.begin_call(FaultPoint::AllocateBuffer);
820        self.validate_resource(&context.0, ResourceKind::Context)?;
821        if let Some(error) = before_error(action) {
822            return Err(error);
823        }
824        let allocation = self.inner.allocate_buffer(context.inner(), desc)?;
825        let (buffer, info) = allocation.into_parts();
826        let id = self
827            .control
828            .register_child(ResourceKind::Buffer, context.id());
829        if let Some(error) = after_error(action) {
830            self.rollback_resource(id, ResourceKind::Buffer, buffer, A::free_buffer);
831            return Err(error);
832        }
833        Ok(AllocatedBuffer::new(
834            FaultBuffer(Tracked { id, inner: buffer }),
835            info,
836        ))
837    }
838
839    fn write_buffer(
840        &self,
841        buffer: &mut Self::Buffer,
842        offset: u64,
843        data: &dyn ByteSource,
844    ) -> Result<(), BackendError> {
845        let action = self.control.begin_call(FaultPoint::WriteBuffer);
846        self.validate_resource(&buffer.0, ResourceKind::Buffer)?;
847        if let Some(error) = before_error(action) {
848            return Err(error);
849        }
850        let result = self.inner.write_buffer(buffer.inner_mut(), offset, data);
851        match (result, after_error(action)) {
852            (Ok(()), Some(error)) => Err(error),
853            (result, _) => result,
854        }
855    }
856
857    fn read_buffer(
858        &self,
859        buffer: &Self::Buffer,
860        offset: u64,
861        data: &mut dyn ByteSink,
862    ) -> Result<(), BackendError> {
863        let action = self.control.begin_call(FaultPoint::ReadBuffer);
864        self.validate_resource(&buffer.0, ResourceKind::Buffer)?;
865        if let Some(error) = before_error(action) {
866            return Err(error);
867        }
868        let result = self.inner.read_buffer(buffer.inner(), offset, data);
869        match (result, after_error(action)) {
870            (Ok(()), Some(error)) => Err(error),
871            (result, _) => result,
872        }
873    }
874
875    fn free_buffer(&self, buffer: Self::Buffer) -> Result<(), ReleaseFailure<Self::Buffer>> {
876        self.release_resource(
877            FaultPoint::FreeBuffer,
878            ResourceKind::Buffer,
879            buffer.0,
880            FaultBuffer,
881            A::free_buffer,
882        )
883    }
884
885    fn load_program(
886        &self,
887        context: &Self::Context,
888        artifact: ArtifactRef<'_>,
889    ) -> Result<Self::Program, BackendError> {
890        let action = self.control.begin_call(FaultPoint::LoadProgram);
891        self.validate_resource(&context.0, ResourceKind::Context)?;
892        if let Some(error) = before_error(action) {
893            return Err(error);
894        }
895        let program = self.inner.load_program(context.inner(), artifact)?;
896        let id = self
897            .control
898            .register_child(ResourceKind::Program, context.id());
899        if let Some(error) = after_error(action) {
900            self.rollback_resource(id, ResourceKind::Program, program, A::unload_program);
901            return Err(error);
902        }
903        Ok(FaultProgram(Tracked { id, inner: program }))
904    }
905
906    fn unload_program(&self, program: Self::Program) -> Result<(), ReleaseFailure<Self::Program>> {
907        self.release_resource(
908            FaultPoint::UnloadProgram,
909            ResourceKind::Program,
910            program.0,
911            FaultProgram,
912            A::unload_program,
913        )
914    }
915
916    fn create_queue(
917        &self,
918        context: &Self::Context,
919        desc: QueueDesc,
920    ) -> Result<Self::Queue, BackendError> {
921        let action = self.control.begin_call(FaultPoint::CreateQueue);
922        self.validate_resource(&context.0, ResourceKind::Context)?;
923        if let Some(error) = before_error(action) {
924            return Err(error);
925        }
926        let queue = self.inner.create_queue(context.inner(), desc)?;
927        let id = self
928            .control
929            .register_child(ResourceKind::Queue, context.id());
930        if let Some(error) = after_error(action) {
931            self.rollback_resource(id, ResourceKind::Queue, queue, A::destroy_queue);
932            return Err(error);
933        }
934        Ok(FaultQueue(Tracked { id, inner: queue }))
935    }
936
937    fn destroy_queue(&self, queue: Self::Queue) -> Result<(), ReleaseFailure<Self::Queue>> {
938        self.release_resource(
939            FaultPoint::DestroyQueue,
940            ResourceKind::Queue,
941            queue.0,
942            FaultQueue,
943            A::destroy_queue,
944        )
945    }
946
947    fn submit(
948        &self,
949        queue: &Self::Queue,
950        program: &Self::Program,
951        bindings: &[BindingRef<'_, Self::Buffer>],
952        timeout: Timeout,
953    ) -> Result<Self::Event, SubmitFailure<Self::Event>> {
954        let action = self.control.begin_call(FaultPoint::Submit);
955        self.validate_resource(&queue.0, ResourceKind::Queue)
956            .map_err(SubmitFailure::Rejected)?;
957        self.validate_resource(&program.0, ResourceKind::Program)
958            .map_err(SubmitFailure::Rejected)?;
959
960        let mut inner_bindings = Vec::with_capacity(bindings.len());
961        for binding in bindings {
962            self.validate_resource(&binding.buffer.0, ResourceKind::Buffer)
963                .map_err(SubmitFailure::Rejected)?;
964            inner_bindings.push(BindingRef {
965                slot: binding.slot,
966                buffer: binding.buffer.inner(),
967                range: binding.range,
968                access: binding.access,
969            });
970        }
971
972        if let Some(FaultAction::Rejected(error)) = action {
973            return Err(SubmitFailure::Rejected(error));
974        }
975        let scripted_indeterminate = match action {
976            Some(FaultAction::Indeterminate(error)) => Some(error),
977            _ => None,
978        };
979        match self
980            .inner
981            .submit(queue.inner(), program.inner(), &inner_bindings, timeout)
982        {
983            Ok(event) => {
984                let mut dependencies = Vec::with_capacity(bindings.len().saturating_add(2));
985                dependencies.push(queue.id());
986                dependencies.push(program.id());
987                dependencies.extend(bindings.iter().map(|binding| binding.buffer.id()));
988                let id = self.control.register_event(dependencies);
989                let event = FaultEvent(Tracked { id, inner: event });
990                match scripted_indeterminate {
991                    Some(error) => Err(SubmitFailure::Indeterminate { error, event }),
992                    None => Ok(event),
993                }
994            }
995            Err(SubmitFailure::Rejected(error)) => Err(SubmitFailure::Rejected(error)),
996            Err(SubmitFailure::Indeterminate { error, event }) => {
997                let mut dependencies = Vec::with_capacity(bindings.len().saturating_add(2));
998                dependencies.push(queue.id());
999                dependencies.push(program.id());
1000                dependencies.extend(bindings.iter().map(|binding| binding.buffer.id()));
1001                let id = self.control.register_event(dependencies);
1002                Err(SubmitFailure::Indeterminate {
1003                    error: scripted_indeterminate.unwrap_or(error),
1004                    event: FaultEvent(Tracked { id, inner: event }),
1005                })
1006            }
1007        }
1008    }
1009
1010    fn poll_event(&self, event: &Self::Event) -> Result<EventState, BackendError> {
1011        let action = self.control.begin_call(FaultPoint::PollEvent);
1012        self.validate_resource(&event.0, ResourceKind::Event)?;
1013        if let Some(error) = before_error(action) {
1014            return Err(error);
1015        }
1016        if let Some(FaultAction::Completion(completion)) = action {
1017            self.control.set_completion(event.id(), completion);
1018        }
1019        let result = match self.control.completion(event.id()) {
1020            Some(completion) => Ok(completion),
1021            None => self.inner.poll_event(event.inner()),
1022        };
1023        match (result, after_error(action)) {
1024            (Ok(_), Some(error)) => Err(error),
1025            (result, _) => result,
1026        }
1027    }
1028
1029    fn cancel_event(&self, event: &Self::Event) -> Result<(), BackendError> {
1030        let action = self.control.begin_call(FaultPoint::CancelEvent);
1031        self.validate_resource(&event.0, ResourceKind::Event)?;
1032        if let Some(error) = before_error(action) {
1033            return Err(error);
1034        }
1035        if self.control.completion(event.id()).is_some() {
1036            return Err(BackendError::Busy);
1037        }
1038        let result = self.inner.cancel_event(event.inner());
1039        match (result, after_error(action)) {
1040            (Ok(()), Some(error)) => Err(error),
1041            (result, _) => result,
1042        }
1043    }
1044
1045    fn destroy_event(&self, event: Self::Event) -> Result<(), ReleaseFailure<Self::Event>> {
1046        let synthetic_completion = self.control.completion(event.id()).is_some();
1047        self.release_resource(
1048            FaultPoint::DestroyEvent,
1049            ResourceKind::Event,
1050            event.0,
1051            FaultEvent,
1052            move |inner, resource| {
1053                if synthetic_completion {
1054                    if let Err(error) = inner.cancel_event(&resource) {
1055                        if error != BackendError::Busy {
1056                            return Err(ReleaseFailure::Rejected { error, resource });
1057                        }
1058                    }
1059                }
1060                inner.destroy_event(resource)
1061            },
1062        )
1063    }
1064}
1065
1066#[cfg(test)]
1067mod tests {
1068    use super::*;
1069    use crate::{MockAccelerator, reference};
1070    use virtio_accel_core::{
1071        AccessMode, ArtifactRef, BindingRef, BufferRange, BufferUsage, ContextDesc, MemoryDomain,
1072    };
1073
1074    fn buffer_desc() -> BufferDesc {
1075        BufferDesc::new(
1076            8,
1077            1,
1078            MemoryDomain::Shared,
1079            BufferUsage::TRANSFER_SOURCE
1080                | BufferUsage::TRANSFER_DESTINATION
1081                | BufferUsage::MUTABLE_STATE,
1082        )
1083        .unwrap()
1084    }
1085
1086    fn load_barrier(
1087        backend: &FaultAccelerator<MockAccelerator>,
1088        context: &FaultContext<crate::MockContext>,
1089    ) -> FaultProgram<crate::MockProgram> {
1090        let artifact = reference::ReferenceArtifact::barrier(0);
1091        backend
1092            .load_program(
1093                context,
1094                ArtifactRef {
1095                    format: reference::ARTIFACT_FORMAT,
1096                    target: reference::TARGET_IDENTITY,
1097                    payload: artifact.as_bytes(),
1098                    resident_bytes: reference::RESIDENT_BYTES,
1099                },
1100            )
1101            .unwrap()
1102    }
1103
1104    fn rejected_resource<R>(failure: ReleaseFailure<R>) -> R {
1105        match failure {
1106            ReleaseFailure::Rejected { resource, .. } => resource,
1107            ReleaseFailure::Indeterminate { error } => {
1108                panic!("expected rejected release, got {error:?}")
1109            }
1110        }
1111    }
1112
1113    #[test]
1114    fn scripts_reject_invalid_or_ambiguous_steps() {
1115        assert_eq!(
1116            FaultScript::new([FaultStep::new(
1117                FaultPoint::Submit,
1118                0,
1119                FaultAction::Rejected(BackendError::Busy),
1120            )]),
1121            Err(FaultScriptError::ZeroOccurrence {
1122                point: FaultPoint::Submit,
1123            })
1124        );
1125        assert!(matches!(
1126            FaultScript::new([
1127                FaultStep::new(
1128                    FaultPoint::PollEvent,
1129                    1,
1130                    FaultAction::ErrorBefore(BackendError::Busy),
1131                ),
1132                FaultStep::new(
1133                    FaultPoint::PollEvent,
1134                    1,
1135                    FaultAction::ErrorAfter(BackendError::DeviceLost),
1136                ),
1137            ]),
1138            Err(FaultScriptError::DuplicateStep { .. })
1139        ));
1140        assert!(matches!(
1141            FaultScript::new([FaultStep::new(
1142                FaultPoint::FreeBuffer,
1143                1,
1144                FaultAction::ErrorBefore(BackendError::Busy),
1145            )]),
1146            Err(FaultScriptError::IncompatibleAction { .. })
1147        ));
1148        assert!(matches!(
1149            FaultScript::new([FaultStep::new(
1150                FaultPoint::PollEvent,
1151                1,
1152                FaultAction::Completion(EventState::Pending),
1153            )]),
1154            Err(FaultScriptError::IncompatibleAction { .. })
1155        ));
1156    }
1157
1158    #[test]
1159    fn every_nonrelease_boundary_is_faultable_and_replayable() {
1160        let script = FaultScript::new([
1161            FaultStep::new(
1162                FaultPoint::DeviceInfo,
1163                1,
1164                FaultAction::ErrorBefore(BackendError::Busy),
1165            ),
1166            FaultStep::new(
1167                FaultPoint::CreateContext,
1168                1,
1169                FaultAction::ErrorBefore(BackendError::OutOfMemory),
1170            ),
1171            FaultStep::new(
1172                FaultPoint::CreateContext,
1173                2,
1174                FaultAction::ErrorAfter(BackendError::OutOfMemory),
1175            ),
1176            FaultStep::new(
1177                FaultPoint::AllocateBuffer,
1178                1,
1179                FaultAction::ErrorBefore(BackendError::OutOfMemory),
1180            ),
1181            FaultStep::new(
1182                FaultPoint::AllocateBuffer,
1183                2,
1184                FaultAction::ErrorAfter(BackendError::OutOfMemory),
1185            ),
1186            FaultStep::new(
1187                FaultPoint::WriteBuffer,
1188                1,
1189                FaultAction::ErrorBefore(BackendError::DeviceLost),
1190            ),
1191            FaultStep::new(
1192                FaultPoint::ReadBuffer,
1193                1,
1194                FaultAction::ErrorAfter(BackendError::DeviceLost),
1195            ),
1196            FaultStep::new(
1197                FaultPoint::LoadProgram,
1198                1,
1199                FaultAction::ErrorBefore(BackendError::OutOfMemory),
1200            ),
1201            FaultStep::new(
1202                FaultPoint::LoadProgram,
1203                2,
1204                FaultAction::ErrorAfter(BackendError::OutOfMemory),
1205            ),
1206            FaultStep::new(
1207                FaultPoint::CreateQueue,
1208                1,
1209                FaultAction::ErrorBefore(BackendError::ResourceLimit),
1210            ),
1211            FaultStep::new(
1212                FaultPoint::CreateQueue,
1213                2,
1214                FaultAction::ErrorAfter(BackendError::ResourceLimit),
1215            ),
1216            FaultStep::new(
1217                FaultPoint::Submit,
1218                1,
1219                FaultAction::Rejected(BackendError::Busy),
1220            ),
1221            FaultStep::new(
1222                FaultPoint::Submit,
1223                2,
1224                FaultAction::Indeterminate(BackendError::DeadlineExpired),
1225            ),
1226            FaultStep::new(
1227                FaultPoint::PollEvent,
1228                1,
1229                FaultAction::ErrorBefore(BackendError::Busy),
1230            ),
1231            FaultStep::new(
1232                FaultPoint::PollEvent,
1233                3,
1234                FaultAction::Completion(EventState::Failed(BackendError::DeadlineExpired)),
1235            ),
1236            FaultStep::new(
1237                FaultPoint::CancelEvent,
1238                1,
1239                FaultAction::ErrorBefore(BackendError::Busy),
1240            ),
1241        ])
1242        .unwrap();
1243        let backend = FaultAccelerator::new(MockAccelerator::default(), script);
1244        let control = backend.control();
1245
1246        assert_eq!(backend.device_info(), Err(BackendError::Busy));
1247        backend.device_info().unwrap();
1248        assert_eq!(
1249            backend.create_context(ContextDesc::default()).unwrap_err(),
1250            BackendError::OutOfMemory
1251        );
1252        assert_eq!(
1253            backend.create_context(ContextDesc::default()).unwrap_err(),
1254            BackendError::OutOfMemory
1255        );
1256        let context = backend.create_context(ContextDesc::default()).unwrap();
1257
1258        assert_eq!(
1259            backend
1260                .allocate_buffer(&context, buffer_desc())
1261                .unwrap_err(),
1262            BackendError::OutOfMemory
1263        );
1264        assert_eq!(
1265            backend
1266                .allocate_buffer(&context, buffer_desc())
1267                .unwrap_err(),
1268            BackendError::OutOfMemory
1269        );
1270        let allocation = backend.allocate_buffer(&context, buffer_desc()).unwrap();
1271        let (mut buffer, _) = allocation.into_parts();
1272        assert_eq!(
1273            backend.write_buffer(&mut buffer, 0, b"faulted!"),
1274            Err(BackendError::DeviceLost)
1275        );
1276        backend.write_buffer(&mut buffer, 0, b"faulted!").unwrap();
1277        let mut output = [0; 8];
1278        assert_eq!(
1279            backend.read_buffer(&buffer, 0, &mut output),
1280            Err(BackendError::DeviceLost)
1281        );
1282        output.fill(0);
1283        backend.read_buffer(&buffer, 0, &mut output).unwrap();
1284        assert_eq!(&output, b"faulted!");
1285
1286        let artifact = reference::ReferenceArtifact::barrier(0);
1287        let artifact = ArtifactRef {
1288            format: reference::ARTIFACT_FORMAT,
1289            target: reference::TARGET_IDENTITY,
1290            payload: artifact.as_bytes(),
1291            resident_bytes: reference::RESIDENT_BYTES,
1292        };
1293        assert_eq!(
1294            backend.load_program(&context, artifact).unwrap_err(),
1295            BackendError::OutOfMemory
1296        );
1297        assert_eq!(
1298            backend.load_program(&context, artifact).unwrap_err(),
1299            BackendError::OutOfMemory
1300        );
1301        let program = backend.load_program(&context, artifact).unwrap();
1302        assert_eq!(
1303            backend
1304                .create_queue(&context, QueueDesc::default())
1305                .unwrap_err(),
1306            BackendError::ResourceLimit
1307        );
1308        assert_eq!(
1309            backend
1310                .create_queue(&context, QueueDesc::default())
1311                .unwrap_err(),
1312            BackendError::ResourceLimit
1313        );
1314        let queue = backend
1315            .create_queue(&context, QueueDesc::default())
1316            .unwrap();
1317        let bindings = [BindingRef {
1318            slot: 0,
1319            buffer: &buffer,
1320            range: BufferRange::new(0, 8).unwrap(),
1321            access: AccessMode::ReadWrite,
1322        }];
1323        assert!(matches!(
1324            backend.submit(&queue, &program, &bindings, Timeout::Infinite),
1325            Err(SubmitFailure::Rejected(BackendError::Busy))
1326        ));
1327        let event = match backend.submit(&queue, &program, &bindings, Timeout::Infinite) {
1328            Err(SubmitFailure::Indeterminate { error, event }) => {
1329                assert_eq!(error, BackendError::DeadlineExpired);
1330                event
1331            }
1332            _ => panic!("second submit must cross the indeterminate boundary"),
1333        };
1334        assert_eq!(backend.poll_event(&event), Err(BackendError::Busy));
1335        assert_eq!(backend.poll_event(&event), Ok(EventState::Pending));
1336        assert_eq!(backend.cancel_event(&event), Err(BackendError::Busy));
1337        backend.cancel_event(&event).unwrap();
1338
1339        backend.destroy_event(event).unwrap();
1340        let failed_event = backend
1341            .submit(&queue, &program, &bindings, Timeout::Infinite)
1342            .unwrap();
1343        let failed = EventState::Failed(BackendError::DeadlineExpired);
1344        assert_eq!(backend.poll_event(&failed_event), Ok(failed));
1345        assert_eq!(backend.poll_event(&failed_event), Ok(failed));
1346        assert_eq!(backend.cancel_event(&failed_event), Err(BackendError::Busy));
1347        backend.destroy_event(failed_event).unwrap();
1348        backend.destroy_queue(queue).unwrap();
1349        backend.unload_program(program).unwrap();
1350        backend.free_buffer(buffer).unwrap();
1351        backend.destroy_context(context).unwrap();
1352
1353        let snapshot = control.snapshot();
1354        assert!(snapshot.pending_faults.is_empty());
1355        assert_eq!(
1356            snapshot
1357                .releases
1358                .iter()
1359                .filter(|record| record.rollback)
1360                .count(),
1361            4
1362        );
1363        assert!(snapshot.is_clean());
1364        for point in [
1365            FaultPoint::DeviceInfo,
1366            FaultPoint::CreateContext,
1367            FaultPoint::DestroyContext,
1368            FaultPoint::AllocateBuffer,
1369            FaultPoint::WriteBuffer,
1370            FaultPoint::ReadBuffer,
1371            FaultPoint::FreeBuffer,
1372            FaultPoint::LoadProgram,
1373            FaultPoint::UnloadProgram,
1374            FaultPoint::CreateQueue,
1375            FaultPoint::DestroyQueue,
1376            FaultPoint::Submit,
1377            FaultPoint::PollEvent,
1378            FaultPoint::CancelEvent,
1379            FaultPoint::DestroyEvent,
1380        ] {
1381            assert!(snapshot.calls.iter().any(|call| call.point == point));
1382        }
1383    }
1384
1385    #[test]
1386    fn every_resource_release_supports_rejected_and_indeterminate_outcomes() {
1387        let context_script = FaultScript::new([
1388            FaultStep::new(
1389                FaultPoint::DestroyContext,
1390                1,
1391                FaultAction::Rejected(BackendError::Busy),
1392            ),
1393            FaultStep::new(
1394                FaultPoint::DestroyContext,
1395                3,
1396                FaultAction::Indeterminate(BackendError::DeviceLost),
1397            ),
1398        ])
1399        .unwrap();
1400        let backend = FaultAccelerator::new(MockAccelerator::default(), context_script);
1401        let control = backend.control();
1402        let first = backend.create_context(ContextDesc::default()).unwrap();
1403        let second = backend.create_context(ContextDesc::default()).unwrap();
1404        let first = rejected_resource(backend.destroy_context(first).unwrap_err());
1405        backend.destroy_context(first).unwrap();
1406        assert!(matches!(
1407            backend.destroy_context(second),
1408            Err(ReleaseFailure::Indeterminate {
1409                error: BackendError::DeviceLost
1410            })
1411        ));
1412        assert_eq!(
1413            control
1414                .snapshot()
1415                .resources_in(ResourceState::Indeterminate),
1416            ResourceTally {
1417                contexts: 1,
1418                ..ResourceTally::default()
1419            }
1420        );
1421        control.discard_all();
1422        assert!(control.snapshot().is_clean());
1423
1424        let buffer_script = FaultScript::new([
1425            FaultStep::new(
1426                FaultPoint::FreeBuffer,
1427                1,
1428                FaultAction::Rejected(BackendError::Busy),
1429            ),
1430            FaultStep::new(
1431                FaultPoint::FreeBuffer,
1432                3,
1433                FaultAction::Indeterminate(BackendError::DeviceLost),
1434            ),
1435        ])
1436        .unwrap();
1437        let backend = FaultAccelerator::new(MockAccelerator::default(), buffer_script);
1438        let control = backend.control();
1439        let context = backend.create_context(ContextDesc::default()).unwrap();
1440        let first = backend
1441            .allocate_buffer(&context, buffer_desc())
1442            .unwrap()
1443            .into_parts()
1444            .0;
1445        let second = backend
1446            .allocate_buffer(&context, buffer_desc())
1447            .unwrap()
1448            .into_parts()
1449            .0;
1450        let first = rejected_resource(backend.free_buffer(first).unwrap_err());
1451        backend.free_buffer(first).unwrap();
1452        assert!(matches!(
1453            backend.free_buffer(second),
1454            Err(ReleaseFailure::Indeterminate { .. })
1455        ));
1456        assert_eq!(
1457            control
1458                .snapshot()
1459                .resources_in(ResourceState::Indeterminate)
1460                .buffers,
1461            1
1462        );
1463        control.discard_all();
1464        assert!(control.snapshot().is_clean());
1465
1466        let program_script = FaultScript::new([
1467            FaultStep::new(
1468                FaultPoint::UnloadProgram,
1469                1,
1470                FaultAction::Rejected(BackendError::Busy),
1471            ),
1472            FaultStep::new(
1473                FaultPoint::UnloadProgram,
1474                3,
1475                FaultAction::Indeterminate(BackendError::DeviceLost),
1476            ),
1477        ])
1478        .unwrap();
1479        let backend = FaultAccelerator::new(MockAccelerator::default(), program_script);
1480        let control = backend.control();
1481        let context = backend.create_context(ContextDesc::default()).unwrap();
1482        let first = load_barrier(&backend, &context);
1483        let second = load_barrier(&backend, &context);
1484        let first = rejected_resource(backend.unload_program(first).unwrap_err());
1485        backend.unload_program(first).unwrap();
1486        assert!(matches!(
1487            backend.unload_program(second),
1488            Err(ReleaseFailure::Indeterminate { .. })
1489        ));
1490        assert_eq!(
1491            control
1492                .snapshot()
1493                .resources_in(ResourceState::Indeterminate)
1494                .programs,
1495            1
1496        );
1497        control.discard_all();
1498        assert!(control.snapshot().is_clean());
1499
1500        let queue_script = FaultScript::new([
1501            FaultStep::new(
1502                FaultPoint::DestroyQueue,
1503                1,
1504                FaultAction::Rejected(BackendError::Busy),
1505            ),
1506            FaultStep::new(
1507                FaultPoint::DestroyQueue,
1508                3,
1509                FaultAction::Indeterminate(BackendError::DeviceLost),
1510            ),
1511        ])
1512        .unwrap();
1513        let backend = FaultAccelerator::new(MockAccelerator::default(), queue_script);
1514        let control = backend.control();
1515        let context = backend.create_context(ContextDesc::default()).unwrap();
1516        let first = backend
1517            .create_queue(&context, QueueDesc::default())
1518            .unwrap();
1519        let second = backend
1520            .create_queue(&context, QueueDesc::default())
1521            .unwrap();
1522        let first = rejected_resource(backend.destroy_queue(first).unwrap_err());
1523        backend.destroy_queue(first).unwrap();
1524        assert!(matches!(
1525            backend.destroy_queue(second),
1526            Err(ReleaseFailure::Indeterminate { .. })
1527        ));
1528        assert_eq!(
1529            control
1530                .snapshot()
1531                .resources_in(ResourceState::Indeterminate)
1532                .queues,
1533            1
1534        );
1535        control.discard_all();
1536        assert!(control.snapshot().is_clean());
1537
1538        let event_script = FaultScript::new([
1539            FaultStep::new(
1540                FaultPoint::DestroyEvent,
1541                1,
1542                FaultAction::Rejected(BackendError::Busy),
1543            ),
1544            FaultStep::new(
1545                FaultPoint::DestroyEvent,
1546                3,
1547                FaultAction::Indeterminate(BackendError::DeviceLost),
1548            ),
1549        ])
1550        .unwrap();
1551        let backend = FaultAccelerator::new(MockAccelerator::default(), event_script);
1552        let control = backend.control();
1553        let context = backend.create_context(ContextDesc::default()).unwrap();
1554        let buffer = backend
1555            .allocate_buffer(&context, buffer_desc())
1556            .unwrap()
1557            .into_parts()
1558            .0;
1559        let program = load_barrier(&backend, &context);
1560        let queue = backend
1561            .create_queue(&context, QueueDesc::default())
1562            .unwrap();
1563        let bindings = [BindingRef {
1564            slot: 0,
1565            buffer: &buffer,
1566            range: BufferRange::new(0, 8).unwrap(),
1567            access: AccessMode::ReadWrite,
1568        }];
1569        let first = backend
1570            .submit(&queue, &program, &bindings, Timeout::Infinite)
1571            .unwrap();
1572        let second = backend
1573            .submit(&queue, &program, &bindings, Timeout::Infinite)
1574            .unwrap();
1575        backend.cancel_event(&first).unwrap();
1576        backend.cancel_event(&second).unwrap();
1577        let first = rejected_resource(backend.destroy_event(first).unwrap_err());
1578        backend.destroy_event(first).unwrap();
1579        assert!(matches!(
1580            backend.destroy_event(second),
1581            Err(ReleaseFailure::Indeterminate { .. })
1582        ));
1583        assert_eq!(
1584            control
1585                .snapshot()
1586                .resources_in(ResourceState::Indeterminate)
1587                .events,
1588            1
1589        );
1590        control.discard_all();
1591        assert!(control.snapshot().is_clean());
1592    }
1593
1594    #[test]
1595    fn ownership_audit_detects_double_release_and_leaks() {
1596        let backend = FaultAccelerator::new(MockAccelerator::default(), FaultScript::default());
1597        let control = backend.control();
1598        let context = backend.create_context(ContextDesc::default()).unwrap();
1599        let duplicate = context.clone();
1600        backend.destroy_context(context).unwrap();
1601        assert!(matches!(
1602            backend.destroy_context(duplicate),
1603            Err(ReleaseFailure::Indeterminate {
1604                error: BackendError::DeviceLost
1605            })
1606        ));
1607        assert!(matches!(
1608            control.snapshot().violations.as_slice(),
1609            [OwnershipViolation::ResourceNotLive {
1610                kind: ResourceKind::Context,
1611                state: ResourceState::Released,
1612                ..
1613            }]
1614        ));
1615
1616        let premature_backend =
1617            FaultAccelerator::new(MockAccelerator::default(), FaultScript::default());
1618        let premature_control = premature_backend.control();
1619        let context = premature_backend
1620            .create_context(ContextDesc::default())
1621            .unwrap();
1622        let duplicate = context.clone();
1623        let _buffer = premature_backend
1624            .allocate_buffer(&context, buffer_desc())
1625            .unwrap()
1626            .into_parts()
1627            .0;
1628        assert!(matches!(
1629            premature_backend.destroy_context(duplicate),
1630            Err(ReleaseFailure::Indeterminate {
1631                error: BackendError::DeviceLost
1632            })
1633        ));
1634        assert!(matches!(
1635            premature_control.snapshot().violations.as_slice(),
1636            [OwnershipViolation::PrematureRelease {
1637                kind: ResourceKind::Context,
1638                ..
1639            }]
1640        ));
1641
1642        let leak_backend =
1643            FaultAccelerator::new(MockAccelerator::default(), FaultScript::default());
1644        let leak_control = leak_backend.control();
1645        {
1646            let _leaked = leak_backend.create_context(ContextDesc::default()).unwrap();
1647        }
1648        assert_eq!(
1649            leak_control
1650                .snapshot()
1651                .resources_in(ResourceState::Live)
1652                .contexts,
1653            1
1654        );
1655        leak_control.discard_all();
1656        assert!(leak_control.snapshot().is_clean());
1657    }
1658}
virtio_accel_mock/fault.rs

virtio_accel_mock/
fault.rs
