diff --git a/pkg/atomicbitops/aligned_32bit_unsafe.go b/pkg/atomicbitops/aligned_32bit_unsafe.go index a76c6ed30fa..9905a64f0bc 100644 --- a/pkg/atomicbitops/aligned_32bit_unsafe.go +++ b/pkg/atomicbitops/aligned_32bit_unsafe.go @@ -198,6 +198,16 @@ func (u *Uint64) RacyStore(v uint64) { *u.ptr() = v } +// StoreRelaxed is actually the same as `Store` on 32-bit architectures, +// since 64-bit plain stores are not atomic there. +// See comment on the analogous of this function in `aligned_64bit.go`. +// +//go:norace +//go:nosplit +func (u *Uint64) StoreRelaxed(v uint64) { + atomic.StoreUint64(u.ptr(), v) +} + // Add is analogous to atomic.AddUint64. // //go:nosplit diff --git a/pkg/atomicbitops/aligned_64bit.go b/pkg/atomicbitops/aligned_64bit.go index ecb37e6bbca..904ae37fd67 100644 --- a/pkg/atomicbitops/aligned_64bit.go +++ b/pkg/atomicbitops/aligned_64bit.go @@ -174,6 +174,20 @@ func (u *Uint64) RacyStore(v uint64) { u.value = v } +// StoreRelaxed stores `v` with **no** ordering guarantee. +// Useful only where readers tolerate a stale value. +// Race detection is disabled, so this must be used sparingly. +// On 64-bit architectures, readers are guaranteed to see either the old +// or the new value, no "partial writes" cases. +// The 32-bit-architecture variant of this function does a real atomic +// write to guarantee the same no-partial-write property. +// +//go:norace +//go:nosplit +func (u *Uint64) StoreRelaxed(v uint64) { + u.value = v +} + // Add is analogous to atomic.AddUint64. // //go:nosplit diff --git a/pkg/sentry/kernel/task.go b/pkg/sentry/kernel/task.go index 0ca12edfc9a..0da01e81145 100644 --- a/pkg/sentry/kernel/task.go +++ b/pkg/sentry/kernel/task.go @@ -108,22 +108,20 @@ type Task struct { // interruptChan is always notified after restore (see Task.run). interruptChan chan struct{} `state:"nosave"` - // gostateSeq allows Task.TaskGoroutineStateTime() to read gostate and - // gostateTime atomically. - // - // gostateSeq is owned by the task goroutine. - gostateSeq sync.SeqCount `state:"nosave"` - - // gostate is the current scheduling state of the task goroutine. + // gostate combines two fields in one 64-bit integer: + // - First `gostateBits` bits are the `TaskGoroutineState` enum value. + // - Rest of the bits are the value of Kernel.cpuClock when the state was + // last updated or refreshed. + // Packing them this way allows efficient atomic reads and writes, which + // is critical for performance on the syscall hot path. + // + // Despite the use of `atomicbitops.Uint64`, `gostate` is written to + // **with no barrier guarantee** from the task goroutine, for syscall hot + // path performance reasons. This means all readers (other than from the + // task goroutine) **must** tolerate stale reads. // // gostate is owned by the task goroutine. - gostate atomicbitops.Uint32 - - // gostateTime was the value of Kernel.cpuClock when gostate was last - // updated or refreshed. - // - // gostateTime is owned by the task goroutine. - gostateTime atomicbitops.Int64 + gostate atomicbitops.Uint64 // appCPUClock approximates the amount of time the task goroutine has spent // in TaskGoroutineRunningApp. @@ -754,7 +752,7 @@ func (t *Task) afterLoad(gocontext.Context) { ts.populateCache(t) } t.interruptChan = make(chan struct{}, 1) - t.gostate.Store(uint32(TaskGoroutineNonexistent)) + t.setGostate(TaskGoroutineNonexistent) if t.stop != nil { t.stopCount = atomicbitops.FromInt32(1) } diff --git a/pkg/sentry/kernel/task_sched.go b/pkg/sentry/kernel/task_sched.go index 43e830632b5..f1bafaa6116 100644 --- a/pkg/sentry/kernel/task_sched.go +++ b/pkg/sentry/kernel/task_sched.go @@ -64,34 +64,49 @@ const ( TaskGoroutineStopped ) +// gostateBits is the width of the `TaskGoroutineState` bits in `Task`'s +// `gostate` field. The remainder of the bits holds `Kernel.cpuClock` +// nanoseconds. With gostateBits = 3, this is enough for 73 years. +const gostateBits = 3 + // TaskGoroutineState returns the current state of the task goroutine. func (t *Task) TaskGoroutineState() TaskGoroutineState { - return TaskGoroutineState(t.gostate.Load()) + return TaskGoroutineState(t.gostate.Load() & (1<> gostateBits)) +} + +// setGostate sets the task goroutine state, timestamped with the current +// `Kernel.cpuClock`. +// +// Preconditions: The caller must be running on the task goroutine. +func (t *Task) setGostate(state TaskGoroutineState) { + // StoreRelaxed due to this being on the syscall hot path, and all readers + // are expected to tolerate stale reads. + t.gostate.StoreRelaxed(uint64(t.k.cpuClock.Load())<