Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions src/core/guest.c
Original file line number Diff line number Diff line change
Expand Up @@ -651,17 +651,19 @@ void guest_destroy(guest_t *g)
* intact.
*
* The wake signals cover workers blocked outside hv_vcpu_run: futex waiters
* poll futex_interrupt_requested, and any thread parked in epoll or poll
* wakes off the shared pipe. Without them, host-blocked workers miss the
* hv_vcpus_exit kick (which only affects threads inside hv_vcpu_run) and
* the 100ms join cap in thread_join_workers detaches them, leaving live
* pthreads to crash on the imminent munmap.
* poll futex_interrupt_requested, any thread parked in epoll or poll wakes
* off the shared pipe, and thread_wake_exit_waiters broadcasts the internal
* condvars (fork barrier, ptrace stop/wait). Without them, host-blocked
* workers miss the hv_vcpus_exit kick (which only affects threads inside
* hv_vcpu_run) and the 100ms join cap in thread_join_workers detaches
* them, leaving live pthreads to crash on the imminent munmap.
*/
if (!proc_exit_group_requested())
proc_request_exit_group(0);
futex_interrupt_request();
wakeup_pipe_signal();
thread_interrupt_all();
thread_wake_exit_waiters();
thread_join_workers();
/* Destroy all remaining worker vCPUs (thread table) before tearing down the
* VM. This prevents hv_vm_destroy from racing with active vCPUs that may
Expand Down
5 changes: 5 additions & 0 deletions src/main.c
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,11 @@ int main(int argc, char **argv)
futex_interrupt_request();
wakeup_pipe_signal();
thread_interrupt_all();
/* Workers parked on internal condvars (fork barrier, ptrace stop/wait)
* see neither the pipe nor the vCPU kick; broadcast so they re-check the
* exit-group flag and terminate before the join below gives up on them.
*/
thread_wake_exit_waiters();
thread_join_workers();

/* Diagnostic counter dump runs before guest_destroy so the shim_data
Expand Down
5 changes: 5 additions & 0 deletions src/runtime/forkipc.c
Original file line number Diff line number Diff line change
Expand Up @@ -1288,6 +1288,11 @@ static void *vm_clone_thread_run(void *arg)
* harmless no-op.
*/
thread_interrupt_all();
/* thread_interrupt_all only reaches threads inside hv_vcpu_run.
* Peers parked on fork_cond or another slot's ptrace_cond/resume_cond
* see neither the vCPU kick nor this exit, so broadcast separately.
*/
thread_wake_exit_waiters();
}

hv_vcpu_destroy(vcpu);
Expand Down
70 changes: 64 additions & 6 deletions src/runtime/thread.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@

#include "runtime/thread.h"
#include "debug/log.h"
#include "core/guest.h" /* guest_t (shim_data_base/ipa_base), BLOCK_2MIB */
#include "hvutil.h" /* vcpu_get_gpr, vcpu_get_sysreg */
#include "core/guest.h" /* guest_t (shim_data_base/ipa_base), BLOCK_2MIB */
#include "hvutil.h" /* vcpu_get_gpr, vcpu_get_sysreg */
#include "syscall/proc.h" /* proc_exit_group_requested */

/* From syscall/signal.h, included here directly to avoid pulling in the full
* signal header (macOS defines sa_handler as a macro that conflicts with the
Expand Down Expand Up @@ -474,6 +475,16 @@ void thread_join_workers(void)
THREAD_FOR_EACH (t) {
if (t == current_thread || t->join_abandoned)
continue;
/* Never wait for the main thread (slot 0): its entry only deactivates
* inside guest_destroy, which runs on the main thread AFTER its own
* thread_join_workers. A worker waiting here (exit_group called from
* a worker) would burn the full cap on it, and the resulting mutual
* wait (worker polling main while main polls the worker) ends in
* pthread_detach on both sides, leaving the loser to touch guest
* memory after guest_destroy unmaps it.
*/
if (t == &thread_table[0])
continue;
/* Inactive slots are included when they still hold an unjoined handle:
* a worker that exited on its own shortly before teardown and whose
* slot was never reused. Its pthread has terminated (or is in final
Expand Down Expand Up @@ -709,14 +720,46 @@ int thread_fork_barrier_check(void)
pthread_cond_signal(&fork_all_quiesced_cond);
}

/* Block until fork is complete */
while (fork_quiesce_active)
/* Block until fork is complete. Bail out on exit_group: the resume
* broadcast comes from the forking thread, whose progress the teardown
* path does not control, so waiting for it would leave this park outside
* the bounded-wake guarantee. thread_wake_exit_waiters broadcasts
* fork_cond after the flag is set; the caller's run loop re-checks
* proc_exit_group_requested and exits.
*/
while (fork_quiesce_active && !proc_exit_group_requested())
pthread_cond_wait(&fork_cond, &thread_lock);

pthread_mutex_unlock(&thread_lock);
return 1;
}

void thread_wake_exit_waiters(void)
{
pthread_mutex_lock(&thread_lock);

/* Fork barrier: siblings parked in thread_fork_barrier_check. Their wait
* loop re-checks proc_exit_group_requested on wake.
*/
pthread_cond_broadcast(&fork_cond);

/* Ptrace parks: tracers blocked in thread_ptrace_wait (ptrace_cond) and
* tracees blocked in thread_ptrace_stop (resume_cond). Scan every slot
* with live condvars, not just active ones: a tracer may still be parked
* on a slot whose thread was deactivated. ptrace_conds_inited only
* transitions under thread_lock with ptrace_waiters == 0, so broadcasting
* here never touches a destroyed condvar.
*/
THREAD_FOR_EACH (t) {
if (!t->ptrace_conds_inited)
continue;
pthread_cond_broadcast(&t->ptrace_cond);
pthread_cond_broadcast(&t->resume_cond);
}

pthread_mutex_unlock(&thread_lock);
}

/* Ptrace helpers. */

pthread_mutex_t *thread_get_lock(void)
Expand Down Expand Up @@ -1037,8 +1080,13 @@ int thread_ptrace_stop(thread_entry_t *t, int sig)
/* Wake the tracer (blocked in thread_ptrace_wait) */
pthread_cond_broadcast(&t->ptrace_cond);

/* Block until tracer calls PTRACE_CONT */
while (t->ptrace_stopped)
/* Block until tracer calls PTRACE_CONT. Bail out on exit_group: only the
* tracer signals resume_cond, and a tracer that exits (or calls
* exit_group itself) will never CONT this stop. thread_wake_exit_waiters
* broadcasts resume_cond; returning 0 sends the caller back to its run
* loop, which re-checks proc_exit_group_requested.
*/
while (t->ptrace_stopped && !proc_exit_group_requested())
pthread_cond_wait(&t->resume_cond, &thread_lock);

/* Apply register changes if tracer wrote via SETREGSET */
Expand Down Expand Up @@ -1093,6 +1141,16 @@ int64_t thread_ptrace_wait(int64_t tracer_tid,
pthread_mutex_lock(&thread_lock);

for (;;) {
/* exit_group teardown: the stop/exit notifications that would signal
* ptrace_cond stop arriving once workers are being torn down. Return 0
* ("no matching children") so the caller falls through and its
* blocking paths re-check proc_exit_group_requested.
*/
if (proc_exit_group_requested()) {
pthread_mutex_unlock(&thread_lock);
return 0;
}

bool found_any = false; /* Any waitable children at all? */

THREAD_FOR_EACH (t) {
Expand Down
11 changes: 11 additions & 0 deletions src/runtime/thread.h
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,17 @@ void thread_destroy_all_vcpus(void);
*/
void thread_interrupt_all(void);

/* Wake workers parked on internal condvars (fork barrier, ptrace stop/wait)
* so exit_group teardown reaches them within a bounded time. hv_vcpus_exit
* only interrupts threads inside hv_vcpu_run, and the wakeup pipe / futex
* interrupt only cover guest syscall waits; a thread parked on fork_cond,
* ptrace_cond, or resume_cond would otherwise sleep past the join cap in
* thread_join_workers and still be live when guest memory is unmapped.
* Callers must set the exit-group flag (proc_request_exit_group) BEFORE
* calling this so woken waiters observe it when they re-check.
*/
void thread_wake_exit_waiters(void);

/* Check if any active thread has sigbit unblocked in its signal mask. Uses
* relaxed reads on per-thread blocked fields; false positives (stale blocked=0)
* cause a harmless spurious interrupt; false negatives (stale blocked=1) are
Expand Down
10 changes: 10 additions & 0 deletions src/syscall/proc.c
Original file line number Diff line number Diff line change
Expand Up @@ -1107,6 +1107,16 @@ int64_t sys_wait4(guest_t *g,
return 0;
}

/* exit_group teardown: stop re-arming the wait. The 100ms quantum
Comment thread
Max042004 marked this conversation as resolved.
* below bounds how stale this check can be, mirroring the futex
* wait quanta. The errno is never guest-visible: the run loop
* breaks on the exit-group flag before returning to the guest.
*/
if (proc_exit_group_requested()) {
pthread_mutex_unlock(&pid_lock);
return -LINUX_EINTR;
}

/* Blocking mode: no child exited yet. Wait on condvar for a child
* exit notification (signaled by proc_mark_child_exited). Use
* timedwait with 100ms timeout as a safety net; the condvar handles
Expand Down
6 changes: 6 additions & 0 deletions src/syscall/syscall.c
Original file line number Diff line number Diff line change
Expand Up @@ -909,6 +909,12 @@ static int64_t sc_exit_group(guest_t *g,
proc_request_exit_group((int) x0);
wakeup_pipe_signal();
thread_for_each(thread_force_exit_cb, NULL);
/* Workers parked on internal condvars (fork barrier, ptrace stop/wait)
* see neither the pipe nor the vCPU kick; broadcast so they re-check the
* exit-group flag and terminate before the authoritative join in main()
* (or guest_destroy, for the forked-child path) gives up on them.
*/
thread_wake_exit_waiters();
Comment thread
Max042004 marked this conversation as resolved.
return SC_EXIT_SENTINEL | ((int) x0 & 0xFF);
}

Expand Down
5 changes: 5 additions & 0 deletions tests/manifest.txt
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,11 @@ test-fd-lifecycle
[section] Multithreaded fork tests
test-mt-fork

[section] Exit-group teardown reachability tests
test-exit-group-teardown wait # expected_rc=42
test-exit-group-teardown stop # expected_rc=42
test-exit-group-teardown fork # expected_rc=42

[section] SysV shared memory tests
test-sysv-shm

Expand Down
Loading
Loading