Syscall Handling — Professional Level¶

//go:nosplit
func entersyscall() {
    reentersyscall(getcallerpc(), getcallersp())
}

//go:nosplit
func reentersyscall(pc, sp uintptr) {
    gp := getg()

    // Disable preemption because we hold m.p.
    gp.m.locks++
    gp.stackguard0 = stackPreempt
    gp.throwsplit = true

    // Leave SP and PC for trace.
    save(pc, sp)
    gp.syscallsp = sp
    gp.syscallpc = pc
    casgstatus(gp, _Grunning, _Gsyscall)

    if staticLockRanking {
        // ... lock rank tracking ...
    }

    if sched.sysmonwait.Load() {
        systemstack(entersyscall_sysmon)
    }

    if gp.m.p.ptr().runSafePointFn != 0 {
        // runSafePointFn may schedule, so it has to be done on the systemstack.
        systemstack(runSafePointFn)
    }

    gp.m.syscalltick = gp.m.p.ptr().syscalltick
    gp.sysblocktraced = true
    pp := gp.m.p.ptr()
    pp.m = 0
    gp.m.oldp.set(pp)
    gp.m.p = 0
    atomic.Store(&pp.status, _Psyscall)
    if sched.gcwaiting.Load() {
        systemstack(entersyscall_gcwait)
    }

    gp.m.locks--
}

//go:nosplit
func exitsyscall() {
    gp := getg()

    gp.m.locks++
    if getcallersp() > gp.syscallsp {
        throw("exitsyscall: syscall frame is no longer valid")
    }

    gp.waitsince = 0
    oldp := gp.m.oldp.ptr()
    gp.m.oldp = 0
    if exitsyscallfast(oldp) {
        // Fast path: re-acquired oldp (or some P).
        // ... profiling hooks ...
        gp.m.locks--
        if gp.preempt {
            // Preemption was requested while we were in syscall.
            gp.stackguard0 = stackPreempt
        } else {
            gp.stackguard0 = gp.stack.lo + _StackGuard
        }
        gp.throwsplit = false

        if sched.disable.user && !schedEnabled(gp) {
            // Schedule was disabled while we were away.
            Gosched()
        }
        return
    }

    gp.m.locks--

    // Slow path. Park M, requeue G.
    mcall(exitsyscall0)

    // Re-enter Go scheduling on the new M.
    gp.syscallsp = 0
    gp.m.syscalltick++
    gp.throwsplit = false
}

//go:nosplit
func exitsyscallfast(oldp *p) bool {
    // Freezetheworld set the stopwait but did not retake P's.
    if sched.stopwait == freezeStopWait {
        return false
    }

    // Try to re-acquire the old P.
    if oldp != nil && oldp.status == _Psyscall && atomic.Cas(&oldp.status, _Psyscall, _Pidle) {
        // Yes! Re-attach.
        wirep(oldp)
        exitsyscallfast_reacquired()
        return true
    }

    // Try to get any other P.
    if sched.pidle != 0 {
        var ok bool
        systemstack(func() {
            ok = exitsyscallfast_pidle()
        })
        if ok {
            return true
        }
    }
    return false
}

atomic.Cas(&oldp.status, _Psyscall, _Pidle)

func exitsyscallfast_pidle() bool {
    lock(&sched.lock)
    pp, _ := pidlegetSpinning(0)
    if pp != nil && atomic.Load(&sched.sysmonwait) != 0 {
        atomic.Store(&sched.sysmonwait, 0)
        notewakeup(&sched.sysmonnote)
    }
    unlock(&sched.lock)
    if pp != nil {
        acquirep(pp)
        return true
    }
    return false
}

func sysmon() {
    lock(&sched.lock)
    sched.nmsys++
    unlock(&sched.lock)

    lasttrace := int64(0)
    idle := 0
    delay := uint32(0)

    for {
        if idle == 0 {
            delay = 20  // 20 microseconds
        } else if idle > 50 {
            delay *= 2
        }
        if delay > 10*1000 {
            delay = 10 * 1000  // cap at 10 ms
        }
        usleep(delay)

        now := nanotime()

        // ... scheduler bookkeeping ...

        // Retake: preempt long-running Gs, hand off syscall Ps.
        if retake(now) != 0 {
            idle = 0
        } else {
            idle++
        }

        // ... GC trigger, scavenge, etc ...
    }
}

func retake(now int64) uint32 {
    n := 0
    lock(&allpLock)
    for i := 0; i < len(allp); i++ {
        pp := allp[i]
        if pp == nil {
            continue
        }
        pd := &pp.sysmontick
        s := pp.status
        sysretake := false
        if s == _Prunning || s == _Psyscall {
            // Preempt G if it's running too long.
            t := int64(pp.schedtick)
            if int64(pd.schedtick) != t {
                pd.schedtick = uint32(t)
                pd.schedwhen = now
            } else if pd.schedwhen+forcePreemptNS <= now {
                preemptone(pp)
                sysretake = true
            }
        }
        if s == _Psyscall {
            // Retake P from syscall if it's been more than 1 sysmon tick.
            t := int64(pp.syscalltick)
            if !sysretake && int64(pd.syscalltick) != t {
                pd.syscalltick = uint32(t)
                pd.syscallwhen = now
                continue
            }
            if runqempty(pp) && sched.nmspinning.Load()+sched.npidle.Load() > 0 &&
                pd.syscallwhen+10*1000 > now {
                continue
            }
            // Need to decrement number of idle locked M's
            // (pretending that one more is running) before the CAS.
            unlock(&allpLock)
            incidlelocked(-1)
            if atomic.Cas(&pp.status, s, _Pidle) {
                if traceEnabled() {
                    traceGoSysBlock(pp)
                    traceProcStop(pp)
                }
                n++
                pp.syscalltick++
                handoffp(pp)
            }
            incidlelocked(1)
            lock(&allpLock)
        }
    }
    unlock(&allpLock)
    return uint32(n)
}

if runqempty(pp) && sched.nmspinning.Load()+sched.npidle.Load() > 0 &&
    pd.syscallwhen+10*1000 > now {
    continue
}

func handoffp(pp *p) {
    // 1. If P has runnable work, start an M to run it.
    if !runqempty(pp) || sched.runqsize != 0 {
        startm(pp, false)
        return
    }
    // 2. If GC wants the P, start an M.
    if gcBlackenEnabled != 0 && gcMarkWorkAvailable(pp) {
        startm(pp, false)
        return
    }
    // 3. If no spinning M and no idle P, start a spinning M.
    if sched.nmspinning.Load()+sched.npidle.Load() == 0 &&
        sched.nmspinning.CompareAndSwap(0, 1) {
        startm(pp, true)
        return
    }
    lock(&sched.lock)
    if sched.gcwaiting.Load() {
        pp.status = _Pgcstop
        sched.stopwait--
        if sched.stopwait == 0 {
            notewakeup(&sched.stopnote)
        }
        unlock(&sched.lock)
        return
    }
    // 4. If safe-point function pending, run it.
    if pp.runSafePointFn != 0 && atomic.Cas(&pp.runSafePointFn, 1, 0) {
        sched.safePointFn(pp)
        sched.safePointWait--
        if sched.safePointWait == 0 {
            notewakeup(&sched.safePointNote)
        }
    }
    if sched.runqsize != 0 {
        unlock(&sched.lock)
        startm(pp, false)
        return
    }
    // ... more special cases ...

    // 5. No work for this P; park it.
    pidleput(pp, 0)
    unlock(&sched.lock)
}

func startm(pp *p, spinning bool) {
    mp := mget()  // try to get a parked M from sched.midle
    if mp == nil {
        // No parked M. Create one.
        var fn func()
        if spinning {
            fn = mspinning
        }
        newm(fn, pp, -1)
        return
    }
    // Wake an existing parked M.
    if mp.spinning {
        throw("startm: m is spinning")
    }
    if mp.nextp != 0 {
        throw("startm: m has p")
    }
    mp.spinning = spinning
    mp.nextp.set(pp)
    notewakeup(&mp.park)
}

// runtime/os_linux.go
func newosproc(mp *m) {
    stk := unsafe.Pointer(mp.g0.stack.hi)
    var oset sigset
    sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
    ret := retryOnEAGAIN(func() int32 {
        r := clone(cloneFlags, stk,
            unsafe.Pointer(mp),
            unsafe.Pointer(mp.g0),
            unsafe.Pointer(abi.FuncPCABI0(mstart)))
        if r >= 0 {
            return 0
        }
        return -r
    })
    sigprocmask(_SIG_SETMASK, &oset, nil)
    if ret != 0 {
        print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n")
        if ret == _EAGAIN {
            println("runtime: may need to increase max user processes (ulimit -u)")
        }
        throw("newosproc")
    }
}

var cloneFlags = uintptr(_CLONE_VM | _CLONE_FS | _CLONE_FILES | _CLONE_SIGHAND |
    _CLONE_SYSVSEM | _CLONE_THREAD)

// runtime/os_darwin.go
func newosproc(mp *m) {
    stk := unsafe.Pointer(mp.g0.stack.hi)
    var attr pthreadattr
    if pthread_attr_init(&attr) != 0 {
        writeErrStr(failthreadcreate)
        exit(1)
    }
    if pthread_attr_setstacksize(&attr, threadStackSize) != 0 {
        writeErrStr(failthreadcreate)
        exit(1)
    }
    if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 {
        writeErrStr(failthreadcreate)
        exit(1)
    }
    var oset sigset
    sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
    err := pthread_create(&attr, abi.FuncPCABI0(mstart_stub), unsafe.Pointer(mp))
    sigprocmask(_SIG_SETMASK, &oset, nil)
    if err != 0 {
        writeErrStr(failthreadcreate)
        exit(1)
    }
}

// runtime/os_windows.go
func newosproc(mp *m) {
    thandle := stdcall6(_CreateThread, 0, 0,
        abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)),
        0, 0)
    // ...
}

// runtime/netpoll.go
func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) {
    pd := pollcache.alloc()
    // ... init pollDesc ...
    var errno int32
    errno = netpollopen(fd, pd)
    return pd, int(errno)
}

// runtime/netpoll_epoll.go
func netpollopen(fd uintptr, pd *pollDesc) int32 {
    var ev epollevent
    ev.events = _EPOLLIN | _EPOLLOUT | _EPOLLRDHUP | _EPOLLET
    *(**pollDesc)(unsafe.Pointer(&ev.data)) = pd
    return -epollctl(epfd, _EPOLL_CTL_ADD, int32(fd), &ev)
}

// runtime/netpoll.go (called by scheduler)
func netpoll(delay int64) gList {
    if epfd == -1 {
        return gList{}
    }
    var waitms int32
    if delay < 0 {
        waitms = -1
    } else if delay == 0 {
        waitms = 0
    } else if delay < 1e6 {
        waitms = 1
    } else if delay < 1e15 {
        waitms = int32(delay / 1e6)
    } else {
        waitms = 1e9
    }

    var events [128]epollevent
retry:
    n := epollwait(epfd, &events[0], int32(len(events)), waitms)
    if n < 0 {
        if n != -_EINTR {
            println("runtime: epollwait on fd", epfd, "failed with", -n)
            throw("runtime: netpoll failed")
        }
        if waitms > 0 {
            return gList{}
        }
        goto retry
    }
    var toRun gList
    for i := int32(0); i < n; i++ {
        ev := &events[i]
        if ev.events == 0 {
            continue
        }
        // ... handle ev_wakeup_pipe ...
        pd := *(**pollDesc)(unsafe.Pointer(&ev.data))
        var mode int32
        if ev.events&(_EPOLLIN|_EPOLLRDHUP|_EPOLLHUP|_EPOLLERR) != 0 {
            mode += 'r'
        }
        if ev.events&(_EPOLLOUT|_EPOLLHUP|_EPOLLERR) != 0 {
            mode += 'w'
        }
        if mode != 0 {
            netpollready(&toRun, pd, mode)
        }
    }
    return toRun
}

//go:nosplit
func entersyscallblock() {
    gp := getg()
    gp.m.locks++
    gp.throwsplit = true
    gp.stackguard0 = stackPreempt
    gp.m.syscalltick = gp.m.p.ptr().syscalltick
    gp.sysblocktraced = true
    gp.m.p.ptr().syscalltick++

    pc := getcallerpc()
    sp := getcallersp()
    save(pc, sp)
    gp.syscallsp = gp.sched.sp
    gp.syscallpc = gp.sched.pc

    casgstatus(gp, _Grunning, _Gsyscall)

    systemstack(entersyscallblock_handoff)

    save(getcallerpc(), getcallersp())
    gp.m.locks--
}

func entersyscallblock_handoff() {
    if traceEnabled() {
        traceGoSysCall()
        traceGoSysBlock(getg().m.p.ptr())
    }
    handoffp(releasep())
}

// runtime/cgocall.go
func cgocall(fn, arg unsafe.Pointer) int32 {
    if !iscgo && GOOS != "solaris" && GOOS != "illumos" && GOOS != "windows" {
        throw("cgocall unavailable")
    }
    if fn == nil {
        throw("cgocall nil")
    }
    if raceenabled {
        racereleasemerge(unsafe.Pointer(&racecgosync))
    }
    mp := getg().m
    mp.ncgocall++
    mp.ncgo++

    // Reset traceback so that it does not see the cgo call.
    mp.cgoCallers = nil

    // Announce we are entering a system call.
    entersyscall()

    osPreemptExtEnter(mp)

    mp.incgo = true
    errno := asmcgocall(fn, arg)

    mp.incgo = false
    mp.ncgo--

    osPreemptExtExit(mp)

    exitsyscall()

    // From the now-current goroutine, scan stack for any pointers and
    // pin them so GC won't reclaim them while we're in cgo.
    KeepAlive(fn)
    KeepAlive(arg)
    KeepAlive(mp)

    return errno
}

func exitsyscall0(gp *g) {
    casgstatus(gp, _Gsyscall, _Grunnable)
    dropg()
    var pp *p
    if schedEnabled(gp) {
        pp, _ = pidleget(0)
    }
    var locked bool
    if pp == nil {
        globrunqput(gp)
        // Below, we stoplockedm if locked. The schedule on
        // the locked M will check the special handle.
        locked = gp.lockedm != 0
    } else {
        // ... ack sysmon, acquirep ...
    }
    if locked {
        stoplockedm()
        execute(gp, false)
    }
    stopm()
    schedule()
}