Issue #545 : yield across pcall or a Go call boundary raises the Lua 5.1 error

_glua-tests/coroutine.lua

@@ -15,3 +15,30 @@ local ok, msg = pcall(function()
   co()
 end)
 assert(not ok and string.find(msg, "can not resume a dead thread"))
+
+-- yield across a Go call boundary (pcall, a Go function calling Lua) is an
+-- error, as in Lua 5.1; it used to crash with a nil pointer dereference (#545)
+local co = coroutine.create(function()
+  local ok, err = pcall(function() coroutine.yield(1) end)
+  return ok, err
+end)
+local ok, a, b = coroutine.resume(co)
+assert(ok and a == false and string.find(b, "attempt to yield across metamethod/C-call boundary", 1, true))
+assert(coroutine.status(co) == "dead")
+
+co = coroutine.create(function()
+  return pcall(table.sort, {3, 1, 2}, function(x, y) coroutine.yield(); return x < y end)
+end)
+ok, a, b = coroutine.resume(co)
+assert(ok and a == false and string.find(b, "attempt to yield across", 1, true))
+
+co = coroutine.create(function()
+  pcall(function() coroutine.yield() end)
+  coroutine.yield("again")
+  return "done"
+end)
+assert(select(2, coroutine.resume(co)) == "again")
+assert(select(2, coroutine.resume(co)) == "done")
+
+local gen = coroutine.wrap(function() coroutine.yield(1); return 2 end)
+assert(gen() == 1 and gen() == 2)

_state.go

@@ -1048,13 +1048,15 @@ func (ls *LState) callR(nargs, nret, rbase int) {
 		Parent:     ls.currentFrame,
 		TailCall:   0,
 	}, lv, meta)
+	ls.nCcalls++
 	if ls.G.MainThread == nil {
 		ls.G.MainThread = ls
 		ls.G.CurrentThread = ls
 		ls.mainLoop(ls, nil)
 	} else {
 		ls.mainLoop(ls, ls.currentFrame)
 	}
+	ls.nCcalls--
 	if nret != MultRet {
 		ls.reg.SetTop(rbase + nret)
 	}
@@ -1817,6 +1819,7 @@ func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) {
 	err = nil
 	sp := ls.stack.Sp()
 	base := ls.reg.Top() - nargs - 1
+	nccalls := ls.nCcalls
 	oldpanic := ls.Panic
 	ls.Panic = panicWithoutTraceback
 	if errfunc != nil {
@@ -1827,6 +1830,7 @@ func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) {
 		ls.hasErrorFunc = false
 		rcv := recover()
 		if rcv != nil {
+			ls.nCcalls = nccalls
 			if _, ok := rcv.(*ApiError); !ok {
 				err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv))
 				if ls.Options.IncludeGoStackTrace {
@@ -1856,6 +1860,7 @@ func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) {
 							err = rcv.(*ApiError)
 							err.(*ApiError).StackTrace = ls.stackTrace(0)
 						}
+						ls.nCcalls = nccalls
 						ls.stack.SetSp(sp)
 						ls.currentFrame = ls.stack.Last()
 						ls.reg.SetTop(base)

_vm.go

@@ -110,6 +110,11 @@ func callGFunction(L *LState, tailcall bool) bool {
 	}
 
 	if gfnret < 0 {
+		if L.nCcalls > 0 && L.Parent != nil {
+			// The yield would unwind a Go frame (pcall, a metamethod, a Go
+			// function calling Lua) that cannot be resumed: Lua 5.1 behavior.
+			L.RaiseError("attempt to yield across metamethod/C-call boundary")
+		}
 		switchToParentThread(L, L.GetTop(), false, false)
 		return true
 	}

state.go

@@ -1261,13 +1261,15 @@ func (ls *LState) callR(nargs, nret, rbase int) {
 		Parent:     ls.currentFrame,
 		TailCall:   0,
 	}, lv, meta)
+	ls.nCcalls++
 	if ls.G.MainThread == nil {
 		ls.G.MainThread = ls
 		ls.G.CurrentThread = ls
 		ls.mainLoop(ls, nil)
 	} else {
 		ls.mainLoop(ls, ls.currentFrame)
 	}
+	ls.nCcalls--
 	if nret != MultRet {
 		ls.reg.SetTop(rbase + nret)
 	}
@@ -2030,6 +2032,7 @@ func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) {
 	err = nil
 	sp := ls.stack.Sp()
 	base := ls.reg.Top() - nargs - 1
+	nccalls := ls.nCcalls
 	oldpanic := ls.Panic
 	ls.Panic = panicWithoutTraceback
 	if errfunc != nil {
@@ -2040,6 +2043,7 @@ func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) {
 		ls.hasErrorFunc = false
 		rcv := recover()
 		if rcv != nil {
+			ls.nCcalls = nccalls
 			if _, ok := rcv.(*ApiError); !ok {
 				err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv))
 				if ls.Options.IncludeGoStackTrace {
@@ -2069,6 +2073,7 @@ func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) {
 							err = rcv.(*ApiError)
 							err.(*ApiError).StackTrace = ls.stackTrace(0)
 						}
+						ls.nCcalls = nccalls
 						ls.stack.SetSp(sp)
 						ls.currentFrame = ls.stack.Last()
 						ls.reg.SetTop(base)

value.go

@@ -192,6 +192,7 @@ type LState struct {
 	wrapped      bool
 	uvcache      *Upvalue
 	hasErrorFunc bool
+	nCcalls      int // Go-to-Lua calls (callR) in progress on this thread
 	mainLoop     func(*LState, *callFrame)
 	ctx          context.Context
 	ctxCancelFn  context.CancelFunc

vm.go

@@ -205,6 +205,11 @@ func callGFunction(L *LState, tailcall bool) bool {
 	}
 
 	if gfnret < 0 {
+		if L.nCcalls > 0 && L.Parent != nil {
+			// The yield would unwind a Go frame (pcall, a metamethod, a Go
+			// function calling Lua) that cannot be resumed: Lua 5.1 behavior.
+			L.RaiseError("attempt to yield across metamethod/C-call boundary")
+		}
 		switchToParentThread(L, L.GetTop(), false, false)
 		return true
 	}