cmd/cgo: add go:notinheap annotation to Windows handle types

Fixes #42018

Change-Id: I6a40f3effe860e67a45fca2e8ab86f3e9887ffee
Reviewed-on: https://go-review.googlesource.com/c/go/+/350070
Trust: Elias Naur <[email protected]>
Reviewed-by: Keith Randall <[email protected]>
Run-TryBot: Keith Randall <[email protected]>

misc/cgo/test/cgo_test.go

@@ -59,6 +59,7 @@ func Test28896(t *testing.T)                 { test28896(t) }
 func Test30065(t *testing.T)                 { test30065(t) }
 func Test32579(t *testing.T)                 { test32579(t) }
 func Test31891(t *testing.T)                 { test31891(t) }
+func Test42018(t *testing.T)                 { test42018(t) }
 func Test45451(t *testing.T)                 { test45451(t) }
 func TestAlign(t *testing.T)                 { testAlign(t) }
 func TestAtol(t *testing.T)                  { testAtol(t) }

misc/cgo/test/issue42018.go

@@ -0,0 +1,14 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !windows
+// +build !windows
+
+package cgotest
+
+import "testing"
+
+func test42018(t *testing.T) {
+	t.Skip("skipping Windows-only test")
+}

misc/cgo/test/issue42018_windows.go

@@ -0,0 +1,46 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cgotest
+
+/*
+typedef void *HANDLE;
+
+struct HWND__{int unused;}; typedef struct HWND__ *HWND;
+*/
+import "C"
+
+import (
+	"testing"
+	"unsafe"
+)
+
+func test42018(t *testing.T) {
+	// Test that Windows handles are marked go:notinheap, by growing the
+	// stack and checking for pointer adjustments. Trick from
+	// test/fixedbugs/issue40954.go.
+	var i int
+	handle := C.HANDLE(unsafe.Pointer(uintptr(unsafe.Pointer(&i))))
+	recurseHANDLE(100, handle, uintptr(unsafe.Pointer(&i)))
+	hwnd := C.HWND(unsafe.Pointer(uintptr(unsafe.Pointer(&i))))
+	recurseHWND(400, hwnd, uintptr(unsafe.Pointer(&i)))
+}
+
+func recurseHANDLE(n int, p C.HANDLE, v uintptr) {
+	if n > 0 {
+		recurseHANDLE(n-1, p, v)
+	}
+	if uintptr(unsafe.Pointer(p)) != v {
+		panic("adjusted notinheap pointer")
+	}
+}
+
+func recurseHWND(n int, p C.HWND, v uintptr) {
+	if n > 0 {
+		recurseHWND(n-1, p, v)
+	}
+	if uintptr(unsafe.Pointer(p)) != v {
+		panic("adjusted notinheap pointer")
+	}
+}

src/cmd/cgo/gcc.go

@@ -2106,6 +2106,9 @@ type typeConv struct {
 	// Type names X for which there exists an XGetTypeID function with type func() CFTypeID.
 	getTypeIDs map[string]bool
 
+	// badStructs contains C structs that should be marked NotInHeap.
+	notInHeapStructs map[string]bool
+
 	// Predeclared types.
 	bool                                   ast.Expr
 	byte                                   ast.Expr // denotes padding
@@ -2117,6 +2120,7 @@ type typeConv struct {
 	string                                 ast.Expr
 	goVoid                                 ast.Expr // _Ctype_void, denotes C's void
 	goVoidPtr                              ast.Expr // unsafe.Pointer or *byte
+	goVoidPtrNoHeap                        ast.Expr // *_Ctype_void_notinheap, like goVoidPtr but marked NotInHeap
 
 	ptrSize int64
 	intSize int64
@@ -2140,6 +2144,7 @@ func (c *typeConv) Init(ptrSize, intSize int64) {
 	c.m = make(map[string]*Type)
 	c.ptrs = make(map[string][]*Type)
 	c.getTypeIDs = make(map[string]bool)
+	c.notInHeapStructs = make(map[string]bool)
 	c.bool = c.Ident("bool")
 	c.byte = c.Ident("byte")
 	c.int8 = c.Ident("int8")
@@ -2158,6 +2163,7 @@ func (c *typeConv) Init(ptrSize, intSize int64) {
 	c.void = c.Ident("void")
 	c.string = c.Ident("string")
 	c.goVoid = c.Ident("_Ctype_void")
+	c.goVoidPtrNoHeap = c.Ident("*_Ctype_void_notinheap")
 
 	// Normally cgo translates void* to unsafe.Pointer,
 	// but for historical reasons -godefs uses *byte instead.
@@ -2538,6 +2544,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 				tt.C = &TypeRepr{"struct %s", []interface{}{tag}}
 			}
 			tt.Go = g
+			tt.NotInHeap = c.notInHeapStructs[tag]
 			typedef[name.Name] = &tt
 		}
 
@@ -2581,6 +2588,30 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 				oldType.BadPointer = true
 			}
 		}
+		if c.badVoidPointerTypedef(dt) {
+			// Treat this typedef as a pointer to a NotInHeap void.
+			s := *sub
+			s.Go = c.goVoidPtrNoHeap
+			sub = &s
+			// Make sure we update any previously computed type.
+			if oldType := typedef[name.Name]; oldType != nil {
+				oldType.Go = sub.Go
+			}
+		}
+		// Check for non-pointer "struct <tag>{...}; typedef struct <tag> *<name>"
+		// typedefs that should be marked NotInHeap.
+		if ptr, ok := dt.Type.(*dwarf.PtrType); ok {
+			if strct, ok := ptr.Type.(*dwarf.StructType); ok {
+				if c.badStructPointerTypedef(dt.Name, strct) {
+					c.notInHeapStructs[strct.StructName] = true
+					// Make sure we update any previously computed type.
+					name := "_Ctype_struct_" + strct.StructName
+					if oldType := typedef[name]; oldType != nil {
+						oldType.NotInHeap = true
+					}
+				}
+			}
+		}
 		t.Go = name
 		t.BadPointer = sub.BadPointer
 		t.NotInHeap = sub.NotInHeap
@@ -3132,6 +3163,48 @@ func (c *typeConv) badPointerTypedef(dt *dwarf.TypedefType) bool {
 	return false
 }
 
+// badVoidPointerTypedef is like badPointerTypeDef, but for "void *" typedefs that should be NotInHeap.
+func (c *typeConv) badVoidPointerTypedef(dt *dwarf.TypedefType) bool {
+	// Match the Windows HANDLE type (#42018).
+	if goos != "windows" || dt.Name != "HANDLE" {
+		return false
+	}
+	// Check that the typedef is "typedef void *<name>".
+	if ptr, ok := dt.Type.(*dwarf.PtrType); ok {
+		if _, ok := ptr.Type.(*dwarf.VoidType); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// badStructPointerTypedef is like badVoidPointerTypedefs but for structs.
+func (c *typeConv) badStructPointerTypedef(name string, dt *dwarf.StructType) bool {
+	// Windows handle types can all potentially contain non-pointers.
+	// badVoidPointerTypedef handles the "void *" HANDLE type, but other
+	// handles are defined as
+	//
+	// struct <name>__{int unused;}; typedef struct <name>__ *name;
+	//
+	// by the DECLARE_HANDLE macro in STRICT mode. The macro is declared in
+	// the Windows ntdef.h header,
+	//
+	// https://github.com/tpn/winsdk-10/blob/master/Include/10.0.16299.0/shared/ntdef.h#L779
+	if goos != "windows" {
+		return false
+	}
+	if len(dt.Field) != 1 {
+		return false
+	}
+	if dt.StructName != name+"__" {
+		return false
+	}
+	if f := dt.Field[0]; f.Name != "unused" || f.Type.Common().Name != "int" {
+		return false
+	}
+	return true
+}
+
 // baseBadPointerTypedef reports whether the base of a chain of typedefs is a bad typedef
 // as badPointerTypedef reports.
 func (c *typeConv) baseBadPointerTypedef(dt *dwarf.TypedefType) bool {

src/cmd/cgo/out.go

@@ -135,6 +135,7 @@ func (p *Package) writeDefs() {
 		fmt.Fprintf(fgo2, "%s", buf.Bytes())
 		fmt.Fprintf(fgo2, "\n\n")
 	}
+	fmt.Fprintf(fgo2, "//go:notinheap\ntype _Ctype_void_notinheap struct{}\n\n")
 	if *gccgo {
 		fmt.Fprintf(fgo2, "type _Ctype_void byte\n")
 	} else {