os: return nil slice when ReadDir is used with a file on file_windows

ReadDir should return (nil, ENOTDIR) when the path points to a file
instead of a directory. That's the behavior on Unix systems, and it also
used to be the behavior on Windows. However, Windows currently returns
([]DirEntry{}, ENOTDIR).

We should change the implementation to match the expected behavior.

Fixed #75157

Change-Id: I3a3ddb71b5cd6e51dbca435a1585f01116844d4a
Reviewed-on: https://go-review.googlesource.com/c/go/+/699375
Reviewed-by: Damien Neil <dneil@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: Steven Hartland <stevenmhartland@gmail.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

src/internal/syscall/windows/at_windows.go

@@ -11,6 +11,11 @@ import (
 	"unsafe"
 )
 
+// Openat flags supported by syscall.Open.
+const (
+	O_DIRECTORY = 0x04000 // target must be a directory
+)
+
 // Openat flags not supported by syscall.Open.
 //
 // These are invented values, use values in the 33-63 bit range
@@ -20,7 +25,6 @@ import (
 // the set of invented O_ values in syscall/types_windows.go
 // to avoid overlap.
 const (
-	O_DIRECTORY    = 0x100000000 // target must be a directory
 	O_NOFOLLOW_ANY = 0x200000000 // disallow symlinks anywhere in the path
 	O_OPEN_REPARSE = 0x400000000 // FILE_OPEN_REPARSE_POINT, used by Lstat
 	O_WRITE_ATTRS  = 0x800000000 // FILE_WRITE_ATTRIBUTES, used by Chmod

src/os/file_windows.go

@@ -125,7 +125,7 @@ func openFileNolog(name string, flag int, perm FileMode) (*File, error) {
 }
 
 func openDirNolog(name string) (*File, error) {
-	return openFileNolog(name, O_RDONLY, 0)
+	return openFileNolog(name, O_RDONLY|windows.O_DIRECTORY, 0)
 }
 
 func (file *file) close() error {

src/os/read_test.go

@@ -6,9 +6,11 @@ package os_test
 
 import (
 	"bytes"
+	"errors"
 	. "os"
 	"path/filepath"
 	"runtime"
+	"syscall"
 	"testing"
 )
 
@@ -103,11 +105,20 @@ func TestReadDir(t *testing.T) {
 	t.Parallel()
 
 	dirname := "rumpelstilzchen"
-	_, err := ReadDir(dirname)
-	if err == nil {
+	if _, err := ReadDir(dirname); err == nil {
 		t.Fatalf("ReadDir %s: error expected, none found", dirname)
 	}
 
+	filename := filepath.Join(t.TempDir(), "foo")
+	f, err := Create(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+	if list, err := ReadDir(filename); list != nil || !errors.Is(err, syscall.ENOTDIR) {
+		t.Fatalf("ReadDir %s: (nil, ENOTDIR) expected, got (%v, %v)", filename, list, err)
+	}
+
 	dirname = "."
 	list, err := ReadDir(dirname)
 	if err != nil {

src/syscall/syscall_windows.go

@@ -443,6 +443,18 @@ func Open(name string, flag int, perm uint32) (fd Handle, err error) {
 		}
 		return h, err
 	}
+	if flag&o_DIRECTORY != 0 {
+		// Check if the file is a directory, else return ENOTDIR.
+		var fi ByHandleFileInformation
+		if err := GetFileInformationByHandle(h, &fi); err != nil {
+			CloseHandle(h)
+			return InvalidHandle, err
+		}
+		if fi.FileAttributes&FILE_ATTRIBUTE_DIRECTORY == 0 {
+			CloseHandle(h)
+			return InvalidHandle, ENOTDIR
+		}
+	}
 	// Ignore O_TRUNC if the file has just been created.
 	if flag&O_TRUNC == O_TRUNC &&
 		(createmode == OPEN_EXISTING || (createmode == OPEN_ALWAYS && err == ERROR_ALREADY_EXISTS)) {

src/syscall/types_windows.go

@@ -47,8 +47,8 @@ const (
 	O_APPEND       = 0x00400
 	O_SYNC         = 0x01000
 	O_ASYNC        = 0x02000
+	o_DIRECTORY    = 0x04000
 	O_CLOEXEC      = 0x80000
-	o_DIRECTORY    = 0x100000000 // used by internal/syscall/windows
 	o_NOFOLLOW_ANY = 0x200000000 // used by internal/syscall/windows
 	o_OPEN_REPARSE = 0x400000000 // used by internal/syscall/windows
 	o_WRITE_ATTRS  = 0x800000000 // used by internal/syscall/windows