Skip to content

Commit af3d04e

Browse files
qmuntalqmuntal
committed
remove long path support and cherry-pick related bug fixes
1 parent 5564ee4 commit af3d04e

3 files changed

Lines changed: 747 additions & 0 deletions

‎patches/0013-remove-long-path-support-hack.patch‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2+
From: qmuntal <[email protected]>
3+
Date: Wed, 3 Apr 2024 12:38:51 +0200
4+
Subject: [PATCH] remove long path support hack
5+
6+
---
7+
src/runtime/os_windows.go | 61 ++-------------------------------------
8+
1 file changed, 2 insertions(+), 59 deletions(-)
9+
10+
diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go
11+
index 55a86986446d52..2c6aaa0ca12eb8 100644
12+
--- a/src/runtime/os_windows.go
13+
+++ b/src/runtime/os_windows.go
14+
@@ -133,8 +133,7 @@ var (
15+
// Load ntdll.dll manually during startup, otherwise Mingw
16+
// links wrong printf function to cgo executable (see issue
17+
// 12030 for details).
18+
- _RtlGetCurrentPeb stdFunction
19+
- _RtlGetVersion stdFunction
20+
+ _RtlGetVersion stdFunction
21+
22+
// These are from non-kernel32.dll, so we prefer to LoadLibraryEx them.
23+
_timeBeginPeriod,
24+
@@ -254,7 +253,6 @@ func loadOptionalSyscalls() {
25+
if n32 == 0 {
26+
throw("ntdll.dll not found")
27+
}
28+
- _RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000"))
29+
_RtlGetVersion = windowsFindfunc(n32, []byte("RtlGetVersion\000"))
30+
31+
m32 := windowsLoadSystemLib(winmmdll[:])
32+
@@ -427,10 +425,6 @@ func initHighResTimer() {
33+
//go:linkname canUseLongPaths os.canUseLongPaths
34+
var canUseLongPaths bool
35+
36+
-// We want this to be large enough to hold the contents of sysDirectory, *plus*
37+
-// a slash and another component that itself is greater than MAX_PATH.
38+
-var longFileName [(_MAX_PATH+1)*2 + 1]byte
39+
-
40+
// initLongPathSupport initializes the canUseLongPaths variable, which is
41+
// linked into os.canUseLongPaths for determining whether or not long paths
42+
// need to be fixed up. In the best case, this function is running on newer
43+
@@ -442,58 +436,7 @@ var longFileName [(_MAX_PATH+1)*2 + 1]byte
44+
// canUseLongPaths is set to true, and later when called, os.fixLongPath
45+
// returns early without doing work.
46+
func initLongPathSupport() {
47+
- const (
48+
- IsLongPathAwareProcess = 0x80
49+
- PebBitFieldOffset = 3
50+
- OPEN_EXISTING = 3
51+
- ERROR_PATH_NOT_FOUND = 3
52+
- )
53+
-
54+
- // Check that we're ≥ 10.0.15063.
55+
- info := _OSVERSIONINFOW{}
56+
- info.osVersionInfoSize = uint32(unsafe.Sizeof(info))
57+
- stdcall1(_RtlGetVersion, uintptr(unsafe.Pointer(&info)))
58+
- if info.majorVersion < 10 || (info.majorVersion == 10 && info.minorVersion == 0 && info.buildNumber < 15063) {
59+
- return
60+
- }
61+
-
62+
- // Set the IsLongPathAwareProcess flag of the PEB's bit field.
63+
- bitField := (*byte)(unsafe.Pointer(stdcall0(_RtlGetCurrentPeb) + PebBitFieldOffset))
64+
- originalBitField := *bitField
65+
- *bitField |= IsLongPathAwareProcess
66+
-
67+
- // Check that this actually has an effect, by constructing a large file
68+
- // path and seeing whether we get ERROR_PATH_NOT_FOUND, rather than
69+
- // some other error, which would indicate the path is too long, and
70+
- // hence long path support is not successful. This whole section is NOT
71+
- // strictly necessary, but is a nice validity check for the near to
72+
- // medium term, when this functionality is still relatively new in
73+
- // Windows.
74+
- targ := longFileName[len(longFileName)-33 : len(longFileName)-1]
75+
- if readRandom(targ) != len(targ) {
76+
- readTimeRandom(targ)
77+
- }
78+
- start := copy(longFileName[:], sysDirectory[:sysDirectoryLen])
79+
- const dig = "0123456789abcdef"
80+
- for i := 0; i < 32; i++ {
81+
- longFileName[start+i*2] = dig[longFileName[len(longFileName)-33+i]>>4]
82+
- longFileName[start+i*2+1] = dig[longFileName[len(longFileName)-33+i]&0xf]
83+
- }
84+
- start += 64
85+
- for i := start; i < len(longFileName)-1; i++ {
86+
- longFileName[i] = 'A'
87+
- }
88+
- stdcall7(_CreateFileA, uintptr(unsafe.Pointer(&longFileName[0])), 0, 0, 0, OPEN_EXISTING, 0, 0)
89+
- // The ERROR_PATH_NOT_FOUND error value is distinct from
90+
- // ERROR_FILE_NOT_FOUND or ERROR_INVALID_NAME, the latter of which we
91+
- // expect here due to the final component being too long.
92+
- if getlasterror() == ERROR_PATH_NOT_FOUND {
93+
- *bitField = originalBitField
94+
- println("runtime: warning: IsLongPathAwareProcess failed to enable long paths; proceeding in fixup mode")
95+
- return
96+
- }
97+
-
98+
- canUseLongPaths = true
99+
+ canUseLongPaths = false
100+
}
101+
102+
func osinit() {

‎patches/0014-os-support-UNC-paths-and-.-segments-in-fixLongPath.patch‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2+
From: qmuntal <[email protected]>
3+
Date: Tue, 12 Mar 2024 12:26:53 +0100
4+
Subject: [PATCH] os: support UNC paths and .. segments in fixLongPath
5+
MIME-Version: 1.0
6+
Content-Type: text/plain; charset=UTF-8
7+
Content-Transfer-Encoding: 8bit
8+
9+
This CL reimplements fixLongPath using syscall.GetFullPathName instead
10+
of a custom implementation that was not handling UNC paths and ..
11+
segments correctly. It also fixes a bug here multiple trailing \
12+
were removed instead of replaced by a single one.
13+
14+
The new implementation is slower than the previous one, as it does a
15+
syscall and needs to convert UTF-8 to UTF-16 (and back), but it is
16+
correct and should be fast enough for most use cases.
17+
18+
goos: windows
19+
goarch: amd64
20+
pkg: os
21+
cpu: Intel(R) Core(TM) i7-10850H CPU @ 2.70GHz
22+
│ old.txt │ new.txt │
23+
│ sec/op │ sec/op vs base │
24+
LongPath-12 1.007µ ± 53% 4.093µ ± 109% +306.41% (p=0.000 n=10)
25+
26+
│ old.txt │ new.txt │
27+
│ B/op │ B/op vs base │
28+
LongPath-12 576.0 ± 0% 1376.0 ± 0% +138.89% (p=0.000 n=10)
29+
30+
│ old.txt │ new.txt │
31+
│ allocs/op │ allocs/op vs base │
32+
LongPath-12 2.000 ± 0% 3.000 ± 0% +50.00% (p=0.000 n=10)
33+
34+
Fixes #41734.
35+
36+
Change-Id: Iced5cf47f56f6ab0ca74a6e2374c31a75100902d
37+
Reviewed-on: https://go-review.googlesource.com/c/go/+/570995
38+
Reviewed-by: Damien Neil <[email protected]>
39+
TryBot-Result: Gopher Robot <[email protected]>
40+
Reviewed-by: David Chase <[email protected]>
41+
LUCI-TryBot-Result: Go LUCI <[email protected]>
42+
(cherry picked from commit 2e8d84f148c69404b8eec86d9149785a3f4e3e92)
43+
---
44+
src/os/path_windows.go | 80 +++++++++++++++++--------------------
45+
src/os/path_windows_test.go | 29 ++++++++++----
46+
2 files changed, 57 insertions(+), 52 deletions(-)
47+
48+
diff --git a/src/os/path_windows.go b/src/os/path_windows.go
49+
index 052202514825ec..de654f1df9e4be 100644
50+
--- a/src/os/path_windows.go
51+
+++ b/src/os/path_windows.go
52+
@@ -4,6 +4,10 @@
53+
54+
package os
55+
56+
+import (
57+
+ "syscall"
58+
+)
59+
+
60+
const (
61+
PathSeparator = '\\' // OS-specific path separator
62+
PathListSeparator = ';' // OS-specific path list separator
63+
@@ -134,10 +138,8 @@ var canUseLongPaths bool
64+
65+
// fixLongPath returns the extended-length (\\?\-prefixed) form of
66+
// path when needed, in order to avoid the default 260 character file
67+
-// path limit imposed by Windows. If path is not easily converted to
68+
-// the extended-length form (for example, if path is a relative path
69+
-// or contains .. elements), or is short enough, fixLongPath returns
70+
-// path unmodified.
71+
+// path limit imposed by Windows. If the path is short enough or is relative,
72+
+// fixLongPath returns path unmodified.
73+
//
74+
// See https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation
75+
func fixLongPath(path string) string {
76+
@@ -161,19 +163,8 @@ func fixLongPath(path string) string {
77+
return path
78+
}
79+
80+
- // The extended form begins with \\?\, as in
81+
- // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
82+
- // The extended form disables evaluation of . and .. path
83+
- // elements and disables the interpretation of / as equivalent
84+
- // to \. The conversion here rewrites / to \ and elides
85+
- // . elements as well as trailing or duplicate separators. For
86+
- // simplicity it avoids the conversion entirely for relative
87+
- // paths or paths containing .. elements. For now,
88+
- // \\server\share paths are not converted to
89+
- // \\?\UNC\server\share paths because the rules for doing so
90+
- // are less well-specified.
91+
- if len(path) >= 2 && path[:2] == `\\` {
92+
- // Don't canonicalize UNC paths.
93+
+ if prefix := path[:4]; prefix == `\\.\` || prefix == `\\?\` || prefix == `\??\` {
94+
+ // Don't fix. Device path or extended path form.
95+
return path
96+
}
97+
if !isAbs(path) {
98+
@@ -181,36 +172,37 @@ func fixLongPath(path string) string {
99+
return path
100+
}
101+
102+
- const prefix = `\\?`
103+
+ var prefix []uint16
104+
+ var isUNC bool
105+
+ if path[:2] == `\\` {
106+
+ // UNC path, prepend the \\?\UNC\ prefix.
107+
+ prefix = []uint16{'\\', '\\', '?', '\\', 'U', 'N', 'C', '\\'}
108+
+ isUNC = true
109+
+ } else {
110+
+ prefix = []uint16{'\\', '\\', '?', '\\'}
111+
+ }
112+
113+
- pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
114+
- copy(pathbuf, prefix)
115+
- n := len(path)
116+
- r, w := 0, len(prefix)
117+
- for r < n {
118+
- switch {
119+
- case IsPathSeparator(path[r]):
120+
- // empty block
121+
- r++
122+
- case path[r] == '.' && (r+1 == n || IsPathSeparator(path[r+1])):
123+
- // /./
124+
- r++
125+
- case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || IsPathSeparator(path[r+2])):
126+
- // /../ is currently unhandled
127+
+ p, err := syscall.UTF16FromString(path)
128+
+ if err != nil {
129+
+ return path
130+
+ }
131+
+ n := uint32(len(p))
132+
+ var buf []uint16
133+
+ for {
134+
+ buf = make([]uint16, n+uint32(len(prefix)))
135+
+ n, err = syscall.GetFullPathName(&p[0], n, &buf[len(prefix)], nil)
136+
+ if err != nil {
137+
return path
138+
- default:
139+
- pathbuf[w] = '\\'
140+
- w++
141+
- for ; r < n && !IsPathSeparator(path[r]); r++ {
142+
- pathbuf[w] = path[r]
143+
- w++
144+
- }
145+
+ }
146+
+ if n <= uint32(len(buf)-len(prefix)) {
147+
+ buf = buf[:n+uint32(len(prefix))]
148+
+ break
149+
}
150+
}
151+
- // A drive's root directory needs a trailing \
152+
- if w == len(`\\?\c:`) {
153+
- pathbuf[w] = '\\'
154+
- w++
155+
+ if isUNC {
156+
+ // Remove leading \\.
157+
+ buf = buf[2:]
158+
}
159+
- return string(pathbuf[:w])
160+
+ copy(buf, prefix)
161+
+ return syscall.UTF16ToString(buf)
162+
}
163+
diff --git a/src/os/path_windows_test.go b/src/os/path_windows_test.go
164+
index 4e5e501d1f124c..1a18a53cae0212 100644
165+
--- a/src/os/path_windows_test.go
166+
+++ b/src/os/path_windows_test.go
167+
@@ -16,10 +16,14 @@ import (
168+
)
169+
170+
func TestFixLongPath(t *testing.T) {
171+
- if os.CanUseLongPaths {
172+
- return
173+
- }
174+
- t.Parallel()
175+
+ // Test fixLongPath even if long path are supported by the system,
176+
+ // else the function might not be tested at all when the test builders
177+
+ // support long paths.
178+
+ old := os.CanUseLongPaths
179+
+ os.CanUseLongPaths = false
180+
+ t.Cleanup(func() {
181+
+ os.CanUseLongPaths = old
182+
+ })
183+
184+
// 248 is long enough to trigger the longer-than-248 checks in
185+
// fixLongPath, but short enough not to make a path component
186+
@@ -38,19 +42,20 @@ func TestFixLongPath(t *testing.T) {
187+
// cases below where it doesn't.
188+
{`C:\long\foo.txt`, `\\?\C:\long\foo.txt`},
189+
{`C:/long/foo.txt`, `\\?\C:\long\foo.txt`},
190+
- {`C:\long\foo\\bar\.\baz\\`, `\\?\C:\long\foo\bar\baz`},
191+
- {`\\unc\path`, `\\unc\path`},
192+
+ {`C:\long\foo\\bar\.\baz\\`, `\\?\C:\long\foo\bar\baz\`},
193+
+ {`\\server\path\long`, `\\?\UNC\server\path\long`},
194+
{`long.txt`, `long.txt`},
195+
{`C:long.txt`, `C:long.txt`},
196+
- {`c:\long\..\bar\baz`, `c:\long\..\bar\baz`},
197+
+ {`c:\long\..\bar\baz`, `\\?\c:\bar\baz`},
198+
{`\\?\c:\long\foo.txt`, `\\?\c:\long\foo.txt`},
199+
{`\\?\c:\long/foo.txt`, `\\?\c:\long/foo.txt`},
200+
+ {`\??\c:\long/foo.txt`, `\??\c:\long/foo.txt`},
201+
} {
202+
in := strings.ReplaceAll(test.in, "long", veryLong)
203+
want := strings.ReplaceAll(test.want, "long", veryLong)
204+
if got := os.FixLongPath(in); got != want {
205+
got = strings.ReplaceAll(got, veryLong, "long")
206+
- t.Errorf("fixLongPath(%q) = %q; want %q", test.in, got, test.want)
207+
+ t.Errorf("fixLongPath(%#q) = %#q; want %#q", test.in, got, test.want)
208+
}
209+
}
210+
}
211+
@@ -155,3 +160,11 @@ func TestMkdirAllVolumeNameAtRoot(t *testing.T) {
212+
volName := syscall.UTF16ToString(buf[:])
213+
testMkdirAllAtRoot(t, volName)
214+
}
215+
+
216+
+func BenchmarkLongPath(b *testing.B) {
217+
+ veryLong := `C:\l` + strings.Repeat("o", 248) + "ng"
218+
+ b.ReportAllocs()
219+
+ for i := 0; i < b.N; i++ {
220+
+ os.FixLongPath(veryLong)
221+
+ }
222+
+}

0 commit comments

Comments
 (0)