fix: preserve repository and object paths across platforms

The Windows Python 3.8 CI job reported 42 failures and 179 setup errors.
Most submodule failures shared a discovery bug: Git resolves relative gitfile
targets using forward slashes even when the caller supplies a native Windows
path. Normalize Git-facing discovery operands with the existing platform
helper, and account for native separators in Git's worktree registry output.

Use matching `surrogateescape` codecs for Git protocol paths so undecodable
tree names round-trip without Windows filesystem encoding changing their
bytes. Verify path/stage, mode, and object ID after materializing a private
index: `git update-index --index-info` can exit successfully while dropping
Windows-incompatible names. Raise `ValueError` before publishing such an
index and document the platform restriction in `changes.rst`.

Keep unusual names in object-only tests when the host cannot represent them
in a checkout. Use native-valid paths for worktree tests, assert rejection of
unsupported index names and quoted file references, and keep full quoted
reference coverage with reftable. Fix separator and LF assumptions in config,
URL, and packed-reference fixtures. Close test-owned repositories before
submodule removal and make the fake Windows Git executable discoverable
without shell execution. Also restore root paths for `Repo.tree()` results
resolved directly from tree IDs while preserving explicit subtree paths.

Validation: 116 repository tests and 10 focused submodule tests passed;
index/helper tests passed 93 with 2 platform skips; the focused format/safety
run passed 72; config/reference/remote tests passed 57 with 24 subtests;
36 refresh tests and 2 revision regressions passed. The Git 2.52 targeted
index matrix passed 19 tests. Ruff, mypy, basedpyright, Sphinx with warnings
as errors, and `git diff --check` pass. Native Windows validation awaits CI.

Author: codex

doc/source/changes.rst

@@ -63,6 +63,10 @@ API changes
   ``IndexFile.from_tree()``, ``IndexFile.new()``, ``write()`` and ``write_tree()``.
   Git preserves untouched metadata when an existing index is edited. Temporary
   indexes isolate tree/merge operations from the real index and working tree.
+  Git's platform-specific index filename restrictions apply. Unsupported entries
+  raise ``ValueError`` before the original index changes, including names with
+  colons or control characters on Windows. Tree objects can still contain names
+  that the working tree or index cannot represent.
   ``version`` is read-only. ``from_tree()`` accepts ``trivial``, ``aggressive``,
   and ``verbose`` options; arbitrary ``read-tree`` keyword forwarding is removed.
 * Standalone binary tree parsers, serializers, and multi-tree traversal helpers

git/index/base.py

@@ -21,7 +21,7 @@
 
 from gitdb.base import IStream
 
-from git.compat import defenc, force_bytes
+from git.compat import defenc, force_bytes, safe_decode
 from git.cmd import Git
 import git.diff as git_diff
 from git.exc import CheckoutError, GitCommandError, GitError, InvalidGitRepositoryError, UnmergedEntriesError
@@ -144,7 +144,7 @@ def _read_entries(self, path: PathLike) -> Dict[Tuple[PathLike, StageType], Inde
                 continue
             metadata, path_bytes = record.split(b"\t", 1)
             tag, mode, oid, stage = metadata.split()
-            entry_path = os.fsdecode(path_bytes)
+            entry_path = safe_decode(path_bytes)
             _validate_repo_path(entry_path)
             binsha = bytes.fromhex(oid.decode("ascii"))
             if len(binsha) != self.repo._oid_size or int(stage) not in range(4):
@@ -198,22 +198,28 @@ def _materialized_index(self) -> Generator[str, None, None]:
                 records = []
                 for name in sorted(changed):
                     _validate_repo_path(name)
-                    records.append(b"0 " + self.repo._null_hexsha.encode("ascii") + b"\t" + os.fsencode(name) + b"\0")
+                    name_bytes = name.encode(defenc, "surrogateescape")
+                    records.append(b"0 " + self.repo._null_hexsha.encode("ascii") + b"\t" + name_bytes + b"\0")
                     for stage in range(4):
                         desired_entry = desired.get((name, stage))
                         if desired_entry is not None:
                             records.append(
                                 ("%o %s %d\t" % (desired_entry.mode, desired_entry.hexsha, stage)).encode("ascii")
-                                + os.fsencode(name)
+                                + name_bytes
                                 + b"\0"
                             )
                 with tempfile.TemporaryFile() as stream:
                     stream.write(b"".join(records))
                     stream.seek(0)
                     self.repo.git._call_process_safe("update_index", "-z", "--index-info", istream=stream, env=env)
+                # Git can report success while ignoring names unsupported on this
+                # platform. Do not publish an index with silently missing entries.
+                actual = self._read_entries(path)
+                if actual.keys() != desired.keys() or any(actual[key][:2] != desired[key][:2] for key in actual):
+                    raise ValueError("Git did not retain the requested index entries; filenames may be unsupported")
                 for option, mask in (("--assume-unchanged", CE_VALID), ("--skip-worktree", CE_EXT_SKIP_WORKTREE << 16)):
                     names = [
-                        os.fsencode(name) + b"\0"
+                        name.encode(defenc, "surrogateescape") + b"\0"
                         for name in sorted(changed)
                         if (name, 0) in desired and desired[(name, 0)].flags & mask
                     ]
@@ -532,7 +538,7 @@ def _write_path_to_stdin(
 
         if proc.stdin is not None:
             try:
-                proc.stdin.write(os.fsencode(filepath) + b"\0")
+                proc.stdin.write(os.fspath(filepath).encode(defenc, "surrogateescape") + b"\0")
             except OSError as e:
                 # Pipe broke, usually because some error happened.
                 raise fmakeexc() from e

git/objects/tree.py

@@ -10,7 +10,7 @@
 
 import git.diff as git_diff
 from git.util import IterableList, join_path, to_bin_sha
-from git.compat import safe_decode
+from git.compat import defenc, safe_decode
 from git.cmd import Git
 
 from . import util
@@ -416,7 +416,7 @@ def _serialize(self, stream: "BytesIO") -> "Tree":
                 if len(oid) != self.repo._oid_size or mode >> 12 not in types:
                     raise ValueError("Invalid tree entry object ID or mode")
                 source.write(("%o %s %s\t" % (mode, types[mode >> 12], oid.hex())).encode("ascii"))
-                source.write(os.fsencode(name) + b"\0")
+                source.write(name.encode(defenc, "surrogateescape") + b"\0")
             source.seek(0)
             oid = self.repo.git._call_process_safe("mktree", "-z", "--missing", istream=source)
         Git._check_operand(oid, "tree object ID")

git/repo/base.py

@@ -43,6 +43,7 @@
     finalize_process,
     hex_to_bin,
     remove_password_if_present,
+    to_native_path_linux,
 )
 
 from .fun import (
@@ -326,7 +327,9 @@ def __init__(
             dotgit = osp.join(curpath, ".git")
             candidate = curpath if explicit_git_dir or not osp.lexists(dotgit) else dotgit
             try:
-                git_dir = probe._call_process_safe("rev_parse", "--resolve-git-dir", candidate)
+                # Git resolves relative gitfile targets using the last forward
+                # slash in this operand, including on Windows.
+                git_dir = probe._call_process_safe("rev_parse", "--resolve-git-dir", to_native_path_linux(candidate))
                 git_dir = osp.abspath(git_dir)
                 if osp.isfile(candidate):
                     # Git canonicalizes gitfile targets. Retain an equivalent
@@ -381,8 +384,14 @@ def __init__(
                 if not fields or not fields[0].startswith("worktree ") or "bare" in fields:
                     continue
                 worktree = fields[0][9:]
+                # Git only strips a forward-slash /.git suffix from its registry.
+                # A Windows gitdir file can instead contain a native backslash.
+                if osp.basename(worktree) == ".git" and osp.isfile(worktree):
+                    worktree = osp.dirname(worktree)
                 try:
-                    resolved = probe._call_process_safe("rev_parse", "--resolve-git-dir", osp.join(worktree, ".git"))
+                    resolved = probe._call_process_safe(
+                        "rev_parse", "--resolve-git-dir", to_native_path_linux(osp.join(worktree, ".git"))
+                    )
                 except GitCommandError:
                     continue
                 if osp.realpath(resolved) == osp.realpath(git_dir):
@@ -867,7 +876,10 @@ def tree(self, rev: Union[Tree_ish, str, None] = None) -> "Tree":
         if rev is None:
             return self.head.commit.tree
         obj = self.rev_parse(str(rev))
-        return obj if obj.type == "tree" else to_commit(obj).tree
+        if obj.type == "tree":
+            obj.path = getattr(obj, "path", "")
+            return obj
+        return to_commit(obj).tree
 
     def iter_commits(
         self,
@@ -1061,7 +1073,7 @@ def alternates(self) -> List[str]:
                 path = line[len(b"alternate: ") :]
                 if path.startswith(b'"') and path.endswith(b'"'):
                     path = _unquote_path(path[1:-1])
-                paths.append(os.fsdecode(path))
+                paths.append(safe_decode(path))
         return paths
 
     def is_dirty(
@@ -1132,7 +1144,7 @@ def _get_untracked_files(self, *args: Any, **kwargs: Any) -> List[str]:
         paths = []
         for record in records:
             if record.startswith(b"?? "):
-                paths.append(os.fsdecode(record[3:]))
+                paths.append(safe_decode(record[3:]))
             elif record[:1] in (b"R", b"C") or record[1:2] in (b"R", b"C"):
                 next(records, None)  # Renames/copies carry a second path record.
         return paths
@@ -1146,7 +1158,7 @@ def ignored(self, *paths: PathLike) -> List[str]:
             if "\0" in os.fspath(path):
                 raise ValueError("Paths cannot contain NUL")
         with tempfile.TemporaryFile() as stream:
-            stream.write(b"\0".join(os.fsencode(path) for path in paths) + b"\0")
+            stream.write(b"\0".join(os.fspath(path).encode(defenc, "surrogateescape") for path in paths) + b"\0")
             stream.seek(0)
             status, output, stderr = self.git._call_process_safe(
                 "check_ignore",
@@ -1161,7 +1173,7 @@ def ignored(self, *paths: PathLike) -> List[str]:
             return []
         if status:
             raise GitCommandError("git check-ignore", status, stderr, output)
-        return [os.fsdecode(path) for path in output.split(b"\0") if path]
+        return [safe_decode(path) for path in output.split(b"\0") if path]
 
     @property
     def active_branch(self) -> Head:

git/repo/fun.py

@@ -26,12 +26,13 @@
 from gitdb.exc import BadName, BadObject
 
 from git.cmd import Git
+from git.compat import defenc
 from git.exc import GitCommandError
 from git.objects import Object
 from git.objects.base import IndexObject
 from git.refs import SymbolicReference
 from git.types import AnyGitObject, Literal, PathLike
-from git.util import bin_to_hex, hex_to_bin
+from git.util import bin_to_hex, hex_to_bin, to_native_path_linux
 
 if TYPE_CHECKING:
     from gitdb.db import CompoundDB, LooseObjectDB
@@ -51,7 +52,7 @@ def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]:
     if not osp.exists(path):
         return None
     try:
-        return Git()._call_process_safe("rev_parse", "--resolve-git-dir", path)
+        return Git()._call_process_safe("rev_parse", "--resolve-git-dir", to_native_path_linux(path))
     except GitCommandError:
         return None
 
@@ -137,7 +138,7 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject:
         # Git resolves the mode, including index stages and executable/symlink
         # entries. No object storage or revision grammar is decoded in Python.
         with tempfile.TemporaryFile() as stream:
-            stream.write(os.fsencode(rev) + b"\0")
+            stream.write(rev.encode(defenc, "surrogateescape") + b"\0")
             stream.seek(0)
             mode = repo.git._call_process_safe("cat_file", "--batch-check=%(objectmode)", "-Z", istream=stream).strip(
                 "\0"

test/test_cli_objects_index.py

@@ -1,10 +1,13 @@
 """Repository-format independent object/index operations and their safety boundary."""
 
+import os
+from io import BytesIO
 from pathlib import Path
 
 import pytest
+from gitdb.base import IStream
 
-from git import Actor, Commit, IndexFile, Repo
+from git import Actor, Commit, IndexFile, Repo, Tree
 from git.exc import HookExecutionError, UnmergedEntriesError, UnsafeOptionError
 from git.index.typ import BaseIndexEntry, IndexEntry
 
@@ -17,7 +20,9 @@ def repo(request, tmp_path):
 
 def test_objects_index_commit_and_merge(repo):
     root = Path(repo.working_tree_dir)
-    names = ["space name", "line\nname", "tab\tname", "--option", "unicode-é", "dir/file"]
+    names = ["space name", "--option", "unicode-é", "dir/file"]
+    if os.name != "nt":
+        names.extend(["line\nname", "tab\tname"])
     for name in names:
         path = root / name
         path.parent.mkdir(exist_ok=True)
@@ -39,9 +44,10 @@ def test_objects_index_commit_and_merge(repo):
     updated = commit.replace(message="updated")
     assert repo.commit(updated.hexsha).message == "updated"
     assert repo.head.commit == commit
-    (root / "line\nname").write_text("modified")
-    index.checkout(["line\nname"], force=True)
-    assert (root / "line\nname").read_text() == "line\nname"
+    checkout_name = "--option" if os.name == "nt" else "line\nname"
+    (root / checkout_name).write_text("modified")
+    index.checkout([checkout_name], force=True)
+    assert (root / checkout_name).read_text() == checkout_name
     before = Path(index.path).read_bytes()
     virtual = IndexFile.from_tree(repo, tree)
     assert virtual.write_tree() == tree
@@ -53,6 +59,18 @@ def test_objects_index_commit_and_merge(repo):
     assert index.write_tree() == tree
 
 
+def test_tree_names_do_not_require_worktree_support(repo):
+    tree = Tree(repo, repo._null_binsha, path="")
+    tree._cache = [
+        (b"a" * repo._oid_size, 0o100644, name) for name in ("line\nname", "tab\tname", "name:with:colons", "\udc9f")
+    ]
+    data = BytesIO()
+    tree._serialize(data)
+    data.seek(0)
+    stored = repo.odb.store(IStream("tree", len(data.getvalue()), data))
+    assert Tree(repo, stored.binsha, path="")._cache == sorted(tree._cache, key=lambda entry: entry[2])
+
+
 def test_index_stages_missing_objects_and_atomic_failure(repo):
     index = repo.index
     missing = b"a" * repo._oid_size

test/test_config.py

@@ -3,6 +3,7 @@
 import configparser
 from collections import UserList
 from io import BytesIO
+from pathlib import Path
 from unittest import mock
 
 import pytest
@@ -46,7 +47,7 @@ def test_config_includes_and_repository_conditions(tmp_path):
         config.set_value("values", "source", "before")
         config.set_value("include", "path", str(included))
         config.add_value("values", "source", "after")
-        config.set_value('includeIf "gitdir:' + repo.git_dir + '"', "path", str(included))
+        config.set_value('includeIf "gitdir:' + Path(repo.git_dir).as_posix() + '"', "path", str(included))
     assert GitConfigParser(config_path, repo=repo).get_values("values", "source") == [
         "before",
         "after",

test/test_git.py

@@ -894,14 +894,9 @@ def test_successful_default_refresh_invalidates_cached_version_info(self):
             stack.enter_context(_patch_out_env("GIT_PYTHON_GIT_EXECUTABLE"))
 
             if sys.platform == "win32":
-                # On Windows, use a shell so "git" finds "git.cmd". The correct and safe
-                # ways to do this straightforwardly are to set GIT_PYTHON_GIT_EXECUTABLE
-                # to git.cmd in the environment, or call git.refresh with the command's
-                # full path. See the Git.USE_SHELL docstring for deprecation details.
-                # But this tests a "default" scenario where neither is done. The
-                # approach used here, setting USE_SHELL to True so PATHEXT is honored,
-                # should not be used in production code (nor even in most test cases).
-                stack.enter_context(mock.patch.object(Git, "USE_SHELL", True))
+                # The fake executable is a batch file. Name its extension explicitly
+                # so PATH lookup works without a shell, including version probes.
+                stack.enter_context(mock.patch.object(Git, "git_exec_name", "git.cmd"))
 
             new_git = Git()
             _rename_with_stem(path2, "git")  # "Install" git, "late" in the PATH.

test/test_index.py

@@ -211,14 +211,45 @@ def test_index_reader_and_writer_reject_unsafe_paths(self, path):
                 index.write()
 
     def test_valid_unusual_index_names_round_trip(self):
-        names = ["a b", "a\nb", "a\tb", "name:value", "dir/.gitignore", "café"]
+        names = ["a b", "--option", "dir/.gitignore", "café"]
+        windows_unsupported = ["a\nb", "a\tb", "name:value"]
         if os.name != "nt":
-            names.append("a\\b")
+            names.extend([*windows_unsupported, "a\\b", "\udc9f"])
         with tempfile.TemporaryDirectory() as directory:
             index = IndexFile(self.rorepo, Path(directory, "index"))
             index.entries = {(name, 0): IndexEntry((0o100644, b"a" * 20, 0, name)) for name in names}
             index.write()
             assert sorted(entry.path for entry in index.update().entries.values()) == sorted(names)
+            if os.name == "nt":
+                before = Path(index.path).read_bytes()
+                for name in windows_unsupported:
+                    index.entries[(name, 0)] = IndexEntry((0o100644, b"a" * 20, 0, name))
+                    with pytest.raises(ValueError, match="Git did not retain"):
+                        index.write()
+                    assert Path(index.path).read_bytes() == before
+                    assert not Path(str(index.path) + ".lock").exists()
+                    del index.entries[(name, 0)]
+
+    @ddt.data("write", "write_tree")
+    def test_index_rejects_silently_ignored_entries_atomically(self, operation):
+        call = Git._call_process_safe
+
+        def ignore_index_updates(git, command, *args, **kwargs):
+            if command == "update_index":
+                return ""
+            return call(git, command, *args, **kwargs)
+
+        with tempfile.TemporaryDirectory() as directory:
+            index = IndexFile(self.rorepo, Path(directory, "index"))
+            index.entries = {("before", 0): IndexEntry((0o100644, b"a" * 20, 0, "before"))}
+            index.write()
+            before = Path(index.path).read_bytes()
+            index.entries[("after", 0)] = IndexEntry((0o100644, b"a" * 20, 0, "after"))
+            with mock.patch.object(Git, "_call_process_safe", ignore_index_updates):
+                with pytest.raises(ValueError, match="Git did not retain"):
+                    getattr(index, operation)()
+            assert Path(index.path).read_bytes() == before
+            assert not Path(str(index.path) + ".lock").exists()
 
     def test_long_index_names_are_fully_validated(self):
         prefix = "a/" + "nested/" * 650

test/test_reflog.py

@@ -1,6 +1,8 @@
 # This module is part of GitPython and is released under the
 # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
 
+import sys
+
 import pytest
 
 from git import Actor, Repo
@@ -138,6 +140,12 @@ def test_checkout_and_reset_reject_interactive_patch_helpers(repo):
 
 
 def test_quoted_branch_configuration_survives_rename(repo):
+    if sys.platform == "win32" and repo.ref_format == "files":
+        # Loose references need filenames that Windows cannot represent.
+        with pytest.raises(OSError, match="Could not create reference") as error:
+            repo.create_head('quoted"branch')
+        assert isinstance(error.value.__cause__, GitCommandError)
+        return
     branch = repo.create_head('quoted"branch')
     with branch.config_writer() as config:
         config.set_value("description", "retained")