From c2f288483c5e6693fe20945842b32635fa9f4551 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Thu, 15 Oct 2026 22:59:25 +0000 Subject: [PATCH 01/20] validate-family.py and family.py: a row's gitlink that no section of HEAD's `.gitmodules` gives is `member-gitmodules-absent`, so a holder whose HEAD lost the file no longer says `family ok`, `remove` refuses it before `deinit` instead of leaving the mount empty, and the gitlink remedies name a case variant and the commit `add` needs (#611) 571(b). In a holder whose HEAD lost `.gitmodules` (the commit #407 describes), `family.py remove --dry-run` exited 0, and the real run did `git submodule deinit` and `git rm`, then died at its commit's `git --literal-pathspecs add -f -- .gitmodules family.yaml` (`pathspec '.gitmodules' did not match any files`); the rollback left `members/

` empty. `_checked_gitmodules_held` now refuses that holder, before any git command that writes and under `--dry-run` alike: no `.gitmodules` in the work tree, the index or HEAD while the index records the member's gitlink. Refused, not committed without the file, as #407 refuses a `.gitmodules` the work tree lacks. The remedy writes back, with `git config -f .gitmodules`, the section `family.py add` writes for each member whose gitlink the index records, then `git add -f .gitmodules`; `remove` then runs as it always has. 571(c), #407's decision 3. `validate-family.py` reads HEAD's `.gitmodules` (`_head_gitmodules`, HEAD's root tree listed with no pathspec) and makes `member-gitmodules-absent` for each row at `members/` whose gitlink no section gives, in every mode, `--no-members` included. Its two lines write the section back as `family.py add` writes one, keeping the url of a section of that name that has one. A member it names that is not checked out is not refused `member-uninitialized`: that refusal named a `git submodule update` that exits 0 there and dropped the finding, which is what a fresh clone and CI met. It reads HEAD, not the work tree, so a deletion nobody committed, an unstaged edit and a sparse checkout that leaves the file out are no finding, and a holder with no members is asked nothing. 574/2 (#363's item 3). A row whose gitlink HEAD records only at a case variant of its path (`Members/

`) is refused `member-gitlink-absent` naming each variant, and the move that puts it back in the index alone: `git rm -r --cached` of the variants and `git update-index --add --cacheinfo 160000,,members/

`, then a commit. `family.py add`, the plain refusal's remedy, cannot help there: the row exists. 574/3. A hand mount's `member-gitlink-unnamed` remedy says to commit the `git rm` before the `add`: since #574, `add` refuses a path under another spelling of `members/` that HEAD holds and the index no longer does. `templates/family-root/scripts/validate-family.py` is a pinned copy: each holder sees it as `upstream-changed` on its next `update-shape.py check`. Tests: tests/test_family_head_lost_gitmodules.py (new, 24); the pinned remedy text and the case-variant states in tests/test_validate_family_unnamed_mounts.py; a docstring in tests/test_family_remove_gitmodules.py. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- scripts/family.py | 73 ++- .../family-root/scripts/validate-family.py | 214 ++++++- tests/test_family_head_lost_gitmodules.py | 596 ++++++++++++++++++ tests/test_family_remove_gitmodules.py | 12 +- tests/test_validate_family_unnamed_mounts.py | 34 +- 5 files changed, 902 insertions(+), 27 deletions(-) create mode 100644 tests/test_family_head_lost_gitmodules.py diff --git a/scripts/family.py b/scripts/family.py index a3eb0e08..85b2f87a 100755 --- a/scripts/family.py +++ b/scripts/family.py @@ -5209,7 +5209,8 @@ def _checked_gitmodules_as_staged(root: Path, verb: str) -> None: its line endings is refused here as `git rm` refuses it, and the refusal says so where `git add` would stage the index's own blob from it. With neither the index, HEAD nor the work tree holding a `.gitmodules`, - nothing is asked. + nothing is asked here; `remove` then asks `_checked_gitmodules_held` + (#611). `add` LEAVES A MISSING FILE TO GIT. `git submodule add` refuses to write a `.gitmodules` the work tree lacks while the index or HEAD holds one, @@ -5403,6 +5404,73 @@ def _checked_one_section(root: Path, project: str, path: str) -> None: f"{GITMODULES}` — and run the same command again.") +def _checked_gitmodules_held(family: Family, path: str) -> None: + """`member-gitmodules-absent`: `remove` refuses, before any git command + that writes and under `--dry-run` alike, a holder that holds no + `.gitmodules` at all, in the work tree, the index or HEAD, while its + index records the member's gitlink (#611). + + A HOLDER WHOSE HEAD LOST `.gitmodules`, as #407's commit left one. `git + rm` edits only a `.gitmodules` the work tree has, and `remove` commits + the file as `git rm` leaves it, so with none anywhere the commit's `git + add` died (`pathspec '.gitmodules' did not match any files`) after + `deinit` and `git rm` had emptied the member, and the rollback left its + directory empty; the dry run exited 0. `_checked_gitmodules_as_staged` + asks nothing here, since no copy exists to compare. + + REFUSED, NOT COMMITTED WITHOUT IT, as #407 refuses a `.gitmodules` the + work tree lacks: the holder is left byte for byte as it was. A clone + checks out none of its members, and the holder's own + `validate-family.py` finds `member-gitmodules-absent` for each, so the + remedy writes back, as `family.py add` writes one, the section of every + member whose gitlink the index records, this one included, which + `remove` then takes out as it always has. Staged, the file goes into + `remove`'s commit, as #407's untracked remedy has it, unless the person + commits it on its own first. + """ + root = family.root + if os.path.lexists(root / GITMODULES): + return + records = _index_records(root) + if GITMODULES.encode() in records or _head_oid(root, GITMODULES): + return + gitlinks = {name.decode("utf-8", "surrogateescape") + for name, held in records.items() + if any(record.startswith(b"160000 ") + and record.partition(b"\t")[0].endswith(b" 0") + for record in held)} + if path not in gitlinks: + return + sections = [] + for row in family.members: + project = str(row.get("project") or "") + member = f"{MEMBERS_DIR}/{project}" + if not project or member not in gitlinks: + continue + url = _member_url(str(row.get("repository") or ""), None) + sections += [f"`git config -f {GITMODULES} " + f"{shlex.quote(f'submodule.{member}.{key}')} " + f"{shlex.quote(value)}`" + for key, value in (("path", member), ("url", url))] + raise Refusal( + "member-gitmodules-absent", + f"the holder has no {GITMODULES}: neither the work tree, the index " + f"nor HEAD holds one, though the index records a gitlink at {path}. " + f"`git rm` edits only a {GITMODULES} the work tree has, and `remove` " + f"commits {GITMODULES} as `git rm` leaves it, so with none to commit " + f"it would fail after `git submodule deinit` and `git rm` had emptied " + f"{path}. A recursive clone of the holder checks out none of its " + "members, and its validate-family.py finds member-gitmodules-absent " + "for each. No git command that writes was run and nothing was " + "written.", + "Remediation: write back the section `family.py add` writes for " + "each member the index records, this one included, in a POSIX shell " + f"— {' and '.join(sections)} — then stage it, `git add -f " + f"{GITMODULES}` (`-f`, since an operator's excludes may match it), " + "and commit it on its own first if the command's commit should " + "carry only its own change, and run the same command again.") + + def _checked_index_unlocked(root: Path, path: str) -> None: """`family-index-locked`: the real `remove` refuses a held `index.lock` BEFORE it unmounts anything (#365). @@ -5591,6 +5659,9 @@ def cmd_remove(args) -> int: # carry a change nobody staged. Where the index holds a copy, every # reading of the work tree's file after this one reads that copy. _checked_gitmodules_as_staged(family.root, "remove") + # A holder that holds no `.gitmodules` anywhere, as one whose HEAD lost + # it does, is refused next (#611): the commit could not stage one. + _checked_gitmodules_held(family, path) # A `.gitmodules` that gives the path to a second submodule is refused # here, before any git command that writes (#360), so the dry run makes # the same refusal, as it makes every one above. diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 450bac3d..e4c2200c 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -40,7 +40,9 @@ project the row claims. A row whose `path:` is not `members/` is the `member-path` finding and none of the three is asked of it: nothing read through a - path the row may not have is a fact about the member (#367). + path the row may not have is a fact about the member (#367). A row + whose gitlink is recorded only at a case variant of its path is refused + naming that variant, and the index move that puts it back (#611). And, across the rows, that no two record ONE `id`: that is one project pinned twice, however each row spelled its repository (#206). This one reads `family.yaml` alone, so `--no-members` asks it too. @@ -57,6 +59,13 @@ asks them too. A path a row names in another case is still one of these, and what it says to do depends on which of the two is the member's own: it names no command that would un-register the member. + - that a section of HEAD's `.gitmodules` gives the path of every row's + gitlink (#611, #407's decision 3). A recursive clone reads HEAD's file, + so a gitlink no section of it gives is left empty there, by CI and by + `make bootstrap`, though the holder it was committed in keeps its + checkout. It needs no member checked out, so `--no-members` asks it + too, and a member it names that is not checked out is not then refused + as uninitialized. - the COPIED shape files, each against its `sha256` row in `contracts/shape-pin.yaml`. Editing a copy in place is drift and the exit is to carry it upstream, never to update the digest. @@ -381,7 +390,7 @@ def _pin_commit(pin: dict, project: str, report: Report) -> str: def _check_member_gitlink(root: Path, path: str, commit: str, project: str, - report: Report) -> None: + gitlinks: dict[str, str], report: Report) -> None: """The submodule gitlink this repository records at `path`, against the pin — THE LOCKSTEP RULE. Split from `_check_member` for #137. Raises `Refusal` when there is no gitlink at all: a row naming a member this @@ -389,9 +398,15 @@ def _check_member_gitlink(root: Path, path: str, commit: str, project: str, `git switch -c` that `add` needs first (#539) and the `add`, from a checkout of openRepoShape and naming this holder by its path, as the holder's other remedies do: `--family-root .` named the checkout the - line runs from.""" + line runs from. Where `gitlinks` (`_member_gitlinks`) holds one at a + case variant of `path`, the refusal names that instead (#611).""" gitlink = recorded_gitlink(root, path) if gitlink is None: + variants = sorted(other for other in gitlinks if other != path + and other.casefold() == path.casefold()) + if variants: + raise _gitlink_in_another_case(path, commit, project, variants, + gitlinks) raise Refusal( "member-gitlink-absent", f"no gitlink (mode 160000) recorded at {path!r}. {project} is " @@ -414,6 +429,45 @@ def _check_member_gitlink(root: Path, path: str, commit: str, project: str, report.note(f"{path}: gitlink == pin {commit[:12]}") +def _gitlink_in_another_case(path: str, commit: str, project: str, + variants: list[str], + gitlinks: dict[str, str]) -> Refusal: + """`member-gitlink-absent` for a row whose gitlink this repository + records only at a case variant of its path (#611, #363's item 3), as a + holder committed in #363's state does: `Members/

`, under a row and a + `.gitmodules` section that say `members/

`. A fresh clone checks out + nothing at the row's path, and the plain refusal's `family.py add` + cannot help: the row is there already. + + EACH VARIANT IS NAMED, AND THE GITLINK IS MOVED IN THE INDEX ALONE, as + #574's remedies act on an index entry under another spelling of + `members/` (Brett Heap, 2026-10-14, "Refuse on every disk"): `git rm -r + --cached` takes each variant out and touches no file, and `git + update-index --cacheinfo` records the gitlink at the row's path, at the + commit the row pins, reading no disk either. So the two lines work on + every disk, wherever the checkout stands, and the commit after them + records the member once, at its own path. `member-gitlink-unnamed` + prints no command for the same gitlink, since taking it out ALONE would + un-register the member; these take it out and record it together. + """ + found = ", ".join(f"{variant} (at {gitlinks[variant]})" + for variant in variants) + taken_out = " ".join(shlex.quote(variant) for variant in variants) + return Refusal( + "member-gitlink-absent", + f"no gitlink (mode 160000) recorded at {path!r}. {project} is named " + f"in {MANIFEST}, and this repository records its gitlink only at a " + "case variant of that path, which macOS and Windows hold as one " + f"directory with it: {found}. A recursive clone checks out nothing " + f"at {path}.", + "Remediation: the member is recorded in the wrong case. Move its " + "gitlink to the row's path, at the commit the row pins, in the index " + "alone, which touches no file, and commit that:\n" + f" git rm -r --cached -- {taken_out}\n" + " git update-index --add --cacheinfo " + f"{shlex.quote(f'160000,{commit},{path}')}") + + def _check_member_tree(member: Path, pin: dict, path: str, commit: str, project: str, report: Report) -> None: """Is `member`'s own object store readable at `commit`, and does its @@ -467,7 +521,8 @@ def _check_member_tree(member: Path, pin: dict, path: str, commit: str, report.note(f"{path}: tree digest recomputes ({recorded[:12]}…)") -def _check_member(root: Path, row: dict, seen: dict, report: Report) -> None: +def _check_member(root: Path, row: dict, seen: dict, report: Report, + gitlinks: dict[str, str], unregistered: set[str]) -> None: """One member: the row itself, the gitlink, the digest, the identity. NOTHING IS READ THROUGH A PATH THE ROW MAY NOT HAVE (#367). A hand-edited @@ -482,6 +537,14 @@ def _check_member(root: Path, row: dict, seen: dict, report: Report) -> None: The git each check runs is run inside `_git_runs`, which names the check when there is no git to run (#368); the identity check reads a file and runs none. + + A MEMBER NO SECTION OF HEAD'S `.gitmodules` GIVES (`unregistered`, the + `member-gitmodules-absent` finding, #611) that is not checked out stops + after its gitlink. Refused `member-uninitialized`, it named `git + submodule update --init`, which exits 0 there and checks nothing out, + and the refusal dropped the finding that says why: the state a fresh + clone of a holder whose HEAD lost `.gitmodules` is in, and CI's. The + finding is made all the same, so the run still fails. """ project = _member_project_name(row, seen, report) if project is None: @@ -500,10 +563,12 @@ def _check_member(root: Path, row: dict, seen: dict, report: Report) -> None: # ---- the gitlink ------------------------------------------------------- with _git_runs(f"the gitlink recorded at {path} could not be read"): - _check_member_gitlink(root, path, commit, project, report) + _check_member_gitlink(root, path, commit, project, gitlinks, report) # ---- the digest and the identity -------------------------------------- member = root / path + if path in unregistered and not (member / ".git").exists(): + return with _git_runs(f"{project}'s pinned commit and its tree digest could not " f"be read in {path}"): _check_member_tree(member, pin, path, commit, project, report) @@ -822,6 +887,13 @@ def _gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: "holder's submodules can be cloned from it as it stands. Mend " f"the file by hand (`git config -f {GITMODULES} --list` says " "where it breaks), then run this again.") from exc + return _sections_listed(listed) + + +def _sections_listed(listed: bytes) -> dict[str, dict[str, str]]: + """`{name: {key: value}}` of every `submodule` section in what `git + config -z --list` printed, each key in the order git lists it: one + reading for the work tree's `.gitmodules` and HEAD's (#611).""" sections: dict[str, dict[str, str]] = {} for record in listed.split(b"\0"): key, _, value = record.decode("utf-8", @@ -833,6 +905,48 @@ def _gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: return sections +def _head_gitmodules(root: Path + ) -> tuple[bool, dict[str, dict[str, str]]] | None: + """`(whether HEAD holds a .gitmodules, its sections)`, read as + `_gitmodules_sections` reads the work tree's, or None for a HEAD git + cannot list, a holder before its first commit, which a clone cannot + take either (#611). + + HEAD'S, NOT THE WORK TREE'S: a recursive clone of this holder, and CI's + checkout, read the file HEAD holds. A work tree's copy nobody has + committed is no fact about what a clone gets, and neither is one a + sparse checkout leaves out of the work tree or a deletion nobody has + committed. HEAD's ROOT TREE is listed and the entry picked by name, with + no pathspec, as `_member_gitlinks` lists HEAD. One git cannot read is + `gitmodules-unreadable`, as the work tree's is. + """ + with _git_runs(f"HEAD's {GITMODULES} could not be read as git reads it"): + try: + tree = git_out(["ls-tree", "-z", "HEAD"], cwd=root, binary=True) + except Refusal: + return None + blob = next((meta.split()[2].decode("ascii") + for meta, _, name in (record.partition(b"\t") + for record in tree.split(b"\0")) + if name == GITMODULES.encode() + and meta.split()[1:2] == [b"blob"]), None) + if blob is None: + return False, {} + try: + listed = git_out(["config", "-z", "--blob", blob, "--list"], + cwd=root, binary=True) + except Refusal as exc: + raise Refusal( + "gitmodules-unreadable", + f"HEAD's {GITMODULES} could not be read as git reads it: " + f"{exc.detail}", + "Remediation: git cannot read it either, so none of this " + "holder's submodules can be cloned from it. Mend the file by " + f"hand (`git config -f {GITMODULES} --list` says where it " + "breaks), commit it, then run this again.") from exc + return True, _sections_listed(listed) + + def _check_gitmodules_paths(sections: dict[str, dict[str, str]], report: Report) -> None: """No two sections of `.gitmodules` give ONE path (#300). @@ -1079,7 +1193,10 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], which refuses while its checkout holds changes, then mounted with `family.py add`: `add` cannot adopt an existing mount, and after `git rm --cached` alone it refuses the checkout left in the way - (`member-path-taken`). `add` then names the object store `git rm` leaves + (`member-path-taken`). The removal is committed between the two (#611): + `add` refuses a path under another spelling of `members/` that HEAD + holds and the index no longer does (#574), which an uncommitted `git rm + -- Members/

` leaves. `add` then names the object store `git rm` leaves in `.git/modules`, if it is still there. `--member` is quoted as the paths are, its project alone: `` is the operator's to fill in. A BRANCH FIRST (#539): `add` refuses the holder's tracking branch @@ -1119,10 +1236,13 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], "with the first command below, since family.py add refuses the " "holder's tracking branch, take it out with the second, which " f"refuses while its checkout holds changes and takes its " - f"{GITMODULES} section out too, then mount it with the third, " - "from a checkout of openRepoShape: family.py add writes its row " - "and its gitlink in one commit, and names the object store git " - "rm leaves under .git/modules if one is still there.\n" + f"{GITMODULES} section out too, and commit that removal, since " + "family.py add refuses a path under another spelling of members/ " + "that HEAD holds and the index no longer does, then mount it " + "with the third, from a checkout of openRepoShape: family.py add " + "writes its row and its gitlink in one commit, and names the " + "object store git rm leaves under .git/modules if one is still " + "there.\n" f" {_switch_line(root, 'add', project)}\n" f" git rm -- {shlex.quote(path)}\n" f" {PYTHON} scripts/family.py add --family-root " @@ -1201,6 +1321,73 @@ def _check_gitmodules_named(sections: dict[str, dict[str, str]], f"of them, then commit {GITMODULES}:{removal}") +def _check_gitlinks_registered(root: Path, members: list, + gitlinks: dict[str, str], + report: Report) -> set[str]: + """Every row's gitlink is at a path a section of HEAD's `.gitmodules` + gives — the `member-gitmodules-absent` finding (#611, #407's decision + 3). Returns the paths it was made for, of which `_check_member` asks no + checkout that is not there. + + A recursive clone checks a gitlink out only where a section gives its + path, so a gitlink with none is left empty in every clone, by CI and by + `make bootstrap` there, whose `git submodule update` exits 0 and checks + nothing out. The holder the commit was made in keeps its checkout, and + its git config the registration, so every other check here passed it: + `family.py remove` committed a `.gitmodules` the work tree had lost + (#407) and the gate said `family ok`. + + HEAD's file (`_head_gitmodules`), and the rows that `_check_member` + reads at their path: one whose `path:` is not `members/` is + `member-path`'s, and one with no gitlink there is + `member-gitlink-absent`'s (`gitlinks`, read as `recorded_gitlink` reads + one). A holder with no members holds no gitlink and is asked nothing. + + The remedy writes the section as `family.py add` writes it, named after + its path, with the url it gives a member, `https://github.com/` and the + row's `repository:`, or `` where that is not `/`. A + section of that name that has a url keeps it: only its path is written. + """ + head = _head_gitmodules(root) + if head is None: + return set() + held, sections = head + given = {keys["path"] for keys in sections.values() if "path" in keys} + unregistered: set[str] = set() + for row in members: + project = row.get("project") if isinstance(row, dict) else None + if not isinstance(project, str) or not project: + continue + path = f"{DEFAULT_MEMBERS_DIR}/{project}" + if (str(row.get("path") or path) != path or path in unregistered + or path not in gitlinks or path in given): + continue + unregistered.add(path) + repository = row.get("repository") + keys = {"path": shlex.quote(path)} + if "url" not in sections.get(path, {}): + keys["url"] = ( + shlex.quote(f"https://github.com/{repository}.git") + if isinstance(repository, str) + and QUALIFIED_RE.match(repository) else "") + lines = "".join( + f"\n git config -f {GITMODULES} " + f"{shlex.quote(f'submodule.{path}.{key}')} {value}" + for key, value in keys.items()) + report.finding( + "member-gitmodules-absent", + f"{path} is a gitlink at {gitlinks[path]} that a row of members: " + "names, and " + + (f"no section of HEAD's {GITMODULES} gives that path" if held + else f"HEAD holds no {GITMODULES}") + + ", so a recursive clone of this holder checks out nothing " + "there, and neither does make bootstrap in that clone, whatever " + "checkout this holder has of it. Write its section back by hand, " + "in a POSIX shell, as family.py add writes one, then stage and " + f"commit {GITMODULES}:{lines}") + return unregistered + + def _check_members(root: Path, members: list, no_members: bool, report: Report) -> None: """Every row in `members:`, or the `--no-members` skip line. @@ -1209,7 +1396,8 @@ def _check_members(root: Path, members: list, no_members: bool, out comes first, so `--no-members` asks them too: the ones that read a file alone, `family.yaml`'s names (#331) and ids (#206) and the paths `.gitmodules` gives twice (#300), then the gitlinks and sections no row - names, which read the index and HEAD as well (#361). + names, which read the index and HEAD as well (#361), and the gitlinks + no section of HEAD's `.gitmodules` gives (#611). """ _check_member_names_unique(members, report) _check_member_ids_unique(root, members, report) @@ -1219,6 +1407,8 @@ def _check_members(root: Path, members: list, no_members: bool, gitlinks = _member_gitlinks(root) _check_gitlinks_named(root, gitlinks, named, report) _check_gitmodules_named(sections, gitlinks, named, report) + unregistered = _check_gitlinks_registered(root, members, gitlinks, + report) if no_members: print(f" -- {len(members)} member(s) SKIPPED (--no-members): " "their gitlinks, digests and project ids are not checked " @@ -1230,7 +1420,7 @@ def _check_members(root: Path, members: list, no_members: bool, report.finding("member-row", f"a member is not a mapping: {row!r}") continue - _check_member(root, row, seen, report) + _check_member(root, row, seen, report, gitlinks, unregistered) if not members: report.note("no members yet: a family with none is empty, not " "wrong") diff --git a/tests/test_family_head_lost_gitmodules.py b/tests/test_family_head_lost_gitmodules.py new file mode 100644 index 00000000..159082b9 --- /dev/null +++ b/tests/test_family_head_lost_gitmodules.py @@ -0,0 +1,596 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A holder whose HEAD lost `.gitmodules`, and the gate's gitlink remedies +(#611). + +THE ISSUE'S FOUR LINES, each reproduced on main before this change: + +- 571(b). In a holder whose HEAD already lost `.gitmodules` (the commit #407 + describes), `family.py remove --dry-run` exited 0, and the real run did + `git submodule deinit` and `git rm`, then died at its commit's `git + --literal-pathspecs add -f -- .gitmodules family.yaml` (`pathspec + '.gitmodules' did not match any files`). Its rollback put HEAD and the + index back and left the member's directory EMPTY. `remove` now refuses + that holder before any git command that writes, `member-gitmodules-absent`, + under `--dry-run` alike, and its remedy writes back each member's section. +- 571(c). `validate-family.py` passed a `members:` row whose gitlink no + section of HEAD's `.gitmodules` gives (#407's decision 3): `family ok` in + the holder, `--no-members` too, while a fresh recursive clone left every + such member empty and its gate refused `member-uninitialized`, naming a `git + submodule update` that exits 0 and checks nothing out. It is the finding + `member-gitmodules-absent` now, in every mode, read from HEAD, and a member + it names that is not checked out is not refused as uninitialized. +- 574/2. `member-gitlink-absent` said nothing of a gitlink HEAD holds at a + case variant of the row's path (#363's item 3), and named `family.py add`, + which cannot help: the row is there. It names the variant now, and the + index move that records the member at its own path. +- 574/3. A hand mount's `member-gitlink-unnamed` remedy said `git rm`, then + `family.py add`. `add` refuses a path under another spelling of `members/` + that HEAD holds and the index no longer does (#574), so it says to commit + the removal between them now. + +THE SAME FAMILY, OFFLINE RULES AND READINGS as +`test_family_remove_gitmodules.py` (#407), whose helpers are imported. Every +remedy is typed as printed: split as a POSIX shell splits it, run with no +shell. The url a written-back section gives is the one `family.py add` +writes, `https://github.com//.git`, and a clone reaches the +fixture's local remotes through it by a `url..insteadOf` for the one +command, as `clone_just_pushed` reaches the legs. The case-variant states are +index records and commits (`stage_gitlink`), never two directories whose +names differ only in case, and each is asked again with +`core.ignorecase=true`, how macOS and Windows clone, so every one runs on all +three systems. +""" + +from __future__ import annotations + +import re +import shlex +from pathlib import Path + +import pytest + +from conftest import clone_just_pushed, run_script +from repo_shape import PYTHON +#: The family fixture, its names, the offline environment and the readings +#: of a holder, imported from the modules that own them rather than rebuilt. +from test_family import ( # noqa: F401 + FAMILY, NAME, ORG, SPARE, family, validate, +) +from test_family_add_excludes import empty_holder # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_dry_run import isolated +from test_family_member_path import MEMBER, OTHER, raw +from test_family_remove_gitmodules import ( + deleted, edited, head_paths, left_out, ran, refused_alike, + typed_as_printed, verb_args, +) +from test_scaffold_ignored_shape_files import excluding +from test_validate_family_unnamed_mounts import ( + commands, findings, spare_commits, stage_gitlink, +) + +GITMODULES = ".gitmodules" +MODES = pytest.mark.parametrize( + "extra", [(), ("--pins",), ("--no-members",)], + ids=["members-checked", "pins", "no-members"]) + +#: The two disks: git's own reading of a case-sensitive one, and how macOS +#: and Windows clone (`core.ignorecase=true`), set for every git command the +#: test types. +DISKS = pytest.mark.parametrize("ignorecase", ["false", "true"], + ids=["linux", "folding"]) + + +def on_disk(env: dict, ignorecase: str) -> dict: + """`env` with `core.ignorecase` set to `ignorecase`, one more + `GIT_CONFIG_*` entry after the ones it already holds.""" + index = int(env.get("GIT_CONFIG_COUNT", "0")) + return {**env, "GIT_CONFIG_COUNT": str(index + 1), + f"GIT_CONFIG_KEY_{index}": "core.ignorecase", + f"GIT_CONFIG_VALUE_{index}": ignorecase} + + +def url(project: str) -> str: + """The url `family.py add` writes for a member of the fixture's org.""" + return f"https://github.com/{ORG}/{project}.git" + + +#: What `remove` says in a holder that holds no `.gitmodules` anywhere. +REMOVE_REFUSED = ( + "REFUSED member-gitmodules-absent: the holder has no .gitmodules: neither " + "the work tree, the index nor HEAD holds one, though the index records a " + "gitlink at members/{member}. `git rm` edits only a .gitmodules the work " + "tree has, and `remove` commits .gitmodules as `git rm` leaves it, so " + "with none to commit it would fail after `git submodule deinit` and `git " + "rm` had emptied members/{member}. A recursive clone of the holder checks " + "out none of its members, and its validate-family.py finds " + "member-gitmodules-absent for each. No git command that writes was run " + "and nothing was written.") + + +def remove_remedy(*projects: str) -> str: + """`remove`'s remediation, writing back the section of each of + `projects`, in that order.""" + written = " and ".join( + f"`git config -f .gitmodules submodule.members/{project}.{key} " + f"{value}`" for project in projects + for key, value in (("path", f"members/{project}"), + ("url", url(project)))) + return ("Remediation: write back the section `family.py add` writes for " + "each member the index records, this one included, in a POSIX " + f"shell — {written} — then stage it, `git add -f .gitmodules` " + "(`-f`, since an operator's excludes may match it), and commit it " + "on its own first if the command's commit should carry only its " + "own change, and run the same command again.") + + +def absent(project: str, commit: str, *, held: bool, + with_url: bool = True) -> str: + """The `member-gitmodules-absent` finding for `project`'s gitlink at + `commit`, WHOLE: its line and the commands under it. `held` is whether + HEAD holds a `.gitmodules` at all; `with_url` whether the section to + write has no url of its own.""" + path = f"members/{project}" + lines = f"\n git config -f .gitmodules submodule.{path}.path {path}" + if with_url: + lines += (f"\n git config -f .gitmodules submodule.{path}.url " + f"{url(project)}") + return (f"FINDING member-gitmodules-absent: {path} is a gitlink at " + f"{commit} that a row of members: names, and " + + ("no section of HEAD's .gitmodules gives that path" if held + else "HEAD holds no .gitmodules") + + ", so a recursive clone of this holder checks out nothing " + "there, and neither does make bootstrap in that clone, whatever " + "checkout this holder has of it. Write its section back by hand, " + "in a POSIX shell, as family.py add writes one, then stage and " + f"commit .gitmodules:{lines}") + + +# --- the states -------------------------------------------------------------- + +def commit_all(root: Path, env: dict, message: str) -> None: + raw(root, env, "commit", "-q", "-m", message) + + +def head_lost(root: Path, env: dict) -> None: + """THE ISSUE'S STATE: `.gitmodules` deleted and the deletion committed, + as the commit #407 describes left it. The registrations stay in the + holder's git config, and its checkouts on disk.""" + raw(root, env, "rm", "-q", "--", GITMODULES) + commit_all(root, env, "The commit #407 describes") + + +def committed_gitmodules(root: Path, env: dict, data: bytes) -> None: + """`data` committed as `.gitmodules`, hashed into git's objects as + BYTES, and the same bytes on disk, so no platform rewrites a line + ending.""" + blob = raw(root, env, "hash-object", "-w", "--stdin", + stdin=data).stdout.decode("ascii").strip() + raw(root, env, "update-index", "--cacheinfo", + f"100644,{blob},{GITMODULES}") + commit_all(root, env, "A .gitmodules edited by hand") + (root / GITMODULES).write_bytes(data) + + +def without_section(root: Path, env: dict, project: str) -> None: + """HEAD's `.gitmodules` with `project`'s section taken out by hand and + committed, the other member's left as `family.py add` wrote it.""" + data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout + section = re.compile(rb'\[submodule "members/' + project.encode() + + rb'"\]\n(?:\t[^\n]*\n)*') + assert len(section.findall(data)) == 1, data + committed_gitmodules(root, env, section.sub(b"", data)) + + +def pinned(root: Path, project: str) -> str: + """The commit `family.yaml` pins `project` at, as git records its + gitlink after `family.py add`.""" + return raw(root, {}, "rev-parse", f"HEAD:members/{project}").stdout.decode( + "ascii").strip() + + +def typed(root: Path, env: dict, finding: str) -> None: + """Every command `finding` prints, typed as printed.""" + for command in commands(finding): + argv = shlex.split(command) + assert argv[0] == "git", command + raw(root, env, *argv[1:]) + + +def staged_and_committed(root: Path, env: dict, message: str) -> None: + """What a finding's "then stage and commit .gitmodules" says.""" + raw(root, env, "add", "--", GITMODULES) + commit_all(root, env, message) + + +def assert_passes_in_every_mode(root: Path, members: int) -> None: + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(root, *extra) + assert result.returncode == 0, (extra, result.stderr + result.stdout) + assert (f"family ok: {NAME} (inkrouter), {members} member(s)" + in result.stdout), extra + + +def github_to(remotes: Path) -> list[str]: + """The `-c` that makes a clone read `https://github.com//` from the + fixture's local remotes, for the one command.""" + uri = remotes.resolve().as_uri().rstrip("/") + "/" + return ["-c", f"url.{uri}.insteadOf=https://github.com/{ORG}/"] + + +def pushed_and_cloned(root: Path, env: dict, family, tmp_path: Path, + branch: str) -> Path: + """HEAD pushed to `branch` of the family's own remote, then a fresh + `clone --recurse-submodules` of it, which must exit 0.""" + remote = family["remotes"] / f"{NAME}.git" + raw(root, env, "push", "-q", str(remote), f"HEAD:refs/heads/{branch}") + clone = tmp_path / f"clone-{branch.replace('/', '-')}" + cloned = clone_just_pushed(remote, clone, branch, + config=github_to(family["remotes"])) + assert cloned.returncode == 0, cloned.stdout + cloned.stderr + return clone + + +# --- 571(b): remove in a holder whose HEAD lost .gitmodules ------------------ + +@pytest.mark.parametrize("rules", [b"", b"*\n"], ids=["no-excludes", "star"]) +def test_remove_refuses_a_holder_whose_head_lost_gitmodules( + family, holder, offline, tmp_path, rules): + """`member-gitmodules-absent`, by `remove` and its dry run alike, exit 2, + with the holder byte for byte as it was, MEMBER's checkout included: + the real run used to empty it and fail at its commit. Typed as printed, + under an operator whose excludes hold `*` too (hence the `-f`), the + remedy writes both members' sections back and stages them; `remove` + then commits MEMBER's removal with OTHER's section back in HEAD, and + the gate passes.""" + env = {**isolated(offline, tmp_path), **excluding(tmp_path / "op", rules)} + head_lost(holder, env) + args = verb_args("remove", holder, family) + + said, remediation = refused_alike(holder, env, args) + + assert said == REMOVE_REFUSED.format(member=MEMBER) + assert remediation == remove_remedy(MEMBER, OTHER) + assert (holder / "members" / MEMBER / "project.yaml").is_file() + assert typed_as_printed(holder, env, remediation)[-1] == ( + "git add -f .gitmodules") + ran(holder, env, args) + assert head_paths(holder, env) == [f"members/{OTHER}"] + assert_passes_in_every_mode(holder, 1) + + +def test_the_holder_remove_leaves_clones_with_the_other_member( + family, holder, offline, tmp_path): + """THE ISSUE'S CLONE. After the remedy and the `remove`, a push and a + fresh `clone --recurse-submodules` check OTHER out, from the url the + written-back section gives, and the clone's own gate passes, where + before the change a clone of this holder checked out no member at all.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + args = verb_args("remove", holder, family) + typed_as_printed(holder, env, refused_alike(holder, env, args)[1]) + ran(holder, env, args) + + clone = pushed_and_cloned(holder, env, family, tmp_path, "w611/removed") + + assert (clone / "members" / OTHER / "project.yaml").is_file() + assert not (clone / "members" / MEMBER).exists() + gate = validate(clone) + assert gate.returncode == 0, gate.stdout + gate.stderr + + +def test_remove_of_a_member_whose_section_alone_is_gone_runs_as_before( + family, holder, offline, tmp_path): + """The refusal is for a holder with NO `.gitmodules`, where the commit + cannot stage one. Where HEAD's file lacks only the member's own section, + `git rm` finds none to take out and `remove` commits the gitlink and the + row, as it did before: the holder it leaves passes the gate.""" + env = isolated(offline, tmp_path) + without_section(holder, env, MEMBER) + + removed = run_script(FAMILY, *verb_args("remove", holder, family), + env=env) + + assert removed.returncode == 0, removed.stdout + removed.stderr + assert head_paths(holder, env) == [f"members/{OTHER}"] + assert_passes_in_every_mode(holder, 1) + + +# --- 571(c): a gitlink no section of HEAD's .gitmodules gives --------------- + +@MODES +def test_a_head_that_lost_gitmodules_is_a_finding_per_member(holder, offline, + tmp_path, extra): + """THE ISSUE'S GATE: `family ok` before the change, in every mode. One + finding per member, each naming its gitlink and the two lines that write + its section back, in every mode, since none needs a member checked out. + Typed as printed, staged and committed, the holder passes again.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + + result = validate(holder, *extra) + + assert result.returncode == 1, result.stderr + result.stdout + assert "REFUSED" not in result.stderr, result.stderr + assert findings(result) == [ + absent(project, pinned(holder, project), held=False) + for project in (MEMBER, OTHER)] + for finding in findings(result): + typed(holder, env, finding) + staged_and_committed(holder, env, "Write the sections back as printed") + assert_passes_in_every_mode(holder, 2) + + +@MODES +def test_a_section_head_lost_alone_is_one_finding(holder, offline, tmp_path, + extra): + """One member's section taken out of HEAD's `.gitmodules` by hand: a + recursive clone checks that member out nowhere, though the file is + there. One finding, for it alone, and the other member is asked + nothing.""" + env = isolated(offline, tmp_path) + without_section(holder, env, OTHER) + + result = validate(holder, *extra) + + assert result.returncode == 1, result.stderr + result.stdout + (finding,) = findings(result) + assert finding == absent(OTHER, pinned(holder, OTHER), held=True) + typed(holder, env, finding) + staged_and_committed(holder, env, "Write the section back as printed") + assert_passes_in_every_mode(holder, 2) + + +def test_a_section_of_the_name_add_gives_keeps_its_url(holder, offline, + tmp_path): + """A section named `members/

`, as `family.py add` names one, whose + path is another case (`members/irss`): no section gives OTHER's path, + so a clone checks it out nowhere. The section's url is kept, and only + its path is written, which clears this finding and the section's own + `gitmodules-path-unnamed` beside it.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + old = f"\tpath = members/{OTHER}\n".encode() + assert data.count(old) == 1, data + committed_gitmodules(holder, env, data.replace( + old, f"\tpath = members/{OTHER.lower()}\n".encode())) + + result = validate(holder) + + assert result.returncode == 1, result.stderr + result.stdout + section, finding = findings(result) + assert section.startswith("FINDING gitmodules-path-unnamed: "), section + assert finding == absent(OTHER, pinned(holder, OTHER), held=True, + with_url=False) + typed(holder, env, finding) + staged_and_committed(holder, env, "Write the path back as printed") + assert_passes_in_every_mode(holder, 2) + + +def test_a_fresh_clone_of_it_says_the_finding_and_not_uninitialized( + family, holder, offline, tmp_path): + """BOOTSTRAP, VALIDATE AND A FRESH RECURSIVE CLONE AGREE. Pushed, the + holder whose HEAD lost `.gitmodules` clones with exit 0 and both + members empty; `make bootstrap`'s script there checks neither out, and + says so for each; and the clone's gate names both, members checked or + not, where before the change it refused `member-uninitialized`, whose + `git submodule update` exits 0 there. With the sections written back + as printed and pushed, a fresh clone checks both out and its gate + passes.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + clone = pushed_and_cloned(holder, env, family, tmp_path, "w611/lost") + expected = [absent(project, pinned(holder, project), held=False) + for project in (MEMBER, OTHER)] + + assert [path.name for path in (clone / "members").iterdir() + if any(path.iterdir())] == [] + booted = run_script(clone / "scripts" / "bootstrap.py", cwd=clone) + for project in (MEMBER, OTHER): + assert f" [{project}] {project}: NOT CHECKED OUT; skipped." in ( + booted.stdout), booted.stdout + for extra in ((), ("--no-members",)): + gate = validate(clone, *extra) + assert gate.returncode == 1, (extra, gate.stderr + gate.stdout) + assert "member-uninitialized" not in gate.stderr, gate.stderr + assert findings(gate) == expected, extra + + for finding in expected: + typed(holder, env, finding) + staged_and_committed(holder, env, "Write the sections back as printed") + mended = pushed_and_cloned(holder, env, family, tmp_path, "w611/mended") + for project in (MEMBER, OTHER): + assert (mended / "members" / project / "project.yaml").is_file() + assert_passes_in_every_mode(mended, 2) + + +#: `(set-up, what it is)`: the work tree's `.gitmodules` is not HEAD's, and +#: HEAD's gives every member's path, so a clone checks each out. +NOT_HEADS = { + "deleted, not committed": deleted, + "edited, not staged": edited, + "left out by a sparse checkout": left_out, +} + + +@pytest.mark.parametrize("set_up", NOT_HEADS.values(), ids=NOT_HEADS) +def test_a_work_tree_copy_that_is_not_heads_is_no_finding(holder, offline, + tmp_path, set_up): + """No misfire: the finding reads HEAD's file, which a clone reads, and + never the work tree's. A deletion nobody committed, an edit nobody + staged and a sparse checkout that leaves the file out are each no fact + about what a clone gets.""" + env = isolated(offline, tmp_path) + set_up(holder, env) + + for extra in ((), ("--no-members",)): + result = validate(holder, *extra) + assert result.returncode == 0, (extra, result.stderr + result.stdout) + + +def test_a_holder_with_no_members_is_asked_nothing(empty_holder): + """No misfire: a holder `init` made holds no `.gitmodules` and no + gitlink, and passes in every mode.""" + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(empty_holder, *extra) + assert result.returncode == 0, (extra, result.stderr + result.stdout) + + +def test_an_untracked_copy_put_back_is_still_a_finding_until_committed( + holder, offline, tmp_path): + """A `.gitmodules` put back by hand, untracked: the holder's own + checkouts still work, but HEAD, which a clone reads, holds none, so the + finding stands until the file is committed, and the holder then + passes.""" + env = isolated(offline, tmp_path) + kept = (holder / GITMODULES).read_bytes() + head_lost(holder, env) + (holder / GITMODULES).write_bytes(kept) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stderr + result.stdout + assert [finding.split(":", 1)[0] for finding in findings(result)] == [ + "FINDING member-gitmodules-absent"] * 2 + staged_and_committed(holder, env, "Put .gitmodules back") + assert_passes_in_every_mode(holder, 2) + + +# --- 574/2: a gitlink recorded only at a case variant of the row's path ----- + +def variant_refused(path: str, commit: str, variants: list[str], + gitlinks: dict[str, str]) -> list[str]: + """`member-gitlink-absent`'s lines for a row whose gitlink is recorded + only at `variants`.""" + project = path.split("/", 1)[1] + found = ", ".join(f"{variant} (at {gitlinks[variant]})" + for variant in variants) + return [ + f"REFUSED member-gitlink-absent: no gitlink (mode 160000) recorded at " + f"'{path}'. {project} is named in family.yaml, and this repository " + "records its gitlink only at a case variant of that path, which " + f"macOS and Windows hold as one directory with it: {found}. A " + f"recursive clone checks out nothing at {path}.", + "Remediation: the member is recorded in the wrong case. Move its " + "gitlink to the row's path, at the commit the row pins, in the index " + "alone, which touches no file, and commit that:", + " git rm -r --cached -- " + " ".join(variants), + f" git update-index --add --cacheinfo 160000,{commit},{path}"] + + +def recorded_at(root: Path, env: dict, project: str, + *spellings: str) -> str: + """`project`'s gitlink taken out of the index at its own path and + recorded at each of `spellings`, then committed: index records alone, + so no disk holds two names and the checkout does not move. Returns the + commit it names.""" + commit = pinned(root, project) + raw(root, env, "rm", "-q", "--cached", "--", f"members/{project}") + for spelling in spellings: + stage_gitlink(root, spelling, commit) + commit_all(root, env, f"{project} recorded at {', '.join(spellings)}") + return commit + + +@DISKS +@pytest.mark.parametrize("spellings", [ + ("Members/{p}",), ("MEMBERS/{p}", "Members/{p}")], ids=["one", "two"]) +def test_a_gitlink_only_at_a_case_variant_is_named_and_moved_as_printed( + holder, offline, tmp_path, ignorecase, spellings): + """#363's item 3: HEAD records MEMBER's gitlink at `Members/IRRS` (and, + in the second case, at `MEMBERS/IRRS` too) while its row and section say + `members/IRRS`. The refusal names each variant with its commit and, + since `family.py add` cannot help, prints the move: the variants out of + the index, the gitlink in at the row's path at the pinned commit. Typed + as printed and committed, on either disk, HEAD records the member once, + at its own path, and the holder passes in every mode.""" + env = on_disk(isolated(offline, tmp_path), ignorecase) + variants = sorted(spelling.format(p=MEMBER) for spelling in spellings) + commit = recorded_at(holder, env, MEMBER, *variants) + + result = validate(holder) + + assert result.returncode == 2, result.stderr + result.stdout + lines = result.stderr.splitlines() + path = f"members/{MEMBER}" + assert lines == variant_refused(path, commit, variants, + dict.fromkeys(variants, commit)) + for line in lines[2:]: + raw(holder, env, *shlex.split(line)[1:]) + commit_all(holder, env, "Move the gitlink as printed") + listed = raw(holder, env, "ls-tree", "-r", "--name-only", "HEAD").stdout + assert f"{path}\n".encode() in listed + for variant in variants: + assert f"{variant}\n".encode() not in listed + assert_passes_in_every_mode(holder, 2) + + +def test_a_fresh_clone_of_a_case_variant_says_it_too(family, holder, offline, + tmp_path): + """The same state pushed: a fresh clone names the variant, where the + refusal named `family.py add` and nothing of `Members/`. MEMBER is the + first row, so its gitlink is asked before any member's checkout.""" + env = isolated(offline, tmp_path) + commit = recorded_at(holder, env, MEMBER, f"Members/{MEMBER}") + remote = family["remotes"] / f"{NAME}.git" + raw(holder, env, "push", "-q", str(remote), "HEAD:refs/heads/w611/case") + clone = tmp_path / "clone" + cloned = clone_just_pushed(remote, clone, "w611/case", recurse=False) + assert cloned.returncode == 0, cloned.stdout + cloned.stderr + + result = validate(clone) + + assert result.returncode == 2, result.stderr + result.stdout + variant = f"Members/{MEMBER}" + assert result.stderr.splitlines() == variant_refused( + f"members/{MEMBER}", commit, [variant], {variant: commit}) + + +# --- 574/3: a hand mount's remedy commits the removal before the add --------- + +@DISKS +def test_a_hand_mount_under_members_spelt_another_way_is_added_as_printed( + family, holder, offline, tmp_path, ignorecase): + """A gitlink at `Members/IRQS` that no row names, with its section, as a + hand mount leaves one (index records, so no disk holds `Members/`). + Typed as printed, the branch, the `git rm`, the commit the text says + the removal needs, and the `add` with the org filled in mount IRQS at + `members/IRQS` with its row, and the holder passes in every mode. With + the removal left uncommitted, as the text said before, `add` refuses: + HEAD still holds `Members/IRQS`.""" + env = on_disk(isolated(offline, tmp_path), ignorecase) + raw(holder, env, "switch", "-q", "-C", "main") + _, tip = spare_commits(family) + hand = f"Members/{SPARE}" + stage_gitlink(holder, hand, tip) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + ( + f'[submodule "{hand}"]\n\tpath = {hand}\n\turl = {url(SPARE)}\n' + .encode()) + committed_gitmodules(holder, env, data) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stderr + result.stdout + gitlink = findings(result)[0] + assert gitlink.startswith(f"FINDING member-gitlink-unnamed: {hand} ") + assert (", and commit that removal, since family.py add refuses a path " + "under another spelling of members/ that HEAD holds and the " + "index no longer does, then mount it with the third, ") in gitlink + switch, rm, add = commands(gitlink) + raw(holder.parent, env, *shlex.split(switch)[1:]) + raw(holder, env, *shlex.split(rm)[1:]) + argv = [arg.replace("", ORG) for arg in shlex.split(add)] + assert argv[:3] == [PYTHON, "scripts/family.py", "add"], add + + def add_as_printed(): + return run_script(FAMILY, *argv[2:], "--local-remote-dir", + str(family["remotes"]), env=env) + + uncommitted = add_as_printed() + assert uncommitted.returncode == 2, uncommitted.stdout + uncommitted.stderr + assert "in HEAD and no longer in the index" in uncommitted.stderr + commit_all(holder, env, "Take the hand mount out, as printed") + added = add_as_printed() + assert added.returncode == 0, added.stdout + added.stderr + assert (holder / "members" / SPARE / "project.yaml").is_file() + assert_passes_in_every_mode(holder, 3) diff --git a/tests/test_family_remove_gitmodules.py b/tests/test_family_remove_gitmodules.py index cc7594f8..282a6b82 100644 --- a/tests/test_family_remove_gitmodules.py +++ b/tests/test_family_remove_gitmodules.py @@ -651,12 +651,12 @@ def test_a_holder_whose_head_has_no_gitmodules_is_not_asked( """Where none of the index, HEAD and the work tree holds a `.gitmodules` there is no copy to compare, and no `gitmodules-` refusal is made. - THE REAL RUN STILL FAILS HERE, a known case from before #407 that goes - with the gate's to #459: in a holder whose HEAD already lost the file, - `remove` runs `deinit` and `git rm`, then dies at its commit's `git add - -f -- .gitmodules family.yaml` and puts back only part of the unmount, - while its dry run exits 0. So this pins only that #407's check asks - nothing, and not the dry run's answer (the review's N3).""" + A holder whose HEAD already lost the file is #611's: `remove` and its + dry run refuse it `member-gitmodules-absent`, before any git command + that writes (`tests/test_family_head_lost_gitmodules.py`), where the + real run used to die at its commit's `git add -f -- .gitmodules + family.yaml` with the member emptied. So this pins only that #407's + check asks nothing (the review's N3).""" env = isolated(offline, tmp_path) deletion_staged(holder, env) raw(holder, env, "commit", "-q", "-m", "The commit #407 describes") diff --git a/tests/test_validate_family_unnamed_mounts.py b/tests/test_validate_family_unnamed_mounts.py index bcce579c..7c4babf5 100644 --- a/tests/test_validate_family_unnamed_mounts.py +++ b/tests/test_validate_family_unnamed_mounts.py @@ -88,9 +88,11 @@ "was mounted by hand, make a branch with the first command below, since " "family.py add refuses the holder's tracking branch, take it out with " "the second, which refuses while its checkout holds changes and takes " - "its .gitmodules section out too, then mount it with the third, from a " - "checkout of openRepoShape: family.py add writes its row and its gitlink " - "in one commit, and names the object store git rm leaves under " + "its .gitmodules section out too, and commit that removal, since " + "family.py add refuses a path under another spelling of members/ that " + "HEAD holds and the index no longer does, then mount it with the third, " + "from a checkout of openRepoShape: family.py add writes its row and its " + "gitlink in one commit, and names the object store git rm leaves under " ".git/modules if one is still there.") @@ -523,17 +525,24 @@ def test_a_member_recorded_only_in_another_case_says_so_and_prints_no_command( none and is the refusal `member-gitlink-absent`, which prints no finding, so only `--no-members` shows the finding; a git that matched the pathspec in any case would find the variant and print it, so either is accepted - there, and what no run may do is print a command.""" + there. What no run may do is print a command that takes the gitlink out + ALONE: since #611 the refusal names the variant and prints the move that + takes it out of the index and records it at the row's path together + (`tests/test_family_head_lost_gitmodules.py` types it).""" named = record_in_another_case(holder, "IRSS", "members/irss") result = validate(holder, *extra) - assert "git rm" not in result.stderr + result.stdout assert "--unset-all" not in result.stderr + result.stdout if result.returncode == 2: assert extra != ("--no-members",), result.stderr assert result.stderr.startswith( "REFUSED member-gitlink-absent: no gitlink (mode 160000) " "recorded at 'members/IRSS'."), result.stderr + assert result.stderr.splitlines()[-2:] == [ + " git rm -r --cached -- members/irss", + " git update-index --add --cacheinfo " + f"160000,{named},members/IRSS"], result.stderr return + assert "git rm" not in result.stderr + result.stdout assert result.returncode == 1, result.stderr + result.stdout (gitlink,) = findings(result) assert gitlink == GITLINK_SAID.format( @@ -546,13 +555,20 @@ def test_a_sections_only_path_in_another_case_says_so_and_prints_no_command( holder, extra): """The section's own state (ii): IRSS's row is canonical, its gitlink is where it belongs, and the ONLY section that registers it gives - `members/irss`. `--unset-all` of that left the member with no section.""" + `members/irss`. `--unset-all` of that left the member with no section. + Since #611 no section of HEAD's file giving `members/IRSS` is + `member-gitmodules-absent` beside it, whose one line writes the path + back into that section and keeps its url.""" url = url_of(holder, "IRSS") with_path_replaced(holder, b"\tpath = members/IRSS\n", b"\tpath = members/irss\n") result = validate(holder, *extra) assert result.returncode == 1, result.stderr + result.stdout - (section,) = findings(result) + section, unregistered = findings(result) + assert unregistered.startswith( + "FINDING member-gitmodules-absent: members/IRSS is a gitlink at ") + assert commands(unregistered) == [ + "git config -f .gitmodules submodule.members/IRSS.path members/IRSS"] assert section == SECTION_SAID.format( path="members/irss", name="members/IRSS", url=f"url {url}" ) + WRONG_CASE.format(row_path="members/IRSS", project="IRSS", @@ -685,7 +701,8 @@ def test_a_hand_mount_is_taken_out_and_added_back_as_printed(family, holder): """ROW (a) FOR REAL, and the gitlink's remedy run as printed, from the holder's TRACKING BRANCH, where `init` leaves it and `add` refuses to commit (#487): the first line makes the branch `add` suggests (#539). - `git rm` takes the checkout and its section out; `family.py add`, run + `git rm` takes the checkout and its section out, and the removal is + committed, as the text says since #611; `family.py add`, run as printed with the org filled in, then refuses `member-git-dir-cached` for the object store `git rm` leaves, as the remedy says it will name; with that gone, the same `add` mounts and pins the member with its row @@ -707,6 +724,7 @@ def test_a_hand_mount_is_taken_out_and_added_back_as_printed(family, holder): assert commands(findings(result)[1]) == [] git(*shlex.split(switch)[1:], cwd=holder.parent) git(*shlex.split(rm)[1:], cwd=holder) + commit(holder, "Take the hand mount out, as printed") assert not (holder / "members" / SPARE).exists() assert f"members/{SPARE}" not in (holder / ".gitmodules").read_text( encoding="utf-8") From 6f938ed1a385890cdb4c001e85b304c2618f638d Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Thu, 15 Oct 2026 23:11:21 +0000 Subject: [PATCH 02/20] tests/test_family_head_lost_gitmodules.py: the rows `member-gitmodules-absent` reads and the members `remove`'s refusal is for are pinned, so a finding for a row with no gitlink or a hand-edited path, or the refusal over a member with no gitlink, no longer passes (#611) Two tests. In a holder whose HEAD lost `.gitmodules`: a row with no gitlink (`member-gitlink-absent`'s) and a row whose `path:` was edited by hand (`member-path`'s, its mount `member-gitlink-unnamed`'s) get no `member-gitmodules-absent`; and `remove` of a member with no gitlink is not that refusal, but the dry run's `did not match any file(s) known to git`, as before. Each holds a condition of the first commit that its other tests did not: both pass on BASE 396c0ff4 and on that commit. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- tests/test_family_head_lost_gitmodules.py | 43 +++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_family_head_lost_gitmodules.py b/tests/test_family_head_lost_gitmodules.py index 159082b9..62337ef1 100644 --- a/tests/test_family_head_lost_gitmodules.py +++ b/tests/test_family_head_lost_gitmodules.py @@ -279,6 +279,25 @@ def test_the_holder_remove_leaves_clones_with_the_other_member( assert gate.returncode == 0, gate.stdout + gate.stderr +def test_remove_of_a_member_with_no_gitlink_is_not_this_refusal( + family, holder, offline, tmp_path): + """The refusal is for a member whose gitlink the index records: that is + the unmount the commit's failure emptied. With none, the run fails at + its first git call, which writes nothing, and the dry run says so by + git's own words, as before; the remedy of this refusal could not help.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + raw(holder, env, "rm", "-q", "--cached", "--", f"members/{MEMBER}") + commit_all(holder, env, "MEMBER's gitlink taken out") + + dry = run_script(FAMILY, *verb_args("remove", holder, family), + "--dry-run", env=env) + + assert dry.returncode == 2, dry.stdout + dry.stderr + assert "member-gitmodules-absent" not in dry.stderr, dry.stderr + assert "did not match any file(s) known to git" in dry.stderr + + def test_remove_of_a_member_whose_section_alone_is_gone_runs_as_before( family, holder, offline, tmp_path): """The refusal is for a holder with NO `.gitmodules`, where the commit @@ -436,6 +455,30 @@ def test_a_holder_with_no_members_is_asked_nothing(empty_holder): assert result.returncode == 0, (extra, result.stderr + result.stdout) +def test_a_row_the_gitlink_checks_do_not_read_is_asked_nothing( + holder, offline, tmp_path): + """The rows `_check_member` reads at their path, and no other, in a + holder whose HEAD lost `.gitmodules`. OTHER's gitlink taken out too: a + row with no gitlink is `member-gitlink-absent`'s, members checked. + MEMBER's row given another `path:` by hand: that row is `member-path`'s, + and its mount `member-gitlink-unnamed`'s.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + raw(holder, env, "rm", "-q", "--cached", "--", f"members/{OTHER}") + commit_all(holder, env, "OTHER's gitlink taken out") + rows = (holder / "family.yaml").read_bytes() + line = f" path: members/{MEMBER}\n".encode() + assert rows.count(line) == 1, "fixture drift: MEMBER's path line" + (holder / "family.yaml").write_bytes(rows.replace( + line, f" path: members/{MEMBER}-moved\n".encode())) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stderr + result.stdout + assert "member-gitlink-unnamed" in result.stderr, result.stderr + assert "member-gitmodules-absent" not in result.stderr, result.stderr + + def test_an_untracked_copy_put_back_is_still_a_finding_until_committed( holder, offline, tmp_path): """A `.gitmodules` put back by hand, untracked: the holder's own From 6aace429305eb4f0c9e1e3f7cfa05091b8fbb67f Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 01:35:46 +0000 Subject: [PATCH 03/20] family.py and validate-family.py: `remove` refuses a holder with no `.gitmodules` over an unmerged gitlink too, and the case-variant move prints `--ignore-unmatch` and the `mv` a case-sensitive disk needs, so the mount is no longer emptied there and the printed lines work half typed or with both directories on disk (#611) Fix round 1, from the review of 6f938ed1. `_checked_gitmodules_held` (scripts/family.py) took the member's gitlink only at stage 0. `git submodule deinit` and `git rm` run on an unmerged gitlink as on a merged one, so with `members/

` held only at stages 1, 2 and 3 in a holder whose HEAD lost `.gitmodules`, the dry run exited 0 and the real run emptied the mount and died at its commit. It now takes a gitlink at any stage, and both runs refuse `member-gitmodules-absent` before `deinit`, with the holder byte for byte as it was. The docstring says so. The case-variant refusal of `member-gitlink-absent` (templates/family-root/scripts/validate-family.py, pinned) now prints `git rm -r --cached --ignore-unmatch -- `: a variant whose removal is staged and not committed is still named from HEAD, and the plain line failed there (`pathspec ... did not match`). Its remediation also says what is left on disk: where nothing is checked out at the row's path, as in a fresh clone, `git submodule update --init --recursive` then checks the member out; on a case-sensitive disk a directory left at a variant stays, untracked, and `family.py add` refuses it as `family-members-dir-variant`, so it gives an `mv` of each out of the holder, to `../`, and says to leave it on macOS and Windows, where it is the row's own directory. The validator's module docstring names `member-gitmodules-absent`, says what a staged, uncommitted `.gitmodules` does (HEAD's file is what counts, as #407's decision 3 says: an uncommitted `add` is the finding for its member, a staged section removal is none, a staged restore is still the finding), and gives EXIT CODES' one exception, an unregistered member that is not checked out (exit 1, not 2). Tests (tests/test_family_head_lost_gitmodules.py, 12 more, 38 in all): the unmerged gitlink refused alike and repaired as printed; one test per condition no test held (a checked-out member still gets its tree check beside the finding; HEAD's unreadable file refused in every mode; the `` placeholder; `remove`'s remedy names only members with a gitlink in the index; one finding per path for a duplicate row); the variant move half typed (both disks); and, on a case-sensitive disk, every printed line typed with both directories present, after which the gate passes and another member is added. tests/test_validate_family_unnamed_mounts.py pins the `--ignore-unmatch` line. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- scripts/family.py | 12 +- .../family-root/scripts/validate-family.py | 60 +++- tests/test_family_head_lost_gitmodules.py | 292 ++++++++++++++++-- tests/test_validate_family_unnamed_mounts.py | 2 +- 4 files changed, 322 insertions(+), 44 deletions(-) diff --git a/scripts/family.py b/scripts/family.py index 85b2f87a..32cc1e34 100755 --- a/scripts/family.py +++ b/scripts/family.py @@ -5408,7 +5408,7 @@ def _checked_gitmodules_held(family: Family, path: str) -> None: """`member-gitmodules-absent`: `remove` refuses, before any git command that writes and under `--dry-run` alike, a holder that holds no `.gitmodules` at all, in the work tree, the index or HEAD, while its - index records the member's gitlink (#611). + index records the member's gitlink at any stage (#611). A HOLDER WHOSE HEAD LOST `.gitmodules`, as #407's commit left one. `git rm` edits only a `.gitmodules` the work tree has, and `remove` commits @@ -5416,7 +5416,11 @@ def _checked_gitmodules_held(family: Family, path: str) -> None: add` died (`pathspec '.gitmodules' did not match any files`) after `deinit` and `git rm` had emptied the member, and the rollback left its directory empty; the dry run exited 0. `_checked_gitmodules_as_staged` - asks nothing here, since no copy exists to compare. + asks nothing here, since no copy exists to compare. It asks only where + none of the work tree, the index and HEAD holds a `.gitmodules`, and the + index records the member's gitlink at any stage: `deinit` and `git rm` + run on a merged or an unmerged gitlink alike, and with no `.gitmodules` + anywhere the commit cannot stage one. REFUSED, NOT COMMITTED WITHOUT IT, as #407 refuses a `.gitmodules` the work tree lacks: the holder is left byte for byte as it was. A clone @@ -5436,9 +5440,7 @@ def _checked_gitmodules_held(family: Family, path: str) -> None: return gitlinks = {name.decode("utf-8", "surrogateescape") for name, held in records.items() - if any(record.startswith(b"160000 ") - and record.partition(b"\t")[0].endswith(b" 0") - for record in held)} + if any(record.startswith(b"160000 ") for record in held)} if path not in gitlinks: return sections = [] diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index e4c2200c..d4283032 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -42,7 +42,8 @@ finding and none of the three is asked of it: nothing read through a path the row may not have is a fact about the member (#367). A row whose gitlink is recorded only at a case variant of its path is refused - naming that variant, and the index move that puts it back (#611). + naming that variant, the index move that puts it back, and what to do + with a directory the variant leaves on disk (#611). And, across the rows, that no two record ONE `id`: that is one project pinned twice, however each row spelled its repository (#206). This one reads `family.yaml` alone, so `--no-members` asks it too. @@ -60,12 +61,18 @@ and what it says to do depends on which of the two is the member's own: it names no command that would un-register the member. - that a section of HEAD's `.gitmodules` gives the path of every row's - gitlink (#611, #407's decision 3). A recursive clone reads HEAD's file, - so a gitlink no section of it gives is left empty there, by CI and by - `make bootstrap`, though the holder it was committed in keeps its - checkout. It needs no member checked out, so `--no-members` asks it - too, and a member it names that is not checked out is not then refused - as uninitialized. + gitlink, the `member-gitmodules-absent` finding (#611, #407's decision + 3). A recursive clone reads HEAD's file, so a gitlink no section of it + gives is left empty there, by CI and by `make bootstrap`, though the + holder it was committed in keeps its checkout. It needs no member + checked out, so `--no-members` asks it too, and a member it names that + is not checked out is not then refused as uninitialized. A + `.gitmodules` staged and not committed counts for nothing until it is + committed, since HEAD's file is the one a clone reads: a `family.py + add` staged and not committed is this finding for its own member; a + section taken out and staged, not committed, is no finding; and a file + put back and staged, not committed, in a holder whose HEAD lost it is + still this finding for every member. - the COPIED shape files, each against its `sha256` row in `contracts/shape-pin.yaml`. Editing a copy in place is drift and the exit is to carry it upstream, never to update the digest. @@ -74,8 +81,10 @@ 0 the manifest is valid, every member is in lockstep, every copy matches 1 a FINDING: drift a reviewer can act on 2 a REFUSAL: the question could not be asked — no manifest, an - uninitialized member, an unreadable pin, no git to run. An - unanswerable question is never an implicit pass. + uninitialized member (but for one no section of HEAD's `.gitmodules` + gives: that is `member-gitmodules-absent`, exit 1), an unreadable + pin, no git to run. An unanswerable question is never an implicit + pass. """ from __future__ import annotations @@ -449,10 +458,26 @@ def _gitlink_in_another_case(path: str, commit: str, project: str, records the member once, at its own path. `member-gitlink-unnamed` prints no command for the same gitlink, since taking it out ALONE would un-register the member; these take it out and record it together. + `--ignore-unmatch`, since a variant whose removal is staged and not yet + committed is still named here, from HEAD, and the plain `git rm` of it + failed (`pathspec ... did not match`). + + WHAT IS LEFT ON DISK. Where nothing is checked out at the row's path, + as in a fresh clone, `git submodule update --init --recursive` checks + the member out there once the move is committed. On a case-sensitive + disk a directory at a variant (the empty one a fresh clone leaves, or a + checkout) stays behind, untracked, and `family.py add` refuses it as + `family-members-dir-variant` (#574), so the text says to move it out of + the holder, by an `mv` with nothing to fill in. On macOS and Windows + that directory is the row's own, and the text says to leave it there. """ found = ", ".join(f"{variant} (at {gitlinks[variant]})" for variant in variants) taken_out = " ".join(shlex.quote(variant) for variant in variants) + moved = " and ".join( + f"`mv -- {shlex.quote(variant)} " + f"{shlex.quote('../' + variant.replace('/', '-'))}`" + for variant in variants) return Refusal( "member-gitlink-absent", f"no gitlink (mode 160000) recorded at {path!r}. {project} is named " @@ -462,8 +487,16 @@ def _gitlink_in_another_case(path: str, commit: str, project: str, f"at {path}.", "Remediation: the member is recorded in the wrong case. Move its " "gitlink to the row's path, at the commit the row pins, in the index " - "alone, which touches no file, and commit that:\n" - f" git rm -r --cached -- {taken_out}\n" + "alone, which touches no file, with the two lines below, and commit " + f"that. Where nothing is checked out at {path}, as in a fresh clone, " + "`git submodule update --init --recursive` then checks the member " + "out there. On a case-sensitive disk, as Linux's usually is, a " + f"directory left at {' or '.join(variants)} stays there, untracked, " + "and family.py add refuses it as family-members-dir-variant: move " + f"each one that is there out of the holder, {moved}. On macOS and " + "Windows it is the row's own directory, so leave it. The two " + "lines:\n" + f" git rm -r --cached --ignore-unmatch -- {taken_out}\n" " git update-index --add --cacheinfo " f"{shlex.quote(f'160000,{commit},{path}')}") @@ -1342,6 +1375,11 @@ def _check_gitlinks_registered(root: Path, members: list, `member-path`'s, and one with no gitlink there is `member-gitlink-absent`'s (`gitlinks`, read as `recorded_gitlink` reads one). A holder with no members holds no gitlink and is asked nothing. + THREE SOURCES: the gitlinks from the index first, the rows from the + work tree's `family.yaml`, and the sections from HEAD alone, as #407's + decision 3 has it, so a `.gitmodules` staged and not committed counts + for nothing until it is committed (the module docstring gives the three + cases). The remedy writes the section as `family.py add` writes it, named after its path, with the url it gives a member, `https://github.com/` and the diff --git a/tests/test_family_head_lost_gitmodules.py b/tests/test_family_head_lost_gitmodules.py index 62337ef1..85f35308 100644 --- a/tests/test_family_head_lost_gitmodules.py +++ b/tests/test_family_head_lost_gitmodules.py @@ -28,6 +28,12 @@ that HEAD holds and the index no longer does (#574), so it says to commit the removal between them now. +AND THE STATES BESIDE THEM that the first round's review found: a member's +gitlink held only at the conflict stages, which `remove` emptied as it +emptied a merged one (its S1); a condition of each new check that no test +held (its S2 and N5); and the variant move typed half way, or leaving a +directory on a case-sensitive disk (its N1 and N2). + THE SAME FAMILY, OFFLINE RULES AND READINGS as `test_family_remove_gitmodules.py` (#407), whose helpers are imported. Every remedy is typed as printed: split as a POSIX shell splits it, run with no @@ -45,6 +51,8 @@ import re import shlex +import shutil +import subprocess from pathlib import Path import pytest @@ -54,19 +62,21 @@ #: The family fixture, its names, the offline environment and the readings #: of a holder, imported from the modules that own them rather than rebuilt. from test_family import ( # noqa: F401 - FAMILY, NAME, ORG, SPARE, family, validate, + FAMILY, NAME, ORG, SPARE, family, manifest, validate, ) from test_family_add_excludes import empty_holder # noqa: F401 from test_family_add_refusals import holder, offline # noqa: F401 -from test_family_dry_run import isolated +from test_family_dry_run import hand_edit_digest, isolated from test_family_member_path import MEMBER, OTHER, raw from test_family_remove_gitmodules import ( deleted, edited, head_paths, left_out, ran, refused_alike, typed_as_printed, verb_args, ) from test_scaffold_ignored_shape_files import excluding +from test_validate_family_duplicates import EXACT, irrs_row, rewrite from test_validate_family_unnamed_mounts import ( - commands, findings, spare_commits, stage_gitlink, + codes, commands, findings, spare_commits, stage_gitlink, + stage_unmerged_gitlink, ) GITMODULES = ".gitmodules" @@ -125,16 +135,17 @@ def remove_remedy(*projects: str) -> str: def absent(project: str, commit: str, *, held: bool, - with_url: bool = True) -> str: + with_url: bool = True, printed_url: str | None = None) -> str: """The `member-gitmodules-absent` finding for `project`'s gitlink at `commit`, WHOLE: its line and the commands under it. `held` is whether HEAD holds a `.gitmodules` at all; `with_url` whether the section to - write has no url of its own.""" + write has no url of its own; `printed_url` the url line's value where + it is not `url(project)`.""" path = f"members/{project}" lines = f"\n git config -f .gitmodules submodule.{path}.path {path}" if with_url: lines += (f"\n git config -f .gitmodules submodule.{path}.url " - f"{url(project)}") + f"{printed_url or url(project)}") return (f"FINDING member-gitmodules-absent: {path} is a gitlink at " f"{commit} that a row of members: names, and " + ("no section of HEAD's .gitmodules gives that path" if held @@ -315,6 +326,54 @@ def test_remove_of_a_member_whose_section_alone_is_gone_runs_as_before( assert_passes_in_every_mode(holder, 1) +def test_remove_refuses_it_where_the_members_gitlink_is_unmerged( + family, holder, offline, tmp_path): + """THE REVIEW'S S1: HEAD lost `.gitmodules`, and MEMBER's gitlink is held + only at the conflict stages 1, 2 and 3 (`stage_unmerged_gitlink`). + `deinit` and `git rm` run on an unmerged gitlink as on a merged one, so + while the refusal asked only for one at stage 0, the dry run exited 0 + and the real run emptied the mount and died at its commit. Both refuse + now, with the holder byte for byte as it was, the mount's files and the + three stages included. Typed as printed, the remedy writes both + sections back, and `remove` then commits.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + path = f"members/{MEMBER}" + stage_unmerged_gitlink(holder, path, pinned(holder, MEMBER)) + listed = raw(holder, env, "ls-files", "-s", "--", path).stdout + assert [line.split()[2] for line in listed.splitlines()] == [ + b"1", b"2", b"3"], "premise: unmerged, no stage 0" + args = verb_args("remove", holder, family) + + said, remediation = refused_alike(holder, env, args) + + assert said == REMOVE_REFUSED.format(member=MEMBER) + assert remediation == remove_remedy(MEMBER, OTHER) + assert (holder / path / "project.yaml").is_file() + typed_as_printed(holder, env, remediation) + ran(holder, env, args) + assert head_paths(holder, env) == [f"members/{OTHER}"] + assert_passes_in_every_mode(holder, 1) + + +def test_removes_remedy_names_only_members_whose_gitlink_is_indexed( + family, holder, offline, tmp_path): + """The remedy writes back the section of each member whose gitlink the + index records, and of no other. OTHER's gitlink taken out and its row + left: a section for it would register a path the holder records + nothing at, so the remedy writes MEMBER's section alone.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + raw(holder, env, "rm", "-q", "--cached", "--", f"members/{OTHER}") + commit_all(holder, env, "OTHER's gitlink taken out") + + said, remediation = refused_alike(holder, env, + verb_args("remove", holder, family)) + + assert said == REMOVE_REFUSED.format(member=MEMBER) + assert remediation == remove_remedy(MEMBER) + + # --- 571(c): a gitlink no section of HEAD's .gitmodules gives --------------- @MODES @@ -499,6 +558,104 @@ def test_an_untracked_copy_put_back_is_still_a_finding_until_committed( assert_passes_in_every_mode(holder, 2) +@pytest.mark.parametrize("extra", [(), ("--pins",)], + ids=["members-checked", "pins"]) +def test_a_checked_out_member_it_names_still_has_its_tree_checked( + holder, offline, tmp_path, extra): + """Only a member it names that is NOT checked out stops after its + gitlink. One that is checked out is asked its tree as before: MEMBER's + digest edited by hand, in a holder whose HEAD lost `.gitmodules`, is + `member-digest-mismatch` beside the two `member-gitmodules-absent`.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + hand_edit_digest(holder, env) + + result = validate(holder, *extra) + + assert result.returncode == 1, result.stderr + result.stdout + assert codes(result) == ["member-gitmodules-absent"] * 2 + [ + "member-digest-mismatch"] + assert findings(result)[2].startswith( + f"FINDING member-digest-mismatch: {MEMBER}: pin.tree_sha256 " + f"{'b' * 64}\n"), result.stderr + + +@MODES +def test_a_head_gitmodules_git_cannot_read_is_refused(holder, offline, + tmp_path, extra): + """HEAD's `.gitmodules` committed broken, the work tree's mended and not + committed. A clone reads HEAD's, which git cannot read either, so the + gate refuses `gitmodules-unreadable` in every mode, saying to commit the + mended file, and never reads the broken file as one that gives no + section. Committed, the holder passes.""" + env = isolated(offline, tmp_path) + mended = (holder / GITMODULES).read_bytes() + committed_gitmodules(holder, env, + f'[submodule "members/{MEMBER}"\n\tpath = x\n' + .encode("utf-8")) + (holder / GITMODULES).write_bytes(mended) + + result = validate(holder, *extra) + + assert result.returncode == 2, result.stderr + result.stdout + lines = result.stderr.splitlines() + assert lines[0].startswith( + "REFUSED gitmodules-unreadable: HEAD's .gitmodules could not be read " + "as git reads it: `git config -z --blob "), result.stderr + assert lines[-1] == ( + "Remediation: git cannot read it either, so none of this holder's " + "submodules can be cloned from it. Mend the file by hand (`git config " + "-f .gitmodules --list` says where it breaks), commit it, then run " + "this again.") + staged_and_committed(holder, env, "Commit the mended .gitmodules") + assert_passes_in_every_mode(holder, 2) + + +def test_a_repository_not_org_and_name_prints_the_url_placeholder( + holder, offline, tmp_path): + """The url line gives `https://github.com/.git`, the url + `family.py add` writes, only where the row's `repository:` is + `/`. From any other value that would be a url nobody chose, + so the line prints `` for the person to fill in.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + rows = (holder / "family.yaml").read_bytes() + line = f" repository: {ORG}/{MEMBER}\n".encode("utf-8") + assert rows.count(line) == 1, "fixture drift: MEMBER's repository line" + (holder / "family.yaml").write_bytes(rows.replace( + line, f" repository: {MEMBER}\n".encode("utf-8"))) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stderr + result.stdout + assert findings(result) == [ + absent(MEMBER, pinned(holder, MEMBER), held=False, + printed_url=""), + absent(OTHER, pinned(holder, OTHER), held=False)] + + +def test_a_row_written_twice_is_one_finding_per_path(holder, offline, + tmp_path): + """MEMBER's row written twice in a holder whose HEAD lost `.gitmodules`: + `member-duplicate` once, and `member-gitmodules-absent` once per path, + not once per row.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + row = irrs_row(holder) + assert row["project"] == MEMBER, "fixture drift: MEMBER is IRRS" + other = next(r for r in manifest(holder)["members"] + if r["project"] == OTHER) + rewrite(holder, [row, dict(row), other]) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stderr + result.stdout + assert findings(result) == [ + EXACT, + absent(MEMBER, pinned(holder, MEMBER), held=False), + absent(OTHER, pinned(holder, OTHER), held=False)] + + # --- 574/2: a gitlink recorded only at a case variant of the row's path ----- def variant_refused(path: str, commit: str, variants: list[str], @@ -508,6 +665,8 @@ def variant_refused(path: str, commit: str, variants: list[str], project = path.split("/", 1)[1] found = ", ".join(f"{variant} (at {gitlinks[variant]})" for variant in variants) + moved = " and ".join(f"`mv -- {variant} ../{variant.replace('/', '-')}`" + for variant in variants) return [ f"REFUSED member-gitlink-absent: no gitlink (mode 160000) recorded at " f"'{path}'. {project} is named in family.yaml, and this repository " @@ -516,11 +675,44 @@ def variant_refused(path: str, commit: str, variants: list[str], f"recursive clone checks out nothing at {path}.", "Remediation: the member is recorded in the wrong case. Move its " "gitlink to the row's path, at the commit the row pins, in the index " - "alone, which touches no file, and commit that:", - " git rm -r --cached -- " + " ".join(variants), + "alone, which touches no file, with the two lines below, and commit " + f"that. Where nothing is checked out at {path}, as in a fresh clone, " + "`git submodule update --init --recursive` then checks the member " + "out there. On a case-sensitive disk, as Linux's usually is, a " + f"directory left at {' or '.join(variants)} stays there, untracked, " + "and family.py add refuses it as family-members-dir-variant: move " + f"each one that is there out of the holder, {moved}. On macOS and " + "Windows it is the row's own directory, so leave it. The two lines:", + " git rm -r --cached --ignore-unmatch -- " + " ".join(variants), f" git update-index --add --cacheinfo 160000,{commit},{path}"] +def refused_at(root: Path, commit: str, variants: list[str]) -> list[str]: + """The members-checked gate in `root`, where MEMBER's gitlink is recorded + only at `variants`, at `commit`: exit 2, in `variant_refused`'s lines, + which it returns.""" + result = validate(root) + assert result.returncode == 2, result.stderr + result.stdout + lines = result.stderr.splitlines() + assert lines == variant_refused(f"members/{MEMBER}", commit, variants, + dict.fromkeys(variants, commit)) + return lines + + +def moved_as_printed(root: Path, env: dict, lines: list[str], + variants: list[str]) -> None: + """The refusal's two command lines typed as printed, and the commit it + says to make: HEAD then records MEMBER once, at its own path, and at + none of `variants`.""" + for line in lines[2:]: + raw(root, env, *shlex.split(line)[1:]) + commit_all(root, env, "Move the gitlink as printed") + listed = raw(root, env, "ls-tree", "-r", "--name-only", "HEAD").stdout + assert f"members/{MEMBER}\n".encode() in listed + for variant in variants: + assert f"{variant}\n".encode() not in listed + + def recorded_at(root: Path, env: dict, project: str, *spellings: str) -> str: """`project`'s gitlink taken out of the index at its own path and @@ -551,23 +743,74 @@ def test_a_gitlink_only_at_a_case_variant_is_named_and_moved_as_printed( variants = sorted(spelling.format(p=MEMBER) for spelling in spellings) commit = recorded_at(holder, env, MEMBER, *variants) - result = validate(holder) + lines = refused_at(holder, commit, variants) - assert result.returncode == 2, result.stderr + result.stdout - lines = result.stderr.splitlines() - path = f"members/{MEMBER}" - assert lines == variant_refused(path, commit, variants, - dict.fromkeys(variants, commit)) - for line in lines[2:]: - raw(holder, env, *shlex.split(line)[1:]) - commit_all(holder, env, "Move the gitlink as printed") - listed = raw(holder, env, "ls-tree", "-r", "--name-only", "HEAD").stdout - assert f"{path}\n".encode() in listed - for variant in variants: - assert f"{variant}\n".encode() not in listed + moved_as_printed(holder, env, lines, variants) assert_passes_in_every_mode(holder, 2) +@DISKS +def test_a_variant_whose_removal_is_staged_is_moved_as_printed( + holder, offline, tmp_path, ignorecase): + """HALF REPAIRED (the review's N1): the variant's removal staged and not + committed. HEAD still records the gitlink there, so the refusal still + names it, and the plain `git rm -r --cached` of it failed, `pathspec + ... did not match`. With `--ignore-unmatch` the lines typed as printed + and committed move the gitlink, and the holder passes.""" + env = on_disk(isolated(offline, tmp_path), ignorecase) + variant = f"Members/{MEMBER}" + commit = recorded_at(holder, env, MEMBER, variant) + raw(holder, env, "rm", "-q", "--cached", "--", variant) + + lines = refused_at(holder, commit, [variant]) + + moved_as_printed(holder, env, lines, [variant]) + assert_passes_in_every_mode(holder, 2) + + +@pytest.fixture +def case_sensitive_disk(tmp_path): + """Skips where `Members/` and `members/` would be one directory, as + macOS and Windows hold them by default (nothing is left behind there to + move), or where there is no `mv` to type.""" + (tmp_path / "CaseProbe").mkdir() + if (tmp_path / "caseprobe").exists(): + pytest.skip("this disk is case-insensitive: Members/IRRS is " + "members/IRRS here, so no second directory can exist") + if shutil.which("mv") is None: + pytest.skip("no mv on PATH to type the printed line with") + + +def test_a_variant_checkout_left_on_disk_is_moved_out_as_printed( + family, holder, offline, tmp_path, case_sensitive_disk): + """THE REVIEW'S N2, in its C3 layout: MEMBER's gitlink recorded only at + `Members/IRRS`, its checkout still at `members/IRRS`, and a second + checkout at `Members/IRRS`, as a case-sensitive disk can hold. The + index move leaves that directory on disk, untracked, where `family.py + add` refused any other member past it (`family-members-dir-variant`). + Every line the refusal prints, typed as printed: the two lines, their + commit, and the `mv` it gives for a case-sensitive disk. The gate then + passes in every mode, and SPARE is added.""" + env = isolated(offline, tmp_path) + variant = f"Members/{MEMBER}" + commit = recorded_at(holder, env, MEMBER, variant) + raw(holder, env, "clone", "-q", "--no-checkout", + str(family["remotes"] / f"{MEMBER}.git"), variant) + raw(holder / variant, env, "checkout", "-q", "--detach", commit) + + lines = refused_at(holder, commit, [variant]) + + moved_as_printed(holder, env, lines, [variant]) + (move,) = re.findall(r"`(mv [^`]*)`", lines[1]) + subprocess.run(shlex.split(move), cwd=str(holder), check=True) + assert not (holder / variant).exists() + assert (tmp_path / f"Members-{MEMBER}" / "project.yaml").is_file() + assert_passes_in_every_mode(holder, 2) + added = run_script(FAMILY, *verb_args("add", holder, family), env=env) + assert added.returncode == 0, added.stdout + added.stderr + assert_passes_in_every_mode(holder, 3) + + def test_a_fresh_clone_of_a_case_variant_says_it_too(family, holder, offline, tmp_path): """The same state pushed: a fresh clone names the variant, where the @@ -581,12 +824,7 @@ def test_a_fresh_clone_of_a_case_variant_says_it_too(family, holder, offline, cloned = clone_just_pushed(remote, clone, "w611/case", recurse=False) assert cloned.returncode == 0, cloned.stdout + cloned.stderr - result = validate(clone) - - assert result.returncode == 2, result.stderr + result.stdout - variant = f"Members/{MEMBER}" - assert result.stderr.splitlines() == variant_refused( - f"members/{MEMBER}", commit, [variant], {variant: commit}) + refused_at(clone, commit, [f"Members/{MEMBER}"]) # --- 574/3: a hand mount's remedy commits the removal before the add --------- diff --git a/tests/test_validate_family_unnamed_mounts.py b/tests/test_validate_family_unnamed_mounts.py index 7c4babf5..3144f710 100644 --- a/tests/test_validate_family_unnamed_mounts.py +++ b/tests/test_validate_family_unnamed_mounts.py @@ -538,7 +538,7 @@ def test_a_member_recorded_only_in_another_case_says_so_and_prints_no_command( "REFUSED member-gitlink-absent: no gitlink (mode 160000) " "recorded at 'members/IRSS'."), result.stderr assert result.stderr.splitlines()[-2:] == [ - " git rm -r --cached -- members/irss", + " git rm -r --cached --ignore-unmatch -- members/irss", " git update-index --add --cacheinfo " f"160000,{named},members/IRSS"], result.stderr return From d660ec564e652be9e2d44de93c53f22b962e757c Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 12:12:05 +0000 Subject: [PATCH 04/20] bootstrap.py and validate-family.py: a member no credential would check out fails the family bootstrap, every row's gitlink is asked before any row's checkout, and `--no-members` prints the case-variant move, so a clone of a holder whose HEAD lost `.gitmodules` or of #363's state no longer ends `family bootstrap ok` or hides the variant behind `member-uninitialized` (#648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (a) `templates/family-root/scripts/bootstrap.py` said "every member and leg fetched" and ended `family bootstrap ok`, exit 0, over members it had not checked out: in a fresh recursive clone of a holder whose HEAD lost `.gitmodules` (git exits 0), and in a plain clone of that holder or of #363's state, where git dies `No url found for submodule path` (exit 128). After the fetch it now asks this holder's own records of each row whose member is not checked out: no gitlink at the row's path (`recorded_gitlink`, as `member-gitlink-absent` reads it), or no section of `.gitmodules` giving that path, or, where git exited 0, any such row. Each is named on one `NOT CHECKED OUT, and not for want of a credential` line with why, which names `validate-family.py`, and the run ends `family bootstrap FAILED:

(not checked out)`, exit 1. "every member and leg fetched" is said only where git exited 0 and every row is checked out. A row git could not fetch for want of a credential is still the skip it was (`test_bootstrap_degrades_when_a_member_cannot_be_fetched`), and a good holder's run prints what it printed. (b) `validate-family.py` asked each row whole, gitlink then checkout, before the next. In a plain clone of #363's state plus `git submodule update --init --recursive`, which dies at `Members/

` before it checks out any member, an earlier row was refused `member-uninitialized` (naming that same update) and the later row's variant refusal was never reached. `_check_member` now stops at the gitlink and returns the member (`Gitlinked`); `_check_member_checkout` asks the digest and identity of each once every row's gitlink has been asked. (c) Under `--no-members`, `member-gitlink-unnamed` for a member's only gitlink, in the wrong case, printed no command. Where the row's path is its canonical one and its pin a commit, it now prints the index move the row's members-mode refusal prints, word for word (`_wrong_case_move`, shared by both), to the pinned commit, every variant on one line. Docs: the validator's WHAT IS CHECKED docstring (the order; the move under --no-members) and bootstrap's (a) and (c). README.md's "A member it cannot fetch is reported and skipped, never a failure" is no longer true, so it says "One it lacks a credential for is reported and skipped, not a failure", in the same line count (README is at its 1449-line cap), and docs/handbook.html says the same, with its § Families marker set to the segment's new blob. Pinned copies changed: templates/family-root/scripts/bootstrap.py and templates/family-root/scripts/validate-family.py. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/handbook.html | 4 +- templates/family-root/scripts/bootstrap.py | 135 ++++- .../family-root/scripts/validate-family.py | 177 +++++-- tests/test_family_bootstrap_gate_order.py | 468 ++++++++++++++++++ tests/test_validate_family_unnamed_mounts.py | 32 +- 6 files changed, 728 insertions(+), 90 deletions(-) create mode 100644 tests/test_family_bootstrap_gate_order.py diff --git a/README.md b/README.md index 2664a842..4ff2989f 100644 --- a/README.md +++ b/README.md @@ -942,7 +942,7 @@ at the right commit. **A family pins assembly roots, never legs**: the GitHub App or `SHAPE_LEGS_TOKEN` credential the workflow does, as a member is a private assembly root mounting two private legs — then runs each member's own `make bootstrap`; `make validate` runs the family validator, then each -member's. A member it cannot fetch is reported and skipped, never a failure. +member's. One it lacks a credential for is reported and skipped, not a failure. The holder carries the same COPY pin an assembly root does, so `update-shape.py` re-syncs it the same way; it mirrors into `family.yaml` diff --git a/docs/handbook.html b/docs/handbook.html index 801cc4ac..82ec73d9 100644 --- a/docs/handbook.html +++ b/docs/handbook.html @@ -1268,7 +1268,7 @@

Adopting an existing repository

- +

Reference

Families: a holder for projects that ship separately

@@ -1361,7 +1361,7 @@

The InkRouter example

resolving the same GitHub App or SHAPE_LEGS_TOKEN credential the workflow does — and then runs each member’s own make bootstrap. make validate runs the family validator and then each member’s own - make validate. A member that could not be fetched is reported and skipped, never + make validate. A member it lacks a credential for is reported and skipped, not a failure. The holder carries the same COPY pin an assembly root does, so update-shape.py re-syncs it the same way; it mirrors into family.yaml instead of project.yaml and must leave diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index d9ddcc30..e3fb3b10 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -27,6 +27,18 @@ refused to do anything because one member is unreachable would be worse than one that says which. + "every member and leg fetched" is said only where git exited 0 AND + every row's member is checked out (#648). A member left NOT CHECKED + OUT where no credential would check it out is said here instead, with + why, and fails the run (c): this holder records no gitlink at the + row's path (a gitlink recorded only in another case, #363), no + section of `.gitmodules` gives that path (a holder whose HEAD lost the + file, #611), or git exited 0 and checked nothing out there. Git exits + 0 over the second where a recursive clone ran first, and 128 (`No url + found for submodule path`) in a plain clone, as it does for a missing + credential, so what the holder records is the test, never git's + words. The line names `validate-family.py`, which says what to commit. + (b) RUNS EACH MEMBER'S OWN `make bootstrap`. Each member is a whole project and knows how to bootstrap itself — its legs onto their tracking branches at their pinned commits, its own validators, its own @@ -35,9 +47,10 @@ `--members-make validate` runs `make validate` instead, which is what the family Makefile's `validate` target uses. - (c) EXITS 0 unless a member's own command failed. A member that is not - checked out is REPORTED and skipped, not a failure: it is the state a - missing credential leaves, and it is already reported in (a). + (c) EXITS 0 unless a member's own command failed, or (a) left a member not + checked out that no credential would check out. Any other member that + is not checked out is REPORTED and skipped, not a failure: it is the + state a missing credential leaves, and it is already reported in (a). THE FAMILY CONFERS NOTHING and neither does this command. It fetches, it delegates, and it says what it could not reach. @@ -54,10 +67,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from repo_shape import ( # noqa: E402 - Refusal, escape_unencodable_stdout, find_repo_root, load_yaml, spelled, + PYTHON, Refusal, escape_unencodable_stdout, find_repo_root, git_out, + load_yaml, recorded_gitlink, spelled, ) MANIFEST = "family.yaml" +GITMODULES = ".gitmodules" DEFAULT_MEMBERS_DIR = "members" #: The two environment variables a leg/member credential may arrive in, in the @@ -79,8 +94,13 @@ def credential() -> tuple[str, str]: return "none", "" -def fetch_members(root: Path) -> bool: - """(a) `git submodule update --init --recursive`, authenticated or not.""" +def fetch_members(root: Path, rows: list[dict]) -> list[str]: + """(a) `git submodule update --init --recursive`, authenticated or not. + + Returns a failure line for each of `rows` whose member it leaves not + checked out where no credential would check it out (`unfetchable`), for + the run to end FAILED on (#648); every member and leg is said to be + fetched only where there is none and git exited 0.""" source, token = credential() args = ["git"] if token: @@ -92,21 +112,83 @@ def fetch_members(root: Path) -> bool: sys.stdout.flush() proc = subprocess.run(args, cwd=str(root), capture_output=True, text=True, encoding="utf-8", errors="replace", check=False) - if proc.returncode == 0: + if proc.returncode != 0: + # The token can appear in git's own error text (it is in the URL it + # tried), so the output is scrubbed before it is printed. + message = (proc.stderr + proc.stdout).strip() + if token: + message = message.replace(token, "") + print(f" FETCH INCOMPLETE (git exit {proc.returncode}) using " + f"credential source '{source}':\n{message}", file=sys.stderr) + print(" If the members are PRIVATE, set SHAPE_LEGS_TOKEN (a " + "fine-grained PAT with contents:read on them) or export an " + "App-minted SHAPE_LEGS_APP_TOKEN, then re-run. What did arrive " + "is bootstrapped below.", file=sys.stderr) + left = unfetchable(root, rows, proc.returncode == 0) + if left: + print(" NOT CHECKED OUT, and not for want of a credential: " + + ", ".join(f"{spelled(path)} ({why})" for _, path, why in left) + + f". `{PYTHON} scripts/validate-family.py` names the cause and " + "the commit that cures it.", file=sys.stderr) + elif proc.returncode == 0: print(" every member and leg fetched") - return True - # The token can appear in git's own error text (it is in the URL it tried), - # so the output is scrubbed before it is printed. - message = (proc.stderr + proc.stdout).strip() - if token: - message = message.replace(token, "") - print(f" FETCH INCOMPLETE (git exit {proc.returncode}) using credential " - f"source '{source}':\n{message}", file=sys.stderr) - print(" If the members are PRIVATE, set SHAPE_LEGS_TOKEN (a fine-grained " - "PAT with contents:read on them) or export an App-minted " - "SHAPE_LEGS_APP_TOKEN, then re-run. What did arrive is bootstrapped " - "below.", file=sys.stderr) - return False + return [f"{spelled(row.get('project') or '?')} (not checked out)" + for row, _, _ in left] + + +def gitmodules_paths(root: Path) -> set[str]: + """Every path a section of `.gitmodules` gives, read as git reads it: + the work tree's file, the one `git submodule update` reads, which a + fresh clone takes from HEAD. None where there is no such file, or one + git cannot read: git has no url for any submodule then either.""" + if not os.path.lexists(root / GITMODULES): + return set() + try: + listed = git_out(["config", "-z", "--file", GITMODULES, "--list"], + cwd=root, binary=True) + except Refusal: + return set() + paths: set[str] = set() + for record in listed.split(b"\0"): + key, _, value = record.decode("utf-8", + "surrogateescape").partition("\n") + if key.startswith("submodule.") and key.endswith(".path"): + paths.add(value) + return paths + + +def unfetchable(root: Path, rows: list[dict], + exited_0: bool) -> list[tuple[dict, str, str]]: + """`(row, path, why)` for each row whose member (a) left NOT CHECKED OUT + where no credential would check it out (#648), in row order. + + Asked of this holder's own records, never of git's words: a row at + whose path the holder records no gitlink (`recorded_gitlink`, + as `validate-family.py`'s `member-gitlink-absent` reads one), or whose + gitlink no section of `.gitmodules` gives a path (as its + `member-gitmodules-absent` reads HEAD's file, which is the work tree's + in a clone); and, where git exited 0, any row at all, since git then + lacked nothing a credential gives. Any other row not checked out is the + missing credential's, and stays the skip it always was. A path two rows + give is said once.""" + left: dict[str, dict] = {} + for row in rows: + if not (member_path(root, row) / ".git").exists(): + left.setdefault(row_path(row), row) + if not left: + return [] + given = gitmodules_paths(root) + found: list[tuple[dict, str, str]] = [] + for path, row in left.items(): + if recorded_gitlink(root, path) is None: + found.append((row, path, "no gitlink is recorded at that path")) + elif path not in given: + found.append((row, path, f"no section of {GITMODULES} gives that " + "path")) + elif exited_0: + found.append((row, path, "git exited 0 and checked nothing out " + "there")) + return found def members_of(root: Path) -> tuple[str, list[dict]]: @@ -128,9 +210,14 @@ def members_of(root: Path) -> tuple[str, list[dict]]: return name, rows +def row_path(row: dict) -> str: + """The path a row mounts its member at, as the holder spells it.""" + return str(row.get("path") + or f"{DEFAULT_MEMBERS_DIR}/{row.get('project')}") + + def member_path(root: Path, row: dict) -> Path: - return root / str(row.get("path") - or f"{DEFAULT_MEMBERS_DIR}/{row.get('project')}") + return root / row_path(row) def run_member_target(root: Path, row: dict, target: str) -> str | None: @@ -187,13 +274,13 @@ def main(argv: list[str] | None = None) -> int: f"{len(rows)} member(s)") print("\n(a) members and their legs") + failed: list[str] = [] if args.no_fetch: print(" skipped (--no-fetch)") else: - fetch_members(root) + failed += fetch_members(root, rows) print(f"\n(b) each member's own `make {spelled(args.members_make)}`") - failed: list[str] = [] if args.skip_members: print(" skipped (--skip-members)") elif not rows: diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index d4283032..aca84f2b 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -43,7 +43,11 @@ path the row may not have is a fact about the member (#367). A row whose gitlink is recorded only at a case variant of its path is refused naming that variant, the index move that puts it back, and what to do - with a directory the variant leaves on disk (#611). + with a directory the variant leaves on disk (#611). The first is asked + of every row before the other two are asked of any (#648), so a later + row's gitlink refusal is never hidden behind an earlier row's + `member-uninitialized`, as a plain clone whose `git submodule update` + died at that gitlink left it. And, across the rows, that no two record ONE `id`: that is one project pinned twice, however each row spelled its repository (#206). This one reads `family.yaml` alone, so `--no-members` asks it too. @@ -59,7 +63,10 @@ them ever asked. Neither needs a member checked out, so `--no-members` asks them too. A path a row names in another case is still one of these, and what it says to do depends on which of the two is the member's own: - it names no command that would un-register the member. + it names no command that would un-register the member, and for a member + recorded only in the wrong case it names the index move that records it + at its row's path, as the row's own refusal does members checked + (#648). - that a section of HEAD's `.gitmodules` gives the path of every row's gitlink, the `member-gitmodules-absent` finding (#611, #407's decision 3). A recursive clone reads HEAD's file, so a gitlink no section of it @@ -456,11 +463,12 @@ def _gitlink_in_another_case(path: str, commit: str, project: str, commit the row pins, reading no disk either. So the two lines work on every disk, wherever the checkout stands, and the commit after them records the member once, at its own path. `member-gitlink-unnamed` - prints no command for the same gitlink, since taking it out ALONE would - un-register the member; these take it out and record it together. - `--ignore-unmatch`, since a variant whose removal is staged and not yet - committed is still named here, from HEAD, and the plain `git rm` of it - failed (`pathspec ... did not match`). + prints the same move for the same gitlink under `--no-members` (#648), + and no command that takes it out ALONE, which would un-register the + member; these take it out and record it together. `--ignore-unmatch`, + since a variant whose removal is staged and not yet committed is still + named here, from HEAD, and the plain `git rm` of it failed (`pathspec + ... did not match`). WHAT IS LEFT ON DISK. Where nothing is checked out at the row's path, as in a fresh clone, `git submodule update --init --recursive` checks @@ -473,11 +481,6 @@ def _gitlink_in_another_case(path: str, commit: str, project: str, """ found = ", ".join(f"{variant} (at {gitlinks[variant]})" for variant in variants) - taken_out = " ".join(shlex.quote(variant) for variant in variants) - moved = " and ".join( - f"`mv -- {shlex.quote(variant)} " - f"{shlex.quote('../' + variant.replace('/', '-'))}`" - for variant in variants) return Refusal( "member-gitlink-absent", f"no gitlink (mode 160000) recorded at {path!r}. {project} is named " @@ -485,17 +488,31 @@ def _gitlink_in_another_case(path: str, commit: str, project: str, "case variant of that path, which macOS and Windows hold as one " f"directory with it: {found}. A recursive clone checks out nothing " f"at {path}.", - "Remediation: the member is recorded in the wrong case. Move its " - "gitlink to the row's path, at the commit the row pins, in the index " - "alone, which touches no file, with the two lines below, and commit " - f"that. Where nothing is checked out at {path}, as in a fresh clone, " - "`git submodule update --init --recursive` then checks the member " - "out there. On a case-sensitive disk, as Linux's usually is, a " - f"directory left at {' or '.join(variants)} stays there, untracked, " - "and family.py add refuses it as family-members-dir-variant: move " - f"each one that is there out of the holder, {moved}. On macOS and " - "Windows it is the row's own directory, so leave it. The two " - "lines:\n" + "Remediation: the member is recorded in the wrong case. " + + _wrong_case_move(path, commit, variants)) + + +def _wrong_case_move(path: str, commit: str, variants: list[str]) -> str: + """The index move for a member recorded only at `variants`, case + variants of its row's `path`, to `path` at the pinned `commit`, with + what it leaves on disk: what `_gitlink_in_another_case` says, and + `member-gitlink-unnamed` under `--no-members` (#648), in one text.""" + taken_out = " ".join(shlex.quote(variant) for variant in variants) + moved = " and ".join( + f"`mv -- {shlex.quote(variant)} " + f"{shlex.quote('../' + variant.replace('/', '-'))}`" + for variant in variants) + return ( + "Move its gitlink to the row's path, at the commit the row pins, in " + "the index alone, which touches no file, with the two lines below, " + f"and commit that. Where nothing is checked out at {path}, as in a " + "fresh clone, `git submodule update --init --recursive` then checks " + "the member out there. On a case-sensitive disk, as Linux's usually " + f"is, a directory left at {' or '.join(variants)} stays there, " + "untracked, and family.py add refuses it as " + "family-members-dir-variant: move each one that is there out of the " + f"holder, {moved}. On macOS and Windows it is the row's own " + "directory, so leave it. The two lines:\n" f" git rm -r --cached --ignore-unmatch -- {taken_out}\n" " git update-index --add --cacheinfo " f"{shlex.quote(f'160000,{commit},{path}')}") @@ -554,9 +571,22 @@ def _check_member_tree(member: Path, pin: dict, path: str, commit: str, report.note(f"{path}: tree digest recomputes ({recorded[:12]}…)") +class Gitlinked(NamedTuple): + """A member `_check_member` asked up to its gitlink, with what + `_check_member_checkout` reads its checkout by.""" + + row: dict + project: str + path: str + pin: dict + commit: str + + def _check_member(root: Path, row: dict, seen: dict, report: Report, - gitlinks: dict[str, str], unregistered: set[str]) -> None: - """One member: the row itself, the gitlink, the digest, the identity. + gitlinks: dict[str, str]) -> Gitlinked | None: + """One member up to its checkout: the row itself and the gitlink. The + member, for `_check_member_checkout` to ask its digest and identity of, + or None where nothing more is asked of it. NOTHING IS READ THROUGH A PATH THE ROW MAY NOT HAVE (#367). A hand-edited `path:` is `member-path`, and this member's checks stop once its pin is @@ -570,18 +600,10 @@ def _check_member(root: Path, row: dict, seen: dict, report: Report, The git each check runs is run inside `_git_runs`, which names the check when there is no git to run (#368); the identity check reads a file and runs none. - - A MEMBER NO SECTION OF HEAD'S `.gitmodules` GIVES (`unregistered`, the - `member-gitmodules-absent` finding, #611) that is not checked out stops - after its gitlink. Refused `member-uninitialized`, it named `git - submodule update --init`, which exits 0 there and checks nothing out, - and the refusal dropped the finding that says why: the state a fresh - clone of a holder whose HEAD lost `.gitmodules` is in, and CI's. The - finding is made all the same, so the run still fails. """ project = _member_project_name(row, seen, report) if project is None: - return + return None path = _check_member_repository_and_path(row, project, report) @@ -589,23 +611,38 @@ def _check_member(root: Path, row: dict, seen: dict, report: Report, if not isinstance(pin, dict): report.finding("member-pin", f"{project}: `pin:` is {pin!r}, expected " "a mapping") - return + return None commit = _pin_commit(pin, project, report) if path is None: - return + return None # ---- the gitlink ------------------------------------------------------- with _git_runs(f"the gitlink recorded at {path} could not be read"): _check_member_gitlink(root, path, commit, project, gitlinks, report) + return Gitlinked(row, project, path, pin, commit) + - # ---- the digest and the identity -------------------------------------- - member = root / path - if path in unregistered and not (member / ".git").exists(): +def _check_member_checkout(root: Path, asked: Gitlinked, report: Report, + unregistered: set[str]) -> None: + """One member's digest and identity, read in its checkout, asked once + every row's gitlink has been (`_check_members`, #648). + + A MEMBER NO SECTION OF HEAD'S `.gitmodules` GIVES (`unregistered`, the + `member-gitmodules-absent` finding, #611) that is not checked out is + asked nothing here. Refused `member-uninitialized`, it named `git + submodule update --init`, which exits 0 there and checks nothing out, + and the refusal dropped the finding that says why: the state a fresh + clone of a holder whose HEAD lost `.gitmodules` is in, and CI's. The + finding is made all the same, so the run still fails. + """ + member = root / asked.path + if asked.path in unregistered and not (member / ".git").exists(): return - with _git_runs(f"{project}'s pinned commit and its tree digest could not " - f"be read in {path}"): - _check_member_tree(member, pin, path, commit, project, report) - _check_member_identity(member, row, project, report) + with _git_runs(f"{asked.project}'s pinned commit and its tree digest " + f"could not be read in {asked.path}"): + _check_member_tree(member, asked.pin, asked.path, asked.commit, + asked.project, report) + _check_member_identity(member, asked.row, asked.project, report) def _check_member_identity(member: Path, row: dict, project: str, @@ -1074,11 +1111,14 @@ def _member_gitlinks(root: Path) -> dict[str, str]: class RowPath(NamedTuple): """What a row names by the path it gives: `who` names the row in a - finding, and `canonical` is the one path its project may have, - `members/`, or None for a row with no `project:`.""" + finding, `canonical` is the one path its project may have, + `members/`, or None for a row with no `project:`, and `pin` is + the commit its `pin:` names, lowered, or None where that is no 40-hex + commit (#648).""" who: str canonical: str | None + pin: str | None = None def _row_paths(members: list) -> dict[str, RowPath]: @@ -1109,9 +1149,11 @@ def _row_paths(members: list) -> dict[str, RowPath]: path = canonical else: continue + pin = row.get("pin") + commit = str(pin.get("commit") or "") if isinstance(pin, dict) else "" named.setdefault(path, RowPath( f"the row for {project}" if project else "a row with no project:", - canonical)) + canonical, commit.lower() if COMMIT_RE.match(commit) else None)) return named @@ -1190,6 +1232,10 @@ def _variant_advice(kind: str, path: str, row_path: str, row: RowPath, is wrong, and where to put its path back; a `WRONG_CASE` says the member is recorded in the wrong case, and sends the operator to `git log -p -- family.yaml`, as an unnamed path that no row folds to does (`RESTORE_ROW`). + A `WRONG_CASE` gitlink whose row's path is its canonical one and whose + pin is a commit never comes here: `_check_gitlinks_named` prints the + index move for it, which records the member as it takes the gitlink out + (#648). """ if kind == ROW_EDITED: return (f" It is {path}, the one path {row.who} may have, and the row " @@ -1220,9 +1266,15 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], still a finding, and says which of three it is (`_variant_case`). A second path to the member's own directory is taken out of the index alone with `git rm --cached`, which touches no file, and committed. The - member's own gitlink, which the ROW misspells, and the only record of a - member there is, in the wrong case, print no command: either could - un-register the member. Any other is taken out with a plain `git rm`, + member's own gitlink, which the ROW misspells, prints no command, since + taking it out could un-register the member. The only record of a member + there is, in the wrong case, prints the index move members mode's + `member-gitlink-absent` prints (`_wrong_case_move`, #648), which records + the member at its row's path as it takes the variants out, so + `--no-members`, the run CI falls back to when the members cannot be + checked out, names the cure too; where the row's path is not its + canonical one, or its pin no commit, it prints no command either. Any + other is taken out with a plain `git rm`, which refuses while its checkout holds changes, then mounted with `family.py add`: `add` cannot adopt an existing mount, and after `git rm --cached` alone it refuses the checkout left in the way @@ -1247,6 +1299,16 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], if variant is not None: row_path, row = variant kind = _variant_case(path, row_path, row, holds) + if kind == WRONG_CASE and row_path == row.canonical and row.pin: + variants = sorted(other for other in holds + if other != row_path + and other.casefold() == row_path.casefold()) + report.finding( + "member-gitlink-unnamed", + f"{said}{_variant_said(row_path, row)} The member is " + "recorded in the wrong case. " + + _wrong_case_move(row_path, row.pin, variants)) + continue if kind != STRAY: report.finding( "member-gitlink-unnamed", @@ -1436,6 +1498,16 @@ def _check_members(root: Path, members: list, no_members: bool, `.gitmodules` gives twice (#300), then the gitlinks and sections no row names, which read the index and HEAD as well (#361), and the gitlinks no section of HEAD's `.gitmodules` gives (#611). + + THEN EVERY ROW'S GITLINK, AND ONLY THEN ANY ROW'S CHECKOUT (#648). Each + row used to be asked whole before the next was asked at all, so in a + plain clone of a holder whose later row's gitlink is recorded only in + another case, where `git submodule update --init --recursive` dies at + that gitlink (`No url found for submodule path`, exit 128) before it + checks out any member, an earlier row was refused + `member-uninitialized`, naming that same update, and the later row's + `member-gitlink-absent`, which names the move that cures both, was + never reached. """ _check_member_names_unique(members, report) _check_member_ids_unique(root, members, report) @@ -1453,12 +1525,17 @@ def _check_members(root: Path, members: list, no_members: bool, "here") return seen: dict[str, bool] = {} + gitlinked: list[Gitlinked] = [] for row in members: if not isinstance(row, dict): report.finding("member-row", f"a member is not a mapping: {row!r}") continue - _check_member(root, row, seen, report, gitlinks, unregistered) + asked = _check_member(root, row, seen, report, gitlinks) + if asked is not None: + gitlinked.append(asked) + for asked in gitlinked: + _check_member_checkout(root, asked, report, unregistered) if not members: report.note("no members yet: a family with none is empty, not " "wrong") diff --git a/tests/test_family_bootstrap_gate_order.py b/tests/test_family_bootstrap_gate_order.py new file mode 100644 index 00000000..9962f731 --- /dev/null +++ b/tests/test_family_bootstrap_gate_order.py @@ -0,0 +1,468 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A holder's own bootstrap and gate after a clone, in every order and mode +(#648). + +THE ISSUE'S THREE LINES, each reproduced on #639's head before this change: + +- (a) `scripts/bootstrap.py` ended `family bootstrap ok`, exit 0, having + checked no member out. In a fresh recursive clone of a holder whose HEAD + lost `.gitmodules`, `git submodule update` exits 0, and it said "every + member and leg fetched", `NOT CHECKED OUT; skipped` for each member, then + ok. In a plain clone of that holder, or of #363's state (a member's + gitlink recorded only at `Members/

`), git dies `No url found for + submodule path` (exit 128), and it still ended ok. It fails now, saying + which members no credential would check out and why, and names + `validate-family.py`. A missing credential is still the skip it was. +- (b) In a plain clone of #363's state followed by `git submodule update + --init --recursive`, as CI and `make bootstrap` run it, the update dies at + `Members/

` before it checks out any member, so where that row is not + the first, the gate refused `member-uninitialized` for an earlier row, + naming that same update, and never reached the variant's refusal. Every + row's gitlink is asked before any row's checkout now. +- (c) Under `--no-members`, the mode CI falls back to there, + `member-gitlink-unnamed` for that gitlink printed no command. It prints + the index move members mode prints now. + +THE SAME FAMILY, OFFLINE RULES AND STATES as +`test_family_head_lost_gitmodules.py` (#611), whose helpers are imported. +Every remedy is typed as printed. The case-variant states are index records +and commits, never two directories whose names differ only in case, and +the holder's own are asked again with `core.ignorecase=true`. A clone's +checkout is git's: on a disk that folds case, `Members/

` IS the row's +directory, so the `mv` the text gives for a case-sensitive disk is typed +only where the disk holds the two apart, as the text says. +""" + +from __future__ import annotations + +import itertools +import os +import re +import shlex +import subprocess +from pathlib import Path + +import pytest + +from conftest import clone_just_pushed, git, run_script +from repo_shape import PYTHON +from test_family import NAME, family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( + DISKS, assert_passes_in_every_mode, commit_all, head_lost, on_disk, + pinned, recorded_at, variant_refused, +) +from test_family_member_path import MEMBER, OTHER, raw +from test_validate_family_unnamed_mounts import ( + GITLINK_SAID, VARIANT, WRONG_CASE, commands, findings, stage_gitlink, +) + +#: The line `(a)` prints for the members it left NOT CHECKED OUT where no +#: credential would check them out, and the one the run then ends on. +NOT_FETCHED = (" NOT CHECKED OUT, and not for want of a credential: {left}. " + f"`{PYTHON} scripts/validate-family.py` names the cause and " + "the commit that cures it.") +NO_SECTION = "no section of .gitmodules gives that path" +NO_GITLINK = "no gitlink is recorded at that path" +EXITED_0 = "git exited 0 and checked nothing out there" + +#: Each test pushes a branch of its own into the module's family remote. Not +#: `tmp_path.name`: a passed test's directory is removed under +#: `tmp_path_retention_policy=failed`, and the next test is given its number. +BRANCHES = itertools.count() + +#: What members mode's variant refusal says before the move it prints. +WRONG_CASE_LEAD = "Remediation: the member is recorded in the wrong case. " + +#: `bootstrap.py --skip-members`'s stdout after its first line, in a clone +#: where every member was checked out, and where not every one was: the +#: second says nothing of (a)'s fetch on stdout, and ends on stderr. +FETCHED = ["", "(a) members and their legs", " credential source: none", + " every member and leg fetched", "", + "(b) each member's own `make bootstrap`", + " skipped (--skip-members)", "", "family bootstrap ok", ""] +NOT_ALL_FETCHED = ["", "(a) members and their legs", + " credential source: none", "", + "(b) each member's own `make bootstrap`", + " skipped (--skip-members)", "", ""] + + +def with_config(env: dict, key: str, value: str) -> dict: + """`env` with `key` set to `value`, one more `GIT_CONFIG_*` entry after + the ones it already holds, for every git the run starts.""" + index = int(env.get("GIT_CONFIG_COUNT", "0")) + return {**env, "GIT_CONFIG_COUNT": str(index + 1), + f"GIT_CONFIG_KEY_{index}": key, f"GIT_CONFIG_VALUE_{index}": value} + + +def fetching(env: dict) -> dict: + """`env` under which `git submodule update` may clone the fixture's + local members: `protocol.file.allow`, as `test_family.py` gives the + holder's own bootstrap.""" + return with_config(env, "protocol.file.allow", "always") + + +def variant(holder: Path, env: dict, project: str = OTHER) -> str: + """#363's state: `project`'s gitlink recorded only at `Members/

`, + committed, its row and section still `members/

`. OTHER is the + second row, so a plain clone's update dies before the first row's + checkout. Returns the commit.""" + return recorded_at(holder, env, project, f"Members/{project}") + + +def cloned(holder: Path, env: dict, family, tmp_path: Path, + *, recurse: bool) -> Path: + """HEAD pushed to a branch of the family's own remote that no other + test pushes (`BRANCHES`), then a fresh clone of it, recursive or plain, + which must exit 0.""" + remote = family["remotes"] / f"{NAME}.git" + branch = f"w648/{next(BRANCHES)}" + raw(holder, env, "push", "-q", str(remote), f"HEAD:refs/heads/{branch}") + clone = tmp_path / "clone" + made = clone_just_pushed(remote, clone, branch, recurse=recurse) + assert made.returncode == 0, made.stdout + made.stderr + return clone + + +def bootstrap(root: Path, env: dict, *extra: str): + return run_script(root / "scripts" / "bootstrap.py", *extra, cwd=root, + env=env) + + +def not_fetched(*left: tuple[str, str]) -> str: + """`NOT_FETCHED` naming each `(path, why)`.""" + return NOT_FETCHED.format( + left=", ".join(f"{path} ({why})" for path, why in left)) + + +def failed_on(*projects: str) -> str: + return "family bootstrap FAILED: " + ", ".join( + f"{project} (not checked out)" for project in projects) + + +def after_first_line(result) -> list[str]: + """Bootstrap's stdout after its first line, which names the holder and + its two members.""" + lines = result.stdout.split("\n") + assert lines[0].startswith(f"bootstrap: {NAME} ("), result.stdout + assert lines[0].endswith(" 2 member(s)"), result.stdout + return lines[1:] + + +def other_older(family) -> str: + """A commit of OTHER that is not the one the holder pins.""" + return git("rev-parse", "main~1", + cwd=family["remotes"] / f"{OTHER}.git").stdout.strip() + + +def type_line(root: Path, env: dict, line: str) -> None: + """One printed `git` line, typed as printed: split as a POSIX shell + splits it, run with no shell.""" + argv = shlex.split(line) + assert argv[0] == "git", line + raw(root, env, *argv[1:]) + + +def moved_out_where_apart(root: Path, remediation: str) -> None: + """Each backticked `mv` the text gives for a case-sensitive disk, typed + as printed where the disk holds its source apart from the row's own + directory, as the text says; on a disk that folds case it is the row's + directory, and the text says to leave it.""" + for move in re.findall(r"`(mv -- [^`]*)`", remediation): + argv = shlex.split(move) + source = root / argv[2] + own = root / "members" / argv[2].split("/", 1)[1] + if not source.exists() or (own.exists() + and os.path.samefile(source, own)): + continue + subprocess.run(argv, cwd=str(root), check=True) + assert not source.exists() + + +# --- (a) bootstrap ----------------------------------------------------------- + +#: `(set-up, flow, the members (a) names with why, the rows it fails on)`. +UNFETCHED = { + "head-lost-recursive": ( + head_lost, True, + ((f"members/{MEMBER}", NO_SECTION), (f"members/{OTHER}", NO_SECTION)), + (MEMBER, OTHER)), + "head-lost-plain": ( + head_lost, False, + ((f"members/{MEMBER}", NO_SECTION), (f"members/{OTHER}", NO_SECTION)), + (MEMBER, OTHER)), + "variant-recursive": ( + variant, True, ((f"members/{OTHER}", NO_GITLINK),), (OTHER,)), + "variant-plain": ( + variant, False, ((f"members/{OTHER}", NO_GITLINK),), (OTHER,)), +} + + +@pytest.mark.parametrize("set_up, recurse, left, projects", + UNFETCHED.values(), ids=UNFETCHED) +def test_bootstrap_fails_over_members_no_credential_would_check_out( + family, holder, offline, tmp_path, set_up, recurse, left, projects): + """THE ISSUE'S (a), both of rv611b's states, after a recursive clone and + after a plain one. Git exits 0 over a holder whose HEAD lost + `.gitmodules` where the clone was recursive, and 128 (`No url found for + submodule path`) where it was plain, and 128 over #363's state in a + plain clone: the run said ok each time. It names the members no + credential would check out, says why, names `validate-family.py`, and + fails; it never says every member was fetched.""" + env = isolated(offline, tmp_path) + set_up(holder, env) + clone = cloned(holder, env, family, tmp_path, + recurse=recurse) + + result = bootstrap(clone, fetching(env), "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert after_first_line(result) == NOT_ALL_FETCHED + errors = result.stderr.splitlines() + assert errors[-2:] == [not_fetched(*left), failed_on(*projects)], ( + result.stderr) + if not recurse: + assert errors[0].startswith(" FETCH INCOMPLETE (git exit 128) "), ( + result.stderr) + assert "No url found for submodule path" in result.stderr + + +def test_a_good_holder_bootstraps_as_it_did(family, holder, offline, + tmp_path): + """A GOOD HOLDER'S RUN IS KEPT: a fresh recursive clone checks every + member out, (a) says every member and leg was fetched, and the run ends + ok with nothing on stderr.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, recurse=True) + + result = bootstrap(clone, fetching(env), "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert after_first_line(result) == FETCHED + assert result.stderr == "" + + +def test_a_missing_credential_is_still_the_skip_it_was(family, holder, + offline, tmp_path): + """No misfire: a plain clone whose update git refuses for want of a + credential (no `protocol.file.allow` for the local members, as a + private member with no token is refused) records every gitlink at its + row's path, and `.gitmodules` gives each. A credential is the cure, so + the members are skipped, the run ends ok, and no line says otherwise.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, + recurse=False) + + result = bootstrap(clone, env) + + assert result.returncode == 0, result.stdout + result.stderr + assert "FETCH INCOMPLETE" in result.stderr + assert "not for want of a credential" not in result.stderr + for project in (MEMBER, OTHER): + assert (f" [{project}] {project}: NOT CHECKED OUT; skipped. See the " + "fetch above.") in result.stdout.splitlines() + assert result.stdout.endswith("\nfamily bootstrap ok\n"), result.stdout + + +def test_an_update_that_exits_0_and_skips_a_member_fails(family, holder, + offline, tmp_path): + """Git exited 0, so it lacked nothing a credential gives, though the + holder's records give OTHER: here `submodule..update=none` set + for the run skips it. (a) does not say every member was fetched, and + the run fails on OTHER alone.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, + recurse=False) + skip = with_config(fetching(env), f"submodule.members/{OTHER}.update", + "none") + + result = bootstrap(clone, skip, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert (clone / "members" / MEMBER / "project.yaml").is_file() + assert "every member and leg fetched" not in result.stdout + assert result.stderr.splitlines() == [ + not_fetched((f"members/{OTHER}", EXITED_0)), failed_on(OTHER)] + + +def test_a_row_written_twice_is_named_once(family, holder, offline, + tmp_path): + """OTHER's row written twice in a holder whose HEAD lost `.gitmodules`: + one path, so one name in (a) and one failure.""" + env = isolated(offline, tmp_path) + head_lost(holder, env) + rows = (holder / "family.yaml").read_bytes() + start = rows.index(f" - project: {OTHER}\n".encode()) + (holder / "family.yaml").write_bytes(rows + rows[start:]) + raw(holder, env, "add", "--", "family.yaml") + commit_all(holder, env, "OTHER's row written twice") + clone = cloned(holder, env, family, tmp_path, recurse=True) + + result = bootstrap(clone, fetching(env), "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines() == [ + not_fetched((f"members/{MEMBER}", NO_SECTION), + (f"members/{OTHER}", NO_SECTION)), + failed_on(MEMBER, OTHER)] + + +# --- (a) and (b) together: the plain clone of #363's state, typed through --- + +def test_a_plain_clone_of_a_case_variant_is_named_and_mended_as_printed( + family, holder, offline, tmp_path): + """The round trip a person makes. In a plain clone of #363's state, with + the variant at the SECOND row, `make bootstrap`'s script fails naming + OTHER and the validator; the validator, members checked, names the + variant and the move, where it refused `member-uninitialized` for the + first row before. Typed as printed in the clone, with the commit and + the `mv` where the disk holds the two apart, the same script checks + both members out and runs each one's own bootstrap, and the gate passes + in every mode.""" + env = fetching(isolated(offline, tmp_path)) + commit = variant(holder, env) + clone = cloned(holder, env, family, tmp_path, recurse=False) + + booted = bootstrap(clone, env) + + assert booted.returncode == 1, booted.stdout + booted.stderr + assert booted.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{OTHER}", NO_GITLINK)), failed_on(OTHER)] + gate = validate(clone) + assert gate.returncode == 2, gate.stdout + gate.stderr + lines = gate.stderr.splitlines() + assert lines == variant_refused(f"members/{OTHER}", commit, + [f"Members/{OTHER}"], + {f"Members/{OTHER}": commit}) + for line in lines[2:]: + type_line(clone, env, line) + commit_all(clone, env, "Move the gitlink as printed") + moved_out_where_apart(clone, lines[1]) + + again = bootstrap(clone, env) + + assert again.returncode == 0, again.stdout + again.stderr + assert " every member and leg fetched" in again.stdout.splitlines() + for project in (MEMBER, OTHER): + assert f" --- {project}: make bootstrap ---" in again.stdout + assert again.stdout.endswith("\nfamily bootstrap ok\n"), again.stdout + assert_passes_in_every_mode(clone, 2) + + +# --- (b) the gate asks every row's gitlink before any row's checkout -------- + +@pytest.mark.parametrize("project", [MEMBER, OTHER], ids=["first", "second"]) +def test_a_plain_clone_after_the_update_names_the_variant_in_any_row( + family, holder, offline, tmp_path, project): + """THE ISSUE'S (b), as CI and `make bootstrap` run it: a plain clone, + then `git submodule update --init --recursive`, which dies at + `Members/

` (exit 128) before it checks out either member. Members + checked and `--pins` alike, the gate refuses the variant with the move, + whichever row it is; for the second it refused `member-uninitialized` + for the first row, naming that same update. Under `--no-members`, CI's + fallback, the finding prints the same two lines (c); typed as printed in + the clone and committed, the update checks both out and the gate + passes.""" + env = fetching(isolated(offline, tmp_path)) + commit = variant(holder, env, project) + clone = cloned(holder, env, family, tmp_path, recurse=False) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode == 128, update.stderr + assert (f"No url found for submodule path 'Members/{project}'".encode() + in update.stderr) + expected = variant_refused(f"members/{project}", commit, + [f"Members/{project}"], + {f"Members/{project}": commit}) + + for extra in ((), ("--pins",)): + gate = validate(clone, *extra) + assert gate.returncode == 2, (extra, gate.stdout + gate.stderr) + assert gate.stderr.splitlines() == expected, extra + fallback = validate(clone, "--no-members") + assert fallback.returncode == 1, fallback.stdout + fallback.stderr + (finding,) = findings(fallback) + assert commands(finding) == [line.strip() for line in expected[2:]] + + for line in commands(finding): + type_line(clone, env, line) + commit_all(clone, env, "Move the gitlink as CI's gate printed") + moved_out_where_apart(clone, finding) + raw(clone, env, "submodule", "update", "--init", "--recursive") + assert_passes_in_every_mode(clone, 2) + + +# --- (c) `--no-members` prints the move members mode prints ----------------- + +def unnamed_moved(variant_path: str, at: str, pin: str, + variants: list[str]) -> str: + """`member-gitlink-unnamed` WHOLE for the gitlink at `variant_path` (at + `at`), OTHER recorded only at `variants`: what every unnamed gitlink + says, the row it is a variant of, and members mode's move to the pin, + word for word.""" + path = f"members/{OTHER}" + refused = variant_refused(path, pin, variants, + dict.fromkeys(variants, at)) + assert refused[1].startswith(WRONG_CASE_LEAD + "Move its gitlink "), ( + "premise: members mode's remedy text") + return "\n".join([ + GITLINK_SAID.format(path=variant_path, commit=at) + + VARIANT.format(row_path=path, project=OTHER) + + " The member is recorded in the wrong case. " + + refused[1][len(WRONG_CASE_LEAD):], *refused[2:]]) + + +@DISKS +@pytest.mark.parametrize("spellings", [ + ("Members/{p}",), ("MEMBERS/{p}", "Members/{p}")], ids=["one", "two"]) +def test_no_members_prints_the_move_members_mode_prints( + family, holder, offline, tmp_path, ignorecase, spellings): + """THE ISSUE'S (c). OTHER's only gitlink at a case variant (two, in the + second case), at a commit that is not its pin. Under `--no-members` each + variant's `member-gitlink-unnamed` printed no command; it prints the + move members mode prints now, to the commit the row PINS, every variant + taken out on one line. Typed as printed and committed, on either disk, + the holder passes in every mode.""" + env = on_disk(isolated(offline, tmp_path), ignorecase) + variants = sorted(spelling.format(p=OTHER) for spelling in spellings) + at = other_older(family) + pin = pinned(holder, OTHER) + assert pin != at, "premise: the variant is not at the pin" + raw(holder, env, "rm", "-q", "--cached", "--", f"members/{OTHER}") + for spelling in variants: + stage_gitlink(holder, spelling, at) + commit_all(holder, env, f"{OTHER} recorded in another case") + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == [unnamed_moved(path, at, pin, variants) + for path in variants] + for line in commands(findings(result)[0]): + type_line(holder, env, line) + commit_all(holder, env, "Move the gitlink as printed") + assert_passes_in_every_mode(holder, 2) + + +def test_a_row_whose_pin_is_no_commit_still_prints_no_command( + holder, offline, tmp_path): + """The move records the gitlink at the commit the row pins, so a row + whose `pin.commit` is no 40-hex commit (members checked, the refusal + `member-pin-commit`) gets the text it had, and no command.""" + env = isolated(offline, tmp_path) + commit = variant(holder, env) + rows = (holder / "family.yaml").read_bytes() + line = f' commit: "{commit}"\n'.encode() + assert rows.count(line) == 1, "fixture drift: OTHER's pin line" + (holder / "family.yaml").write_bytes( + rows.replace(line, f' commit: "{commit[:12]}"\n'.encode())) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + (finding,) = findings(result) + assert finding == GITLINK_SAID.format( + path=f"Members/{OTHER}", commit=commit) + WRONG_CASE.format( + row_path=f"members/{OTHER}", project=OTHER, what="gitlink") + assert "git rm" not in result.stderr, result.stderr diff --git a/tests/test_validate_family_unnamed_mounts.py b/tests/test_validate_family_unnamed_mounts.py index 3144f710..a9ab815c 100644 --- a/tests/test_validate_family_unnamed_mounts.py +++ b/tests/test_validate_family_unnamed_mounts.py @@ -28,7 +28,9 @@ the wrong case: the same command, run in either, takes the member's own record out of the index, after which `--no-members`, the run a private family's CI falls back to, says `family ok` over a member that is no longer -there. Those two print no command. +there. Those two print no such command: the first none, and the second, +since #648, the index move members mode prints, which records the member +at its row's path as it takes the variant out. When a hand mount's gitlink says to `git rm` it, its section finding says that the `git rm` takes the section out too and prints no `--unset-all` line: run @@ -512,14 +514,13 @@ def test_a_row_hand_edited_into_a_case_variant_of_its_mount_is_the_edit( @MODES -def test_a_member_recorded_only_in_another_case_says_so_and_prints_no_command( +def test_a_member_recorded_only_in_another_case_says_so_and_prints_the_move( holder, extra): """STATE (ii), REVIEW SHOULD-1: IRSS's row is canonical, and the only gitlink the holder records for it is at `members/irss`. `git rm --cached` of that took the member's only record out, and `--no-members` then said `family ok` over a holder that records one gitlink fewer than it has rows. - It says the member is recorded in the wrong case instead, and sends the - operator to git's history, as every other unnamed gitlink does. + It says the member is recorded in the wrong case instead. Members checked, the row's own gitlink is read at its exact path, finds none and is the refusal `member-gitlink-absent`, which prints no finding, @@ -528,26 +529,31 @@ def test_a_member_recorded_only_in_another_case_says_so_and_prints_no_command( there. What no run may do is print a command that takes the gitlink out ALONE: since #611 the refusal names the variant and prints the move that takes it out of the index and records it at the row's path together - (`tests/test_family_head_lost_gitmodules.py` types it).""" + (`tests/test_family_head_lost_gitmodules.py` types it), and since #648 + the finding prints the same two lines + (`tests/test_family_bootstrap_gate_order.py` types them).""" named = record_in_another_case(holder, "IRSS", "members/irss") result = validate(holder, *extra) assert "--unset-all" not in result.stderr + result.stdout + move = [ + " git rm -r --cached --ignore-unmatch -- members/irss", + f" git update-index --add --cacheinfo 160000,{named},members/IRSS"] if result.returncode == 2: assert extra != ("--no-members",), result.stderr assert result.stderr.startswith( "REFUSED member-gitlink-absent: no gitlink (mode 160000) " "recorded at 'members/IRSS'."), result.stderr - assert result.stderr.splitlines()[-2:] == [ - " git rm -r --cached --ignore-unmatch -- members/irss", - " git update-index --add --cacheinfo " - f"160000,{named},members/IRSS"], result.stderr + assert result.stderr.splitlines()[-2:] == move, result.stderr return - assert "git rm" not in result.stderr + result.stdout assert result.returncode == 1, result.stderr + result.stdout (gitlink,) = findings(result) - assert gitlink == GITLINK_SAID.format( - path="members/irss", commit=named) + WRONG_CASE.format( - row_path="members/IRSS", project="IRSS", what="gitlink") + said = GITLINK_SAID.format(path="members/irss", commit=named) + ( + VARIANT.format(row_path="members/IRSS", project="IRSS")) + assert gitlink.startswith( + said + " The member is recorded in the wrong case. Move its gitlink " + "to the row's path, at the commit the row pins, in the index " + "alone, "), gitlink + assert gitlink.splitlines()[1:] == move, gitlink @MODES From 1a69505d8814ae0cc37e0c4281353c5caaedefaf Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 18:57:59 +0000 Subject: [PATCH 05/20] bootstrap.py: a plain clone's update that dies at a gitlink no section gives a url fails the run naming every row, and a member git exited 0 over is given the update that checks it out, so a stray case variant, a hand mount or a url-less section no longer ends `family bootstrap ok` (#648, rv648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fix round for rv648's review of 8b39cd19 (1 MUST, 2 SHOULD, 5 NIT). M1. In a plain clone, `git submodule update --init` dies in its init step at the first gitlink, in index order, that no section of `.gitmodules` gives a url, and checks out no member. `unfetchable` asked only each row's own gitlink and section, so a stray `Members/

` beside `members/

` (#361's STRAY), a member mounted by hand with no section, or a row's section that lost its url (rv648's S1, H1 and U1) passed every row as the credential skip, and the run ended ok. `init_stop` now asks the holder's records which gitlink git's init stops at: the index's gitlinks in order, each against the work tree's sections (`gitmodules_sections`, which replaces `gitmodules_paths`) and the git config. Measured before it was taken, rv648's prototype failed three states that are a missing credential: a recursive clone of the stray or of the hand mount with no credential, and a url-less section an earlier `git submodule init` registered. So two of git's rules are kept: where the config sets `submodule.active` (a recursive clone sets `.`), init asks only the gitlinks a section gives, and a section with no url does not stop it where the config holds a url under its name. Where git exited non-zero and init stops at a gitlink, every row not checked out is named `git stopped at , a gitlink no section of .gitmodules gives a url`, and the run fails. What cures it: `validate-family.py --no-members` names a gitlink no row names and the commit that cures it (members checked, it refuses `member-uninitialized` for the first row in these states, a checklist candidate). Where a row names the gitlink, nothing in the validator does, so the line says to write that section back as `family.py add` writes one. Each why now carries its own sentence. S1. For "git exited 0 and checked nothing out there" (a subset clone, an `update = none`), the line printed the validator sentence, whose `member-uninitialized` remedy exits 0 and checks nothing out. It prints `git submodule update --init --checkout --recursive -- ` instead; typed, it checks the member out in both states and the run ends ok. S2. `test_a_missing_credential_is_still_the_skip_it_was` asserts that " every member and leg fetched" is not printed (rv648's mutant R2). NITs. README and the handbook say `make bootstrap` skips a member it lacks a credential for, and the § Families marker is the new segment's blob, b3d19092; README stays at 1459 lines, its cap. The docstrings say "an empty mapping" and "a missing credential exits non-zero too", and `validate-family.py`'s WHAT IS CHECKED says the move under `--no-members` is printed only for a canonical path and a 40-hex pin. Tests: the variant-plain case and the round trip name MEMBER too; new tests for S1 and H1 (typed through `--no-members` and committed, the run checks both out), U1 (the section written back), the two recursive clones with no credential and the registered url (still the skip), and the update that exits 0, by `update = none` and by a subset clone, with its printed line typed. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/handbook.html | 6 +- templates/family-root/scripts/bootstrap.py | 181 ++++++++-- .../family-root/scripts/validate-family.py | 7 +- tests/test_family_bootstrap_gate_order.py | 310 +++++++++++++++--- 5 files changed, 424 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 0f17b2b8..4a4e78fa 100644 --- a/README.md +++ b/README.md @@ -952,7 +952,7 @@ at the right commit. **A family pins assembly roots, never legs**: the GitHub App or `SHAPE_LEGS_TOKEN` credential the workflow does, as a member is a private assembly root mounting two private legs — then runs each member's own `make bootstrap`; `make validate` runs the family validator, then each -member's. One it lacks a credential for is reported and skipped, not a failure. +member's. `make bootstrap` reports and skips one it lacks a credential for. The holder carries the same COPY pin an assembly root does, so `update-shape.py` re-syncs it the same way; it mirrors into `family.yaml` diff --git a/docs/handbook.html b/docs/handbook.html index f2e382e2..0c63875c 100644 --- a/docs/handbook.html +++ b/docs/handbook.html @@ -1282,7 +1282,7 @@

Adopting an existing repository

- +

Reference

Families: a holder for projects that ship separately

@@ -1375,8 +1375,8 @@

The InkRouter example

resolving the same GitHub App or SHAPE_LEGS_TOKEN credential the workflow does — and then runs each member’s own make bootstrap. make validate runs the family validator and then each member’s own - make validate. A member it lacks a credential for is reported and skipped, not - a failure. The holder carries the same COPY pin an assembly root does, so + make validate. A member make bootstrap lacks a credential for is + reported and skipped, not a failure. The holder carries the same COPY pin an assembly root does, so update-shape.py re-syncs it the same way; it mirrors into family.yaml instead of project.yaml and must leave validate-family.py green.

diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index e3fb3b10..4e089c99 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -31,13 +31,22 @@ every row's member is checked out (#648). A member left NOT CHECKED OUT where no credential would check it out is said here instead, with why, and fails the run (c): this holder records no gitlink at the - row's path (a gitlink recorded only in another case, #363), no + row's path (a gitlink recorded only in another case, #363); no section of `.gitmodules` gives that path (a holder whose HEAD lost the - file, #611), or git exited 0 and checked nothing out there. Git exits - 0 over the second where a recursive clone ran first, and 128 (`No url - found for submodule path`) in a plain clone, as it does for a missing - credential, so what the holder records is the test, never git's - words. The line names `validate-family.py`, which says what to commit. + file, #611); git exited 0 and checked nothing out there; or git + stopped at a gitlink no section gives a url, the row's own or another + (a stray case variant, a member mounted by hand, a section that lost + its url), since git's init dies at the first such gitlink it asks + about, before it checks out any member. Git exits 0 over the second + where a recursive clone ran first, and 128 (`No url found for + submodule path`) in a plain clone; a missing credential exits + non-zero too, so what the holder records is the test, never git's + words. The line says what cures each: `validate-family.py` names the + commit for the first two; for the third it prints the `git submodule + update --init --checkout` that checks the member out whatever the + clone chose; and for the last, `validate-family.py --no-members` + names a gitlink no row names, and where the gitlink is a row's own + the line says to write its section back. (b) RUNS EACH MEMBER'S OWN `make bootstrap`. Each member is a whole project and knows how to bootstrap itself — its legs onto their tracking @@ -60,6 +69,7 @@ import argparse import os +import shlex import shutil import subprocess import sys @@ -74,6 +84,17 @@ MANIFEST = "family.yaml" GITMODULES = ".gitmodules" DEFAULT_MEMBERS_DIR = "members" +#: A submodule's entry in the index, as `ls-files -s` spells its mode. +GITLINK_MODE = b"160000" + +#: Why (a) left a row's member NOT CHECKED OUT where no credential would +#: check it out (#648), in the order `unfetchable` asks. `STOPPED` is +#: filled with the gitlink git's init died at (`init_stop`). +NO_GITLINK = "no gitlink is recorded at that path" +NO_SECTION = f"no section of {GITMODULES} gives that path" +EXITED_0 = "git exited 0 and checked nothing out there" +STOPPED = ("git stopped at {}, a gitlink no section of " + GITMODULES + + " gives a url") #: The two environment variables a leg/member credential may arrive in, in the #: order the `validate` workflow resolves them. An App-minted installation @@ -124,71 +145,159 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: "fine-grained PAT with contents:read on them) or export an " "App-minted SHAPE_LEGS_APP_TOKEN, then re-run. What did arrive " "is bootstrapped below.", file=sys.stderr) - left = unfetchable(root, rows, proc.returncode == 0) + left, stopper = unfetchable(root, rows, proc.returncode == 0) if left: print(" NOT CHECKED OUT, and not for want of a credential: " - + ", ".join(f"{spelled(path)} ({why})" for _, path, why in left) - + f". `{PYTHON} scripts/validate-family.py` names the cause and " - "the commit that cures it.", file=sys.stderr) + + ", ".join(f"{spelled(path)} (" + + (STOPPED.format(spelled(stopper)) + if why == STOPPED else why) + ")" + for _, path, why in left) + + ". " + cures(left, stopper, rows), file=sys.stderr) elif proc.returncode == 0: print(" every member and leg fetched") return [f"{spelled(row.get('project') or '?')} (not checked out)" for row, _, _ in left] -def gitmodules_paths(root: Path) -> set[str]: - """Every path a section of `.gitmodules` gives, read as git reads it: - the work tree's file, the one `git submodule update` reads, which a - fresh clone takes from HEAD. None where there is no such file, or one - git cannot read: git has no url for any submodule then either.""" +def cures(left: list[tuple[dict, str, str]], stopper: str | None, + rows: list[dict]) -> str: + """What (a)'s line says cures the rows `left` names, one sentence for + each why among them (#648). `validate-family.py`, which names the + commit, for a gitlink or a section the holder lacks at the row's path. + For the gitlink git stopped at, `validate-family.py --no-members`, which + names a gitlink no row names and the commit that cures it (members + checked, it refuses `member-uninitialized` for the first row there + instead), or, where a row names it, its section written back. For a row + git exited 0 over, the update that checks it out whatever the clone + chose: `--init` with the path registers a member a subset clone left + out, and `--checkout` overrides an `update = none`.""" + whys = {why for _, _, why in left} + said = [] + if whys & {NO_GITLINK, NO_SECTION}: + said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " + "and the commit that cures it.") + if STOPPED in whys and stopper in {row_path(row) for row in rows}: + said.append(f"Write the section of {GITMODULES} for " + f"{spelled(stopper)} back as `family.py add` writes one, " + f"with its path and url, and commit {GITMODULES}.") + elif STOPPED in whys: + said.append(f"`{PYTHON} scripts/validate-family.py --no-members` " + f"names each gitlink in {DEFAULT_MEMBERS_DIR}/ that no " + "row names, and the commit that cures it.") + exited = [path for _, path, why in left if why == EXITED_0] + if exited: + update = shlex.join(["git", "submodule", "update", "--init", + "--checkout", "--recursive", "--", *exited]) + said.append(f"`{spelled(update)}` checks " + f"{'it' if len(exited) == 1 else 'them'} out.") + return " ".join(said) + + +def gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: + """`{name: {key: value}}` of every section of `.gitmodules`, read as git + reads it: the work tree's file, the one `git submodule update` reads, + which a fresh clone takes from HEAD. An empty mapping where there is no + such file, or one git cannot read: git has no url for any submodule + then either.""" if not os.path.lexists(root / GITMODULES): - return set() + return {} try: listed = git_out(["config", "-z", "--file", GITMODULES, "--list"], cwd=root, binary=True) except Refusal: - return set() - paths: set[str] = set() + return {} + sections: dict[str, dict[str, str]] = {} for record in listed.split(b"\0"): key, _, value = record.decode("utf-8", "surrogateescape").partition("\n") - if key.startswith("submodule.") and key.endswith(".path"): - paths.add(value) - return paths + section, _, variable = key.rpartition(".") + if section.startswith("submodule."): + sections.setdefault(section[len("submodule."):], + {})[variable] = value + return sections + + +def init_stop(root: Path, sections: dict[str, dict[str, str]]) -> str | None: + """The gitlink `git submodule update --init` dies at, or None: the first + in the index, in index order, that its init step asks about and finds + no url for (`No url found for submodule path`, exit 128), before it + checks out any member (#648). + + ASKED AS GIT 2.43 ASKS IT, of the index, `sections` and this + repository's git config. Every gitlink the index records is asked, but + where the config sets `submodule.active`, as a recursive clone sets it + (`.`), only those a section gives a path: git counts no other active. + One no section gives stops it; so does one whose section gives no url, + unless the config has one under the section's name, as an earlier init + writes it. So a stray case variant or a member mounted by hand stops a + plain clone's update, and leaves a recursive clone's to the credential. + Read nothing, it answers None.""" + try: + index = git_out(["ls-files", "-s", "-z"], cwd=root, binary=True) + except Refusal: + return None + try: + config = git_out(["config", "-z", "--get-regexp", r"^submodule\."], + cwd=root, binary=True) + except Refusal: + config = b"" + keys = {record.decode("utf-8", "surrogateescape").partition("\n")[0] + for record in config.split(b"\0")} + names: dict[str, list[str]] = {} + for name, held in sections.items(): + if "path" in held: + names.setdefault(held["path"], []).append(name) + for record in index.split(b"\0"): + meta, _, entry = record.partition(b"\t") + if meta.split()[:1] != [GITLINK_MODE]: + continue + path = entry.decode("utf-8", "surrogateescape") + given = names.get(path, []) + if not given and "submodule.active" in keys: + continue + if not any("url" in sections[name] or f"submodule.{name}.url" in keys + for name in given): + return path + return None -def unfetchable(root: Path, rows: list[dict], - exited_0: bool) -> list[tuple[dict, str, str]]: +def unfetchable(root: Path, rows: list[dict], exited_0: bool + ) -> tuple[list[tuple[dict, str, str]], str | None]: """`(row, path, why)` for each row whose member (a) left NOT CHECKED OUT - where no credential would check it out (#648), in row order. + where no credential would check it out (#648), in row order, and the + gitlink git's init died at (`init_stop`), or None. Asked of this holder's own records, never of git's words: a row at whose path the holder records no gitlink (`recorded_gitlink`, as `validate-family.py`'s `member-gitlink-absent` reads one), or whose gitlink no section of `.gitmodules` gives a path (as its `member-gitmodules-absent` reads HEAD's file, which is the work tree's - in a clone); and, where git exited 0, any row at all, since git then - lacked nothing a credential gives. Any other row not checked out is the - missing credential's, and stays the skip it always was. A path two rows - give is said once.""" + in a clone); where git exited 0, any row at all, since git then lacked + nothing a credential gives; and where git's init died at a gitlink no + section gives a url, any row at all, since git then checked out no + member and asked for no credential. Any other row not checked out is + the missing credential's, and stays the skip it always was. A path two + rows give is said once.""" left: dict[str, dict] = {} for row in rows: if not (member_path(root, row) / ".git").exists(): left.setdefault(row_path(row), row) if not left: - return [] - given = gitmodules_paths(root) + return [], None + sections = gitmodules_sections(root) + given = {held["path"] for held in sections.values() if "path" in held} + stopper = None if exited_0 else init_stop(root, sections) found: list[tuple[dict, str, str]] = [] for path, row in left.items(): if recorded_gitlink(root, path) is None: - found.append((row, path, "no gitlink is recorded at that path")) + found.append((row, path, NO_GITLINK)) elif path not in given: - found.append((row, path, f"no section of {GITMODULES} gives that " - "path")) + found.append((row, path, NO_SECTION)) elif exited_0: - found.append((row, path, "git exited 0 and checked nothing out " - "there")) - return found + found.append((row, path, EXITED_0)) + elif stopper is not None: + found.append((row, path, STOPPED)) + return found, stopper def members_of(root: Path) -> tuple[str, list[dict]]: diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 1e593d20..cee36ce5 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -64,9 +64,10 @@ asks them too. A path a row names in another case is still one of these, and what it says to do depends on which of the two is the member's own: it names no command that would un-register the member, and for a member - recorded only in the wrong case it names the index move that records it - at its row's path, as the row's own refusal does members checked - (#648). + recorded only in the wrong case, where its row's path is its canonical + `members/` and its pin a 40-hex commit, it names the index move + that records it at that path, as the row's own refusal does members + checked (#648). - that a section of HEAD's `.gitmodules` gives the path of every row's gitlink, the `member-gitmodules-absent` finding (#611, #407's decision 3). A recursive clone reads HEAD's file, so a gitlink no section of it diff --git a/tests/test_family_bootstrap_gate_order.py b/tests/test_family_bootstrap_gate_order.py index 9962f731..bbe04750 100644 --- a/tests/test_family_bootstrap_gate_order.py +++ b/tests/test_family_bootstrap_gate_order.py @@ -11,8 +11,13 @@ ok. In a plain clone of that holder, or of #363's state (a member's gitlink recorded only at `Members/

`), git dies `No url found for submodule path` (exit 128), and it still ended ok. It fails now, saying - which members no credential would check out and why, and names - `validate-family.py`. A missing credential is still the skip it was. + which members no credential would check out and why, and what cures + them. So it does where git's init dies at a gitlink no section gives a + url, a stray case variant, a member mounted by hand or a section that + lost its url (rv648's S1, H1 and U1), and where git exits 0 over a member + a subset clone or `update = none` leaves out, whose line prints the + update that checks it out. A missing credential is still the skip it + was, after a recursive clone too. - (b) In a plain clone of #363's state followed by `git submodule update --init --recursive`, as CI and `make bootstrap` run it, the update dies at `Members/

` before it checks out any member, so where that row is not @@ -26,11 +31,13 @@ THE SAME FAMILY, OFFLINE RULES AND STATES as `test_family_head_lost_gitmodules.py` (#611), whose helpers are imported. Every remedy is typed as printed. The case-variant states are index records -and commits, never two directories whose names differ only in case, and -the holder's own are asked again with `core.ignorecase=true`. A clone's -checkout is git's: on a disk that folds case, `Members/

` IS the row's -directory, so the `mv` the text gives for a case-sensitive disk is typed -only where the disk holds the two apart, as the text says. +and commits, and no test makes two directories whose names differ only in +case; the holder's own are asked again with `core.ignorecase=true`. A +clone's checkout is git's: on a disk that folds case, `Members/

` IS the +row's directory, so the `mv` the text gives for a case-sensitive disk is +typed only where the disk holds the two apart, as the text says, and a +clone of a stray (`Members/

` beside `members/

`, rv648's S1) gets its +two empty mounts there as one directory, which no assertion reads. """ from __future__ import annotations @@ -46,26 +53,45 @@ from conftest import clone_just_pushed, git, run_script from repo_shape import PYTHON -from test_family import NAME, family, validate # noqa: F401 +from test_family import NAME, SPARE, family, validate # noqa: F401 from test_family_add_refusals import holder, offline # noqa: F401 from test_family_dry_run import isolated from test_family_head_lost_gitmodules import ( - DISKS, assert_passes_in_every_mode, commit_all, head_lost, on_disk, - pinned, recorded_at, variant_refused, + DISKS, GITMODULES, assert_passes_in_every_mode, commit_all, + committed_gitmodules, head_lost, on_disk, pinned, recorded_at, + variant_refused, ) from test_family_member_path import MEMBER, OTHER, raw from test_validate_family_unnamed_mounts import ( - GITLINK_SAID, VARIANT, WRONG_CASE, commands, findings, stage_gitlink, + GITLINK_SAID, VARIANT, WRONG_CASE, commands, findings, spare_commits, + stage_gitlink, ) #: The line `(a)` prints for the members it left NOT CHECKED OUT where no -#: credential would check them out, and the one the run then ends on. +#: credential would check them out, each with why, then what cures them; +#: the run then ends on `failed_on`. NOT_FETCHED = (" NOT CHECKED OUT, and not for want of a credential: {left}. " - f"`{PYTHON} scripts/validate-family.py` names the cause and " - "the commit that cures it.") + "{cure}") NO_SECTION = "no section of .gitmodules gives that path" NO_GITLINK = "no gitlink is recorded at that path" EXITED_0 = "git exited 0 and checked nothing out there" +STOPPED = ("git stopped at {path}, a gitlink no section of .gitmodules " + "gives a url") + +#: What the line says cures each why: the validator for the first two; for +#: the gitlink git stopped at, the validator's `--no-members`, or, where a +#: row names that gitlink, its section written back; and for a member git +#: exited 0 over, the update that checks it out. +VALIDATOR_CURE = (f"`{PYTHON} scripts/validate-family.py` names the cause " + "and the commit that cures it.") +UNNAMED_CURE = (f"`{PYTHON} scripts/validate-family.py --no-members` names " + "each gitlink in members/ that no row names, and the commit " + "that cures it.") +SECTION_CURE = ("Write the section of .gitmodules for {path} back as " + "`family.py add` writes one, with its path and url, and " + "commit .gitmodules.") +CHECKOUT_CURE = ("`git submodule update --init --checkout --recursive -- " + "{path}` checks it out.") #: Each test pushes a branch of its own into the module's family remote. Not #: `tmp_path.name`: a passed test's directory is removed under @@ -130,10 +156,43 @@ def bootstrap(root: Path, env: dict, *extra: str): env=env) -def not_fetched(*left: tuple[str, str]) -> str: - """`NOT_FETCHED` naming each `(path, why)`.""" +def not_fetched(*left: tuple[str, str], cure: str = VALIDATOR_CURE) -> str: + """`NOT_FETCHED` naming each `(path, why)`, then `cure`.""" return NOT_FETCHED.format( - left=", ".join(f"{path} ({why})" for path, why in left)) + left=", ".join(f"{path} ({why})" for path, why in left), cure=cure) + + +def stray(holder: Path, env: dict, family) -> str: + """#361's STRAY: OTHER's gitlink recorded at `Members/

` too, at its + pin, beside its own, and committed; no section gives the second. Index + order puts it before every member's. Returns its path.""" + stage_gitlink(holder, f"Members/{OTHER}", pinned(holder, OTHER)) + commit_all(holder, env, "A stray case variant of OTHER's gitlink") + return f"Members/{OTHER}" + + +def hand_mounted(holder: Path, env: dict, family) -> str: + """SPARE's gitlink recorded at `members/` by hand, with no row + and no section, and committed. Returns its path.""" + stage_gitlink(holder, f"members/{SPARE}", spare_commits(family)[1]) + commit_all(holder, env, "A member mounted by hand") + return f"members/{SPARE}" + + +#: The two gitlinks no row names at which git's init dies in a plain clone +#: (rv648's S1 and H1). +UNNAMED_STOPS = {"stray": stray, "hand-mount": hand_mounted} + + +def without_url(data: bytes, project: str) -> bytes: + """`.gitmodules` bytes with the `url` line of `project`'s section taken + out, the rest as `family.py add` wrote it (rv648's U1). A checkout's + copy may end its lines CRLF.""" + section = re.compile(rb'(\[submodule "members/' + project.encode() + + rb'"\]\r?\n(?:\t[^\n]*\n)*?)\turl = [^\n]*\n') + edited, count = section.subn(rb"\1", data) + assert count == 1, data + return edited def failed_on(*projects: str) -> str: @@ -182,34 +241,42 @@ def moved_out_where_apart(root: Path, remediation: str) -> None: # --- (a) bootstrap ----------------------------------------------------------- -#: `(set-up, flow, the members (a) names with why, the rows it fails on)`. +#: `(set-up, flow, the members (a) names with why, the rows it fails on, +#: what it says cures them)`. In #363's plain clone git dies at the variant, +#: `Members/

`, before it checks out MEMBER, so MEMBER is named too, and +#: the variant is a gitlink no row names (rv648's M1). UNFETCHED = { "head-lost-recursive": ( head_lost, True, ((f"members/{MEMBER}", NO_SECTION), (f"members/{OTHER}", NO_SECTION)), - (MEMBER, OTHER)), + (MEMBER, OTHER), VALIDATOR_CURE), "head-lost-plain": ( head_lost, False, ((f"members/{MEMBER}", NO_SECTION), (f"members/{OTHER}", NO_SECTION)), - (MEMBER, OTHER)), + (MEMBER, OTHER), VALIDATOR_CURE), "variant-recursive": ( - variant, True, ((f"members/{OTHER}", NO_GITLINK),), (OTHER,)), + variant, True, ((f"members/{OTHER}", NO_GITLINK),), (OTHER,), + VALIDATOR_CURE), "variant-plain": ( - variant, False, ((f"members/{OTHER}", NO_GITLINK),), (OTHER,)), + variant, False, + ((f"members/{MEMBER}", STOPPED.format(path=f"Members/{OTHER}")), + (f"members/{OTHER}", NO_GITLINK)), + (MEMBER, OTHER), f"{VALIDATOR_CURE} {UNNAMED_CURE}"), } -@pytest.mark.parametrize("set_up, recurse, left, projects", +@pytest.mark.parametrize("set_up, recurse, left, projects, cure", UNFETCHED.values(), ids=UNFETCHED) def test_bootstrap_fails_over_members_no_credential_would_check_out( - family, holder, offline, tmp_path, set_up, recurse, left, projects): + family, holder, offline, tmp_path, set_up, recurse, left, projects, + cure): """THE ISSUE'S (a), both of rv611b's states, after a recursive clone and after a plain one. Git exits 0 over a holder whose HEAD lost `.gitmodules` where the clone was recursive, and 128 (`No url found for submodule path`) where it was plain, and 128 over #363's state in a plain clone: the run said ok each time. It names the members no - credential would check out, says why, names `validate-family.py`, and - fails; it never says every member was fetched.""" + credential would check out, says why and what cures them, and fails; + it never says every member was fetched.""" env = isolated(offline, tmp_path) set_up(holder, env) clone = cloned(holder, env, family, tmp_path, @@ -220,8 +287,8 @@ def test_bootstrap_fails_over_members_no_credential_would_check_out( assert result.returncode == 1, result.stdout + result.stderr assert after_first_line(result) == NOT_ALL_FETCHED errors = result.stderr.splitlines() - assert errors[-2:] == [not_fetched(*left), failed_on(*projects)], ( - result.stderr) + assert errors[-2:] == [not_fetched(*left, cure=cure), + failed_on(*projects)], result.stderr if not recurse: assert errors[0].startswith(" FETCH INCOMPLETE (git exit 128) "), ( result.stderr) @@ -259,31 +326,191 @@ def test_a_missing_credential_is_still_the_skip_it_was(family, holder, assert result.returncode == 0, result.stdout + result.stderr assert "FETCH INCOMPLETE" in result.stderr assert "not for want of a credential" not in result.stderr + assert (" every member and leg fetched" + not in result.stdout.splitlines()), result.stdout for project in (MEMBER, OTHER): assert (f" [{project}] {project}: NOT CHECKED OUT; skipped. See the " "fetch above.") in result.stdout.splitlines() assert result.stdout.endswith("\nfamily bootstrap ok\n"), result.stdout +def skip_by_update_none(clone: Path, env: dict) -> dict: + """`submodule..update=none` for OTHER, set for every git the run + starts, as a `.gitmodules` committed with `update = none` sets it.""" + return with_config(env, f"submodule.members/{OTHER}.update", "none") + + +def skip_by_subset(clone: Path, env: dict) -> dict: + """`submodule.active` in the clone's own git config naming MEMBER alone, + as `git clone --recurse-submodules=members/

` stores it.""" + raw(clone, env, "config", "submodule.active", f"members/{MEMBER}") + return env + + +@pytest.mark.parametrize("skip", [skip_by_update_none, skip_by_subset], + ids=["update-none", "subset"]) def test_an_update_that_exits_0_and_skips_a_member_fails(family, holder, - offline, tmp_path): + offline, tmp_path, + skip): """Git exited 0, so it lacked nothing a credential gives, though the - holder's records give OTHER: here `submodule..update=none` set - for the run skips it. (a) does not say every member was fetched, and - the run fails on OTHER alone.""" - env = isolated(offline, tmp_path) + holder's records give OTHER: an `update = none`, or a clone that chose + a subset of members, skips it (rv648's N1 and P1). (a) does not say + every member was fetched, and the run fails on OTHER alone, printing + the update that checks it out: `--init` with its path registers it, and + `--checkout` overrides the `none`. `git submodule update --init + --recursive`, `member-uninitialized`'s remedy, exits 0 there and checks + nothing out. Typed as printed, the line lets the same run end ok.""" + env = fetching(isolated(offline, tmp_path)) clone = cloned(holder, env, family, tmp_path, recurse=False) - skip = with_config(fetching(env), f"submodule.members/{OTHER}.update", - "none") + env = skip(clone, env) - result = bootstrap(clone, skip, "--skip-members") + result = bootstrap(clone, env, "--skip-members") assert result.returncode == 1, result.stdout + result.stderr assert (clone / "members" / MEMBER / "project.yaml").is_file() assert "every member and leg fetched" not in result.stdout assert result.stderr.splitlines() == [ - not_fetched((f"members/{OTHER}", EXITED_0)), failed_on(OTHER)] + not_fetched((f"members/{OTHER}", EXITED_0), + cure=CHECKOUT_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)] + raw(clone, env, "submodule", "update", "--init", "--recursive") + assert not (clone / "members" / OTHER / ".git").exists(), ( + "premise: the validator's remedy checks nothing out here") + (line,) = re.findall(r"`(git submodule update [^`]*)`", result.stderr) + type_line(clone, env, line) + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + assert again.stderr == "" + + +@pytest.mark.parametrize("set_up", UNNAMED_STOPS.values(), ids=UNNAMED_STOPS) +def test_a_gitlink_no_row_names_that_stops_a_plain_clones_update_fails( + family, holder, offline, tmp_path, set_up): + """rv648's S1 and H1. In a plain clone of a holder that records a stray + case variant of a member's gitlink, or a member mounted by hand, git's + init dies at that gitlink (`No url found for submodule path`, exit 128) + before it checks out any member, though every row's gitlink and section + are there: the run said ok. It names both rows, says where git stopped, + and names `--no-members`, which names that gitlink and the commit that + cures it. Typed as printed (all but the hand mount's `family.py add`, + which mounts it again) and committed, the same run checks both members + out.""" + env = fetching(isolated(offline, tmp_path)) + stopper = set_up(holder, env, family) + commit = raw(holder, env, "rev-parse", + f"HEAD:{stopper}").stdout.decode("ascii").strip() + clone = cloned(holder, env, family, tmp_path, recurse=False) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = STOPPED.format(path=stopper) + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=UNNAMED_CURE), + failed_on(MEMBER, OTHER)], result.stderr + assert f"No url found for submodule path '{stopper}'" in result.stderr + fallback = validate(clone, "--no-members") + assert fallback.returncode == 1, fallback.stdout + fallback.stderr + (finding,) = findings(fallback) + assert finding.startswith(GITLINK_SAID.format(path=stopper, + commit=commit)), finding + for line in commands(finding): + if not line.startswith(f"{PYTHON} "): + type_line(clone, env, line) + commit_all(clone, env, "Take the gitlink out as printed") + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +def test_a_section_with_no_url_that_stops_the_update_says_to_write_it_back( + family, holder, offline, tmp_path): + """rv648's U1. OTHER's section lost its url, committed: in a plain + clone git's init dies at OTHER's own gitlink, after MEMBER's and before + it checks out either. The run said ok, and no mode of the validator + names it. It names both rows and says to write that section back; + written back as `family.py add` wrote it and committed, the same run + checks both members out.""" + env = fetching(isolated(offline, tmp_path)) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, without_url(data, OTHER)) + clone = cloned(holder, env, family, tmp_path, recurse=False) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = STOPPED.format(path=f"members/{OTHER}") + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=SECTION_CURE.format(path=f"members/{OTHER}")), + failed_on(MEMBER, OTHER)], result.stderr + assert (f"No url found for submodule path 'members/{OTHER}'" + in result.stderr) + (clone / GITMODULES).write_bytes(data) + raw(clone, env, "add", "--", GITMODULES) + commit_all(clone, env, "OTHER's section written back") + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +@pytest.mark.parametrize("set_up", UNNAMED_STOPS.values(), ids=UNNAMED_STOPS) +def test_a_recursive_clone_with_no_credential_is_still_the_skip( + family, holder, offline, tmp_path, set_up): + """No misfire. A recursive clone sets `submodule.active` (`.`), and + git's init then asks only the gitlinks a section gives a path, so the + stray or the hand mount stops nothing there, and git fails on the + members' own fetch, as a private member with no token fails. The + members are the credential's: skipped, and the run ends ok.""" + env = isolated(offline, tmp_path) + set_up(holder, env, family) + remote = family["remotes"] / f"{NAME}.git" + branch = f"w648/{next(BRANCHES)}" + raw(holder, env, "push", "-q", str(remote), f"HEAD:refs/heads/{branch}") + clone = tmp_path / "clone" + # The holder is fetched, as a person with access to it fetches it, and + # its members are not: `user` refuses the file transport to the clones + # git makes for submodules. + made = clone_just_pushed(remote, clone, branch, + config=["-c", "protocol.file.allow=user"]) + assert made.returncode != 0, "premise: the members were not fetched" + assert "transport 'file' not allowed" in made.stderr, made.stderr + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert "FETCH INCOMPLETE" in result.stderr + assert "not for want of a credential" not in result.stderr + + +def test_a_url_an_earlier_init_registered_is_still_the_skip(family, holder, + offline, + tmp_path): + """No misfire. `git submodule init` wrote each member's url into the + clone's git config, then the work tree's section for OTHER lost its + url: git's init reads the url it wrote and does not stop there, and git + fails on the fetch for want of a credential. The members are skipped and + the run ends ok.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, recurse=False) + raw(clone, env, "submodule", "init") + data = (clone / GITMODULES).read_bytes() + (clone / GITMODULES).write_bytes(without_url(data, OTHER)) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert "FETCH INCOMPLETE" in result.stderr + assert "not for want of a credential" not in result.stderr def test_a_row_written_twice_is_named_once(family, holder, offline, @@ -314,7 +541,8 @@ def test_a_plain_clone_of_a_case_variant_is_named_and_mended_as_printed( family, holder, offline, tmp_path): """The round trip a person makes. In a plain clone of #363's state, with the variant at the SECOND row, `make bootstrap`'s script fails naming - OTHER and the validator; the validator, members checked, names the + both rows, since git stopped at the variant before it checked out + MEMBER, and the validator; the validator, members checked, names the variant and the move, where it refused `member-uninitialized` for the first row before. Typed as printed in the clone, with the commit and the `mv` where the disk holds the two apart, the same script checks @@ -328,7 +556,11 @@ def test_a_plain_clone_of_a_case_variant_is_named_and_mended_as_printed( assert booted.returncode == 1, booted.stdout + booted.stderr assert booted.stderr.splitlines()[-2:] == [ - not_fetched((f"members/{OTHER}", NO_GITLINK)), failed_on(OTHER)] + not_fetched((f"members/{MEMBER}", + STOPPED.format(path=f"Members/{OTHER}")), + (f"members/{OTHER}", NO_GITLINK), + cure=f"{VALIDATOR_CURE} {UNNAMED_CURE}"), + failed_on(MEMBER, OTHER)] gate = validate(clone) assert gate.returncode == 2, gate.stdout + gate.stderr lines = gate.stderr.splitlines() From 45c512eb797b6e6a8d614e34bb5750f5ccba5221 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 19:43:52 +0000 Subject: [PATCH 06/20] tests/test_doctor_legs_members_next.py: the bootstrap that cures nothing in #414's url-less member state now fails, as 1a69505d makes it, so the test pins exit 1 and `git stopped at members/Ghost` where it pinned the old `ok` (#648, rv648's U1) `test_414_3_there_the_bootstrap_cures_nothing_and_the_git_line_cures_one` builds rv648's U1 state: a member, Ghost, whose `.gitmodules` section gives its path and no url. Git's init stops the holder's update there, before it checks out the other member. The test asserted bootstrap's exit 0, the false `ok` rv648's M1 asked to end. It asserts exit 1 now, and the line naming where git stopped. Its point is unchanged: the bootstrap cures nothing, and the doctor's git line cures the other member. Found by the targeted run of every test file that drives the family bootstrap (1 failed, 3797 passed, 32 skipped); the file now passes alone, 124 passed. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- tests/test_doctor_legs_members_next.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_doctor_legs_members_next.py b/tests/test_doctor_legs_members_next.py index 8c908562..b6429df0 100644 --- a/tests/test_doctor_legs_members_next.py +++ b/tests/test_doctor_legs_members_next.py @@ -1584,7 +1584,10 @@ def test_414_3_there_the_bootstrap_cures_nothing_and_the_git_line_cures_one( f"{module.quote_arg(tried / 'scripts' / 'bootstrap.py')} " f"--root {module.quote_arg(tried)}", tmp_path / "boot") output = bootstrap.stdout + bootstrap.stderr - assert bootstrap.returncode == 0, output + # It cures nothing, and since #648 it says so and fails: git stopped + # at Ghost, whose section gives no url, before it checked out either. + assert bootstrap.returncode == 1, output + assert "git stopped at members/Ghost" in output, output assert "No url found" in output, output assert not any((tried / "members" / FAMILY_MEMBER).iterdir()), output result = paste(row_of(standard, root, "members"), tmp_path / "elsewhere") From 23e11056b320e874a9ecd88fad2ad98fa1ded74f Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 00:03:06 +0000 Subject: [PATCH 07/20] bootstrap.py: `.gitmodules` is read as git's own parser reads it, a member git has no url for is its own reason, and a gitlink outside members/ gets a cure true for it, so a later duplicate section, a section under a name git ignores or a url git drops no longer ends `family bootstrap ok`, and a subset clone's printed update no longer dies `No url found` (#648, rv648's verify) rv648's verify of 45c512eb (1 MUST, 1 SHOULD, 3 NIT). M1. `init_stop` read `.gitmodules` with `git config --file --list`, which keeps entries git's submodule parser drops or overrides. In three plain clones git's init died (`No url found`, exit 128) and the run still said ok: a second section for a member's path, with no url, after its own (D1); the member's section under the name `../Nova` (Q1); and its url `-nowhere` (O1). `gitmodules_sections` now returns a `Gitmodules` with git 2.43's own reading beside the one the validator's findings use: - a section whose name is empty or has a `..` component, split on `/` and on a backslash, is ignored whole (`check_submodule_name`); - a `path` or `url` that begins with `-` is dropped, and the value before it stands (`looks_like_command_line_option`); - where two sections give one path, the later decides, and a section that gives a path again lets go of the one it gave, even where a later section gave that one too (`cache_put_path`, `cache_remove_path`). `has_url` asks the section git reads for a path, then the config under that section's name. `init_stop` and the reasons ask it. Where more than one section gives the path git has no url for, the line names `validate-family.py --no-members`, whose `gitmodules-path-twice` says to take the others out: a section written back would be one more (D1). S1. In a subset clone of a member whose section has no url (A1), git exits 0 and the line printed the exit-0 skip's update, which dies `No url found` when typed. "git has no url for that path" is now its own reason, asked before "git exited 0", with the write-it-back cure. The stop's sentence says "a gitlink it has no url for", which is true in D1 too, where a section does give the path a url. NIT 1. A gitlink outside members/ that git stopped at (H2) was sent to `--no-members`, which reads only members/ and says `family ok` there. It now gets `git rm --cached -- ` and a commit: measured, that ends ok and the gate passes in every mode, while a section written for it leaves `gitmodules-path-unnamed` in every mode. NIT 2. The `cures` docstring says what members mode refuses for #363's variant (`member-gitlink-absent`, with the move) as well as for a stray or a hand mount (`member-uninitialized`). Tests in tests/test_family_bootstrap_gate_order.py: U1, Q1 (three names) and O1 in one parametrized test, each written back; D1, typed through `gitmodules-path-twice`; D2, a name with dots and a url `-x` after a url, each still the credential's skip and then a true ok; A1, typed through both rounds; and H2, typed and the gate passing in every mode. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 270 +++++++++++++------- tests/test_family_bootstrap_gate_order.py | 280 ++++++++++++++++++--- 2 files changed, 436 insertions(+), 114 deletions(-) diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 4e089c99..25893717 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -33,19 +33,25 @@ why, and fails the run (c): this holder records no gitlink at the row's path (a gitlink recorded only in another case, #363); no section of `.gitmodules` gives that path (a holder whose HEAD lost the - file, #611); git exited 0 and checked nothing out there; or git - stopped at a gitlink no section gives a url, the row's own or another - (a stray case variant, a member mounted by hand, a section that lost - its url), since git's init dies at the first such gitlink it asks - about, before it checks out any member. Git exits 0 over the second - where a recursive clone ran first, and 128 (`No url found for - submodule path`) in a plain clone; a missing credential exits - non-zero too, so what the holder records is the test, never git's - words. The line says what cures each: `validate-family.py` names the - commit for the first two; for the third it prints the `git submodule - update --init --checkout` that checks the member out whatever the - clone chose; and for the last, `validate-family.py --no-members` - names a gitlink no row names, and where the gitlink is a row's own + file, #611); git has no url for that path, in the section it reads + for it or in its config (a section that lost its url, or a second + one for the path after the member's); git exited 0 and checked + nothing out there; or git stopped at another gitlink it has no url + for (a stray case variant, a member mounted by hand), since git's + init dies at the first such gitlink it asks about, before it checks + out any member. Which section and which url git reads is its own + parser's answer, not `git config`'s (`gitmodules_sections`). Git + exits 0 over the second where a recursive clone ran first, and 128 + (`No url found for submodule path`) in a plain clone; a missing + credential exits non-zero too, so what the holder records is the + test, never git's words. The line says what cures each (`cures`): + `validate-family.py` names the commit for the first two; for the + fourth it prints the `git submodule update --init --checkout` that + checks the member out whatever the clone chose. For the third and + the last, `validate-family.py --no-members` names a path two + sections give, or a gitlink in members/ that no row names; `git rm + --cached` takes out one outside members/, which the validator does + not read; and where the gitlink is a row's own, with one section, the line says to write its section back. (b) RUNS EACH MEMBER'S OWN `make bootstrap`. Each member is a whole project @@ -73,7 +79,9 @@ import shutil import subprocess import sys +from collections import Counter from pathlib import Path +from typing import NamedTuple sys.path.insert(0, str(Path(__file__).resolve().parent)) from repo_shape import ( # noqa: E402 @@ -92,9 +100,9 @@ #: filled with the gitlink git's init died at (`init_stop`). NO_GITLINK = "no gitlink is recorded at that path" NO_SECTION = f"no section of {GITMODULES} gives that path" +NO_URL = "git has no url for that path" EXITED_0 = "git exited 0 and checked nothing out there" -STOPPED = ("git stopped at {}, a gitlink no section of " + GITMODULES - + " gives a url") +STOPPED = "git stopped at {}, a gitlink it has no url for" #: The two environment variables a leg/member credential may arrive in, in the #: order the `validate` workflow resolves them. An App-minted installation @@ -145,14 +153,14 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: "fine-grained PAT with contents:read on them) or export an " "App-minted SHAPE_LEGS_APP_TOKEN, then re-run. What did arrive " "is bootstrapped below.", file=sys.stderr) - left, stopper = unfetchable(root, rows, proc.returncode == 0) + left, stopper, modules = unfetchable(root, rows, proc.returncode == 0) if left: print(" NOT CHECKED OUT, and not for want of a credential: " + ", ".join(f"{spelled(path)} (" + (STOPPED.format(spelled(stopper)) if why == STOPPED else why) + ")" for _, path, why in left) - + ". " + cures(left, stopper, rows), file=sys.stderr) + + ". " + cures(left, stopper, rows, modules), file=sys.stderr) elif proc.returncode == 0: print(" every member and leg fetched") return [f"{spelled(row.get('project') or '?')} (not checked out)" @@ -160,30 +168,63 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: def cures(left: list[tuple[dict, str, str]], stopper: str | None, - rows: list[dict]) -> str: + rows: list[dict], modules: Gitmodules) -> str: """What (a)'s line says cures the rows `left` names, one sentence for - each why among them (#648). `validate-family.py`, which names the - commit, for a gitlink or a section the holder lacks at the row's path. - For the gitlink git stopped at, `validate-family.py --no-members`, which - names a gitlink no row names and the commit that cures it (members - checked, it refuses `member-uninitialized` for the first row there - instead), or, where a row names it, its section written back. For a row - git exited 0 over, the update that checks it out whatever the clone - chose: `--init` with the path registers a member a subset clone left - out, and `--checkout` overrides an `update = none`.""" + each kind of cause among them (#648): + + - A gitlink or a section the holder lacks at the row's path: + `validate-family.py`, which names it and the commit that cures it, + `member-gitlink-absent` (with the move, for a gitlink recorded only + in another case, #363) or `member-gitmodules-absent`. + - A path git has no url for (`has_url`), a row's own or the gitlink git + stopped at. Where more than one section gives it, as a merge that + kept both sides leaves them, `validate-family.py --no-members`, whose + `gitmodules-path-twice` says which to keep and to take the others + out: a section written back would be one more. Otherwise a row's own + is its section written back. A gitlink in `members/` no row names is + `--no-members` too, whose `member-gitlink-unnamed` names it and the + commit that cures it: members checked, the validator refuses + `member-uninitialized` for the first row where every row's gitlink + is there, as with a stray or a hand mount, and `member-gitlink-absent` + where a row's is only in another case, which the first sentence + names. A gitlink outside `members/`, which no mode of the validator + asks about, is taken out of the index: no row names it, and with no + url no clone can check it out. + - A row git exited 0 over: the update that checks it out whatever the + clone chose. `--init` with the path registers a member a subset + clone left out, and `--checkout` overrides an `update = none`.""" whys = {why for _, _, why in left} said = [] if whys & {NO_GITLINK, NO_SECTION}: said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " "and the commit that cures it.") - if STOPPED in whys and stopper in {row_path(row) for row in rows}: - said.append(f"Write the section of {GITMODULES} for " - f"{spelled(stopper)} back as `family.py add` writes one, " - f"with its path and url, and commit {GITMODULES}.") - elif STOPPED in whys: + unread = [path for _, path, why in left if why == NO_URL] + if STOPPED in whys and stopper not in {path for _, path, _ in left}: + unread.append(stopper) + named = {row_path(row) for row in rows} + twice = [path for path in unread if modules.given[path] > 1] + own = [path for path in unread if path in named and path not in twice] + if twice: said.append(f"`{PYTHON} scripts/validate-family.py --no-members` " - f"names each gitlink in {DEFAULT_MEMBERS_DIR}/ that no " - "row names, and the commit that cures it.") + f"names each path more than one section of {GITMODULES} " + "gives, and how to take the others out.") + if own: + said.append(f"Write {'the' if len(own) == 1 else 'each'} section of " + f"{GITMODULES} for {', '.join(map(spelled, own))} back " + "as `family.py add` writes one, with its path and url, " + f"and commit {GITMODULES}.") + for path in unread: + if path in named or path in twice: + continue + if path.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR: + said.append(f"`{PYTHON} scripts/validate-family.py --no-members` " + f"names each gitlink in {DEFAULT_MEMBERS_DIR}/ that " + "no row names, and the commit that cures it.") + else: + removal = shlex.join(["git", "rm", "--cached", "--", path]) + said.append(f"No row names {spelled(path)} and no clone can check " + f"it out: `{spelled(removal)}` takes its gitlink " + "out, then commit that.") exited = [path for _, path, why in left if why == EXITED_0] if exited: update = shlex.join(["git", "submodule", "update", "--init", @@ -193,111 +234,170 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, return " ".join(said) -def gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: - """`{name: {key: value}}` of every section of `.gitmodules`, read as git - reads it: the work tree's file, the one `git submodule update` reads, - which a fresh clone takes from HEAD. An empty mapping where there is no +class Gitmodules(NamedTuple): + """`.gitmodules` as (a) asks it (#648), read by `gitmodules_sections`. + + `given` counts the sections that give each path, every section kept, as + `validate-family.py` reads them for the findings (a)'s line names: + `member-gitmodules-absent` and `gitmodules-path-twice`. `read` is + `{path: name}`, the section git reads for each path, and `urls` is + `{name: url}`, the url git takes from each, as git's own parser has + them.""" + + given: Counter + read: dict[str, str] + urls: dict[str, str] + + +def gitmodules_sections(root: Path) -> Gitmodules: + """`.gitmodules` as `git submodule update` reads it: the work tree's + file, which a fresh clone takes from HEAD. Nothing where there is no such file, or one git cannot read: git has no url for any submodule - then either.""" + then either. + + `read` and `urls` keep git 2.43's own rules (submodule-config.c), which + drop what `git config --list` keeps. Entry by entry, in file order: + + - a section whose name is empty or has a `..` component, split on `/` + and on a backslash, is ignored whole (git warns `ignoring suspicious + submodule name`); + - a `path` or `url` that begins with `-` is dropped, and the value + before it stands (`may be interpreted as a command-line option`); + - where two sections give one path, the later in the file decides, as + a later value in one section replaces an earlier. A section that + gives a path again lets go of the one it gave, even where a later + section gave that one too: git then reads no section there.""" + nothing = Gitmodules(Counter(), {}, {}) if not os.path.lexists(root / GITMODULES): - return {} + return nothing try: listed = git_out(["config", "-z", "--file", GITMODULES, "--list"], cwd=root, binary=True) except Refusal: - return {} + return nothing sections: dict[str, dict[str, str]] = {} + read: dict[str, str] = {} + held: dict[str, str] = {} + urls: dict[str, str] = {} for record in listed.split(b"\0"): key, _, value = record.decode("utf-8", "surrogateescape").partition("\n") section, _, variable = key.rpartition(".") - if section.startswith("submodule."): - sections.setdefault(section[len("submodule."):], - {})[variable] = value - return sections + if not section.startswith("submodule."): + continue + name = section[len("submodule."):] + sections.setdefault(name, {})[variable] = value + if (not name or ".." in name.replace("\\", "/").split("/") + or value.startswith("-")): + continue + if variable == "path": + if name in held: + read.pop(held[name], None) + held[name] = value + read[value] = name + elif variable == "url": + urls[name] = value + return Gitmodules(Counter(keys["path"] for keys in sections.values() + if "path" in keys), read, urls) + + +def submodule_config(root: Path) -> set[str]: + """Every `submodule.` key this repository's git config sets, in every + scope git reads, as git's init asks them: `submodule.active`, and + `submodule..url` as an earlier init writes it. None where git + lists none.""" + try: + config = git_out(["config", "-z", "--get-regexp", r"^submodule\."], + cwd=root, binary=True) + except Refusal: + return set() + return {record.decode("utf-8", "surrogateescape").partition("\n")[0] + for record in config.split(b"\0")} + +def has_url(modules: Gitmodules, config: set[str], path: str) -> bool: + """Whether git's init finds a url for the gitlink at `path`: in the + section git reads for that path (`Gitmodules.read`), or in the git + config under that section's name, as an earlier init writes it. Where + git reads no section there it has none, whatever the config holds.""" + name = modules.read.get(path) + return name is not None and (name in modules.urls + or f"submodule.{name}.url" in config) -def init_stop(root: Path, sections: dict[str, dict[str, str]]) -> str | None: + +def init_stop(root: Path, modules: Gitmodules, config: set[str] + ) -> str | None: """The gitlink `git submodule update --init` dies at, or None: the first - in the index, in index order, that its init step asks about and finds - no url for (`No url found for submodule path`, exit 128), before it + in the index, in index order, that its init step asks about and has no + url for (`No url found for submodule path`, exit 128), before it checks out any member (#648). - ASKED AS GIT 2.43 ASKS IT, of the index, `sections` and this + ASKED AS GIT 2.43 ASKS IT, of the index, `modules` and `config`, this repository's git config. Every gitlink the index records is asked, but where the config sets `submodule.active`, as a recursive clone sets it - (`.`), only those a section gives a path: git counts no other active. - One no section gives stops it; so does one whose section gives no url, - unless the config has one under the section's name, as an earlier init - writes it. So a stray case variant or a member mounted by hand stops a - plain clone's update, and leaves a recursive clone's to the credential. - Read nothing, it answers None.""" + (`.`), only those a section git reads gives a path: git counts no other + active. A gitlink git has no url for (`has_url`) stops it. So a stray + case variant or a member mounted by hand stops a plain clone's update, + and leaves a recursive clone's to the credential. Read nothing, it + answers None.""" try: index = git_out(["ls-files", "-s", "-z"], cwd=root, binary=True) except Refusal: return None - try: - config = git_out(["config", "-z", "--get-regexp", r"^submodule\."], - cwd=root, binary=True) - except Refusal: - config = b"" - keys = {record.decode("utf-8", "surrogateescape").partition("\n")[0] - for record in config.split(b"\0")} - names: dict[str, list[str]] = {} - for name, held in sections.items(): - if "path" in held: - names.setdefault(held["path"], []).append(name) for record in index.split(b"\0"): meta, _, entry = record.partition(b"\t") if meta.split()[:1] != [GITLINK_MODE]: continue path = entry.decode("utf-8", "surrogateescape") - given = names.get(path, []) - if not given and "submodule.active" in keys: + if path not in modules.read and "submodule.active" in config: continue - if not any("url" in sections[name] or f"submodule.{name}.url" in keys - for name in given): + if not has_url(modules, config, path): return path return None def unfetchable(root: Path, rows: list[dict], exited_0: bool - ) -> tuple[list[tuple[dict, str, str]], str | None]: + ) -> tuple[list[tuple[dict, str, str]], str | None, + Gitmodules]: """`(row, path, why)` for each row whose member (a) left NOT CHECKED OUT - where no credential would check it out (#648), in row order, and the - gitlink git's init died at (`init_stop`), or None. + where no credential would check it out (#648), in row order; the + gitlink git's init died at (`init_stop`), or None; and `.gitmodules` + as it was read, for `cures`. Asked of this holder's own records, never of git's words: a row at whose path the holder records no gitlink (`recorded_gitlink`, as `validate-family.py`'s `member-gitlink-absent` reads one), or whose gitlink no section of `.gitmodules` gives a path (as its `member-gitmodules-absent` reads HEAD's file, which is the work tree's - in a clone); where git exited 0, any row at all, since git then lacked - nothing a credential gives; and where git's init died at a gitlink no - section gives a url, any row at all, since git then checked out no - member and asked for no credential. Any other row not checked out is - the missing credential's, and stays the skip it always was. A path two - rows give is said once.""" + in a clone); one git has no url for (`has_url`), where no credential + helps either, whatever git exited; where git exited 0, any row at all, + since git then lacked nothing a credential gives; and where git's init + died at a gitlink it has no url for, any row at all, since git then + checked out no member and asked for no credential. Any other row not + checked out is the missing credential's, and stays the skip it always + was. A path two rows give is said once.""" left: dict[str, dict] = {} for row in rows: if not (member_path(root, row) / ".git").exists(): left.setdefault(row_path(row), row) if not left: - return [], None - sections = gitmodules_sections(root) - given = {held["path"] for held in sections.values() if "path" in held} - stopper = None if exited_0 else init_stop(root, sections) + return [], None, Gitmodules(Counter(), {}, {}) + modules = gitmodules_sections(root) + config = submodule_config(root) + stopper = None if exited_0 else init_stop(root, modules, config) found: list[tuple[dict, str, str]] = [] for path, row in left.items(): if recorded_gitlink(root, path) is None: found.append((row, path, NO_GITLINK)) - elif path not in given: + elif not modules.given[path]: found.append((row, path, NO_SECTION)) + elif not has_url(modules, config, path): + found.append((row, path, NO_URL)) elif exited_0: found.append((row, path, EXITED_0)) elif stopper is not None: found.append((row, path, STOPPED)) - return found, stopper + return found, stopper, modules def members_of(root: Path) -> tuple[str, list[dict]]: diff --git a/tests/test_family_bootstrap_gate_order.py b/tests/test_family_bootstrap_gate_order.py index bbe04750..06598e5a 100644 --- a/tests/test_family_bootstrap_gate_order.py +++ b/tests/test_family_bootstrap_gate_order.py @@ -12,12 +12,15 @@ gitlink recorded only at `Members/

`), git dies `No url found for submodule path` (exit 128), and it still ended ok. It fails now, saying which members no credential would check out and why, and what cures - them. So it does where git's init dies at a gitlink no section gives a - url, a stray case variant, a member mounted by hand or a section that - lost its url (rv648's S1, H1 and U1), and where git exits 0 over a member - a subset clone or `update = none` leaves out, whose line prints the - update that checks it out. A missing credential is still the skip it - was, after a recursive clone too. + them. So it does where git's init dies at a gitlink it has no url for: + a stray case variant, a member mounted by hand, in `members/` or outside + it, a section that lost its url (rv648's S1, H1, H2 and U1), and, as + git's own parser reads `.gitmodules`, a second section for a member's + path after its own, a section under a name git ignores and a url git + drops (D1, Q1 and O1). So it does where git exits 0 over a member a + subset clone or `update = none` leaves out, whose line prints the update + that checks it out, unless git has no url for it (A1). A missing + credential is still the skip it was, after a recursive clone too. - (b) In a plain clone of #363's state followed by `git submodule update --init --recursive`, as CI and `make bootstrap` run it, the update dies at `Members/

` before it checks out any member, so where that row is not @@ -74,13 +77,15 @@ "{cure}") NO_SECTION = "no section of .gitmodules gives that path" NO_GITLINK = "no gitlink is recorded at that path" +NO_URL = "git has no url for that path" EXITED_0 = "git exited 0 and checked nothing out there" -STOPPED = ("git stopped at {path}, a gitlink no section of .gitmodules " - "gives a url") +STOPPED = "git stopped at {path}, a gitlink it has no url for" #: What the line says cures each why: the validator for the first two; for -#: the gitlink git stopped at, the validator's `--no-members`, or, where a -#: row names that gitlink, its section written back; and for a member git +#: a path git has no url for, a row's own or the gitlink git stopped at, +#: the validator's `--no-members` where two sections give it, a row's own +#: section written back, `--no-members` for a gitlink in members/ no row +#: names and `git rm --cached` for one outside it; and for a member git #: exited 0 over, the update that checks it out. VALIDATOR_CURE = (f"`{PYTHON} scripts/validate-family.py` names the cause " "and the commit that cures it.") @@ -90,6 +95,11 @@ SECTION_CURE = ("Write the section of .gitmodules for {path} back as " "`family.py add` writes one, with its path and url, and " "commit .gitmodules.") +TWICE_CURE = (f"`{PYTHON} scripts/validate-family.py --no-members` names " + "each path more than one section of .gitmodules gives, and how " + "to take the others out.") +OUTSIDE_CURE = ("No row names {path} and no clone can check it out: `git rm " + "--cached -- {path}` takes its gitlink out, then commit that.") CHECKOUT_CURE = ("`git submodule update --init --checkout --recursive -- " "{path}` checks it out.") @@ -184,17 +194,49 @@ def hand_mounted(holder: Path, env: dict, family) -> str: UNNAMED_STOPS = {"stray": stray, "hand-mount": hand_mounted} -def without_url(data: bytes, project: str) -> bytes: - """`.gitmodules` bytes with the `url` line of `project`'s section taken - out, the rest as `family.py add` wrote it (rv648's U1). A checkout's - copy may end its lines CRLF.""" +def url_edited(data: bytes, project: str, edit) -> bytes: + """`.gitmodules` bytes with the `url` line of `project`'s section put + through `edit`, the rest as `family.py add` wrote it. A checkout's copy + may end its lines CRLF.""" section = re.compile(rb'(\[submodule "members/' + project.encode() - + rb'"\]\r?\n(?:\t[^\n]*\n)*?)\turl = [^\n]*\n') - edited, count = section.subn(rb"\1", data) + + rb'"\]\r?\n(?:\t[^\n]*\n)*?)(\turl = [^\n]*\n)') + edited, count = section.subn( + lambda found: found.group(1) + edit(found.group(2)), data) assert count == 1, data return edited +def without_url(data: bytes, project: str) -> bytes: + """`.gitmodules` bytes with the `url` line of `project`'s section taken + out (rv648's U1).""" + return url_edited(data, project, lambda line: b"") + + +def renamed(data: bytes, name: str) -> bytes: + """`.gitmodules` bytes with OTHER's section under the name `name`, its + keys as `family.py add` wrote them, escaped as git config escapes a + subsection name.""" + own = f'[submodule "members/{OTHER}"]'.encode() + assert data.count(own) == 1, data + escaped = name.replace("\\", "\\\\").replace('"', '\\"') + return data.replace(own, f'[submodule "{escaped}"]'.encode("utf-8")) + + +#: A second section, `dup`, that gives OTHER's path and no url, as a merge +#: that kept both sides of `.gitmodules` can leave one (rv648's D1 and D2). +DUPLICATE = f'[submodule "dup"]\n\tpath = members/{OTHER}\n'.encode() + + +def edited_clone(holder: Path, env: dict, family, tmp_path: Path, + edit) -> tuple[Path, bytes]: + """HEAD's `.gitmodules` put through `edit` by hand and committed, then a + plain clone of it (`cloned`). Returns the clone and the bytes `family.py + add` wrote, which writing the file back restores.""" + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, edit(data)) + return cloned(holder, env, family, tmp_path, recurse=False), data + + def failed_on(*projects: str) -> str: return "family bootstrap FAILED: " + ", ".join( f"{project} (not checked out)" for project in projects) @@ -430,37 +472,217 @@ def test_a_gitlink_no_row_names_that_stops_a_plain_clones_update_fails( assert after_first_line(again) == FETCHED -def test_a_section_with_no_url_that_stops_the_update_says_to_write_it_back( - family, holder, offline, tmp_path): - """rv648's U1. OTHER's section lost its url, committed: in a plain - clone git's init dies at OTHER's own gitlink, after MEMBER's and before - it checks out either. The run said ok, and no mode of the validator - names it. It names both rows and says to write that section back; - written back as `family.py add` wrote it and committed, the same run - checks both members out.""" +#: `(edit, what git warns)`: OTHER's section with no url (rv648's U1), or +#: one git's own parser drops, though `git config --list` keeps it: a name +#: that is empty or has a `..` component, split on `/` or a backslash +#: (Q1), and a url that begins with `-` (O1). +NO_URL_FOR_OTHER = { + "no-url": (lambda data: without_url(data, OTHER), ""), + "dotdot-name": (lambda data: renamed(data, f"../{OTHER}"), + f"ignoring suspicious submodule name: ../{OTHER}\n"), + "empty-name": (lambda data: renamed(data, ""), + "ignoring suspicious submodule name: \n"), + "backslash-name": (lambda data: renamed(data, f"{OTHER}\\..\\x"), + "ignoring suspicious submodule name: " + f"{OTHER}\\..\\x\n"), + "option-url": (lambda data: url_edited(data, OTHER, + lambda line: b"\turl = -nowhere\n"), + "which may be interpreted as a command-line option: " + "-nowhere\n"), +} + + +@pytest.mark.parametrize("edit, warned", NO_URL_FOR_OTHER.values(), + ids=NO_URL_FOR_OTHER) +def test_a_member_git_has_no_url_for_stops_the_update_and_is_written_back( + family, holder, offline, tmp_path, edit, warned): + """rv648's U1, Q1 and O1. OTHER's section lost its url, or is one git + reads no url from, committed: in a plain clone git's init dies at + OTHER's own gitlink, after MEMBER's and before it checks out either. + The run said ok in Q1 and O1, reading the section with `git config`, + which keeps what git drops, and no mode of the validator names any of + the three. It names both rows, OTHER for want of a url and MEMBER + stopped there, and says to write OTHER's section back; written back as + `family.py add` wrote it and committed, the same run checks both + members out.""" env = fetching(isolated(offline, tmp_path)) - data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout - committed_gitmodules(holder, env, without_url(data, OTHER)) - clone = cloned(holder, env, family, tmp_path, recurse=False) + clone, data = edited_clone(holder, env, family, tmp_path, edit) result = bootstrap(clone, env, "--skip-members") assert result.returncode == 1, result.stdout + result.stderr - why = STOPPED.format(path=f"members/{OTHER}") assert result.stderr.splitlines()[-2:] == [ - not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + not_fetched((f"members/{MEMBER}", + STOPPED.format(path=f"members/{OTHER}")), + (f"members/{OTHER}", NO_URL), cure=SECTION_CURE.format(path=f"members/{OTHER}")), failed_on(MEMBER, OTHER)], result.stderr assert (f"No url found for submodule path 'members/{OTHER}'" in result.stderr) + assert warned in result.stderr + (clone / GITMODULES).write_bytes(data) + raw(clone, env, "add", "--", GITMODULES) + commit_all(clone, env, "OTHER's section written back") + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +def test_a_second_section_git_reads_for_a_row_s_path_names_the_two( + family, holder, offline, tmp_path): + """rv648's D1. A second section, `dup`, gives OTHER's path and no url, + after OTHER's own: git reads the later of two sections for one path, so + in a plain clone its init dies at OTHER's gitlink, after MEMBER's and + before it checks out either. The run said ok, since one section gave + the path a url. It names both rows and `--no-members`, whose + `gitmodules-path-twice` names the two sections and says to take the + other out: a section written back would be a third. Taken out as that + says and committed, the same run checks both members out.""" + env = fetching(isolated(offline, tmp_path)) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: data + DUPLICATE) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", + STOPPED.format(path=f"members/{OTHER}")), + (f"members/{OTHER}", NO_URL), cure=TWICE_CURE), + failed_on(MEMBER, OTHER)], result.stderr + assert (f"No url found for submodule path 'members/{OTHER}'" + in result.stderr) + fallback = validate(clone, "--no-members") + assert fallback.returncode == 1, fallback.stdout + fallback.stderr + (finding,) = [found for found in findings(fallback) + if found.startswith("FINDING gitmodules-path-twice: ")] + assert finding.startswith( + f"FINDING gitmodules-path-twice: .gitmodules gives the path " + f"members/{OTHER} to 2 submodules, `members/{OTHER}` (url "), finding + assert "`dup` (no url)" in finding, finding + assert "--unset-all submodule..` for every key" in finding + raw(clone, env, "config", "-f", GITMODULES, "--unset-all", + "submodule.dup.path") + raw(clone, env, "add", "--", GITMODULES) + commit_all(clone, env, "The second section taken out as printed") + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +#: `.gitmodules` edits git's own parser reads past: rv648's D2, the second +#: section placed BEFORE OTHER's own, which git then reads; a name whose +#: dots are no `..` component; and a url that begins with `-` after OTHER's +#: own, which git drops, keeping the one before it. +READ_PAST = { + "duplicate-first": lambda data: DUPLICATE + data, + "dots-in-a-name": lambda data: renamed(data, f"..{OTHER}.."), + "option-url-after-a-url": lambda data: url_edited( + data, OTHER, lambda line: line + b"\turl = -x\n"), +} + + +@pytest.mark.parametrize("edit", READ_PAST.values(), ids=READ_PAST) +def test_what_git_reads_past_is_still_the_skip_and_a_true_ok( + family, holder, offline, tmp_path, edit): + """No misfire. Where git's parser still reads a url for OTHER, a plain + clone's update with no credential for the members fails on their fetch, + not in its init, so they are the credential's: skipped, and the run + ends ok. With the credential, the same update checks both members out, + exit 0, and the run is a good holder's.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, edit) + + skipped = bootstrap(clone, env, "--skip-members") + + assert skipped.returncode == 0, skipped.stdout + skipped.stderr + assert "FETCH INCOMPLETE" in skipped.stderr + assert "not for want of a credential" not in skipped.stderr + fetched = bootstrap(clone, fetching(env), "--skip-members") + assert fetched.returncode == 0, fetched.stdout + fetched.stderr + assert after_first_line(fetched) == FETCHED + assert fetched.stderr == "" + + +def test_a_subset_clone_of_a_member_git_has_no_url_for_is_written_back( + family, holder, offline, tmp_path): + """rv648's A1. OTHER's section lost its url, committed, and the clone + chose MEMBER alone: git exits 0, and the line printed the exit-0 skip's + update, which dies `No url found` when typed (exit 128) and checks + nothing out. Git has no url for OTHER, in its section or its config, + so the line says to write the section back. Written back and + committed, the run is the exit-0 skip, and the update it prints, typed + as printed, lets the same run end ok.""" + env = fetching(isolated(offline, tmp_path)) + clone, data = edited_clone(holder, env, family, tmp_path, + lambda data: without_url(data, OTHER)) + env = skip_by_subset(clone, env) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines() == [ + not_fetched((f"members/{OTHER}", NO_URL), + cure=SECTION_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)] (clone / GITMODULES).write_bytes(data) raw(clone, env, "add", "--", GITMODULES) commit_all(clone, env, "OTHER's section written back") + skipped = bootstrap(clone, env, "--skip-members") + assert skipped.returncode == 1, skipped.stdout + skipped.stderr + assert skipped.stderr.splitlines() == [ + not_fetched((f"members/{OTHER}", EXITED_0), + cure=CHECKOUT_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)] + (line,) = re.findall(r"`(git submodule update [^`]*)`", skipped.stderr) + type_line(clone, env, line) + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + assert again.stderr == "" + + +def test_a_hand_mount_outside_members_is_taken_out_of_the_index( + family, holder, offline, tmp_path): + """rv648's H2. SPARE's gitlink recorded by hand at `vendor/`, + outside members/, with no row and no section, and committed: in a plain + clone git's init dies there before it checks out any member. No mode + of the validator reads a gitlink outside members/, and `--no-members` + says `family ok`, so the line names the `git rm --cached` that takes + the gitlink out. Typed as printed and committed, the same run checks + both members out, and the gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + stopper = f"vendor/{SPARE}" + stage_gitlink(holder, stopper, spare_commits(family)[1]) + commit_all(holder, env, "A gitlink mounted by hand outside members/") + clone = cloned(holder, env, family, tmp_path, recurse=False) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = STOPPED.format(path=stopper) + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=OUTSIDE_CURE.format(path=stopper)), + failed_on(MEMBER, OTHER)], result.stderr + assert f"No url found for submodule path '{stopper}'" in result.stderr + fallback = validate(clone, "--no-members") + assert fallback.returncode == 0, "premise: it reads only members/" + (line,) = re.findall(r"`(git rm [^`]*)`", result.stderr) + type_line(clone, env, line) + commit_all(clone, env, "The gitlink taken out as printed") again = bootstrap(clone, env, "--skip-members") assert again.returncode == 0, again.stdout + again.stderr assert after_first_line(again) == FETCHED + assert_passes_in_every_mode(clone, 2) @pytest.mark.parametrize("set_up", UNNAMED_STOPS.values(), ids=UNNAMED_STOPS) From f74f640ca69de5dcda0bd84dbbb441f601f26cde Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 00:29:07 +0000 Subject: [PATCH 08/20] bootstrap.py: a `.gitmodules` a merge left unmerged in the index is its own reason, since git reads no section from it, so a plain clone mid-merge whose update dies `No url found` no longer ends `family bootstrap ok` (#648, rv648's MM1) rv648's verify of 23e11056 (1 MUST). In a plain clone, a merge that conflicts in `.gitmodules`, resolved in the work tree and not added, leaves the index holding the file only at its three conflict stages. Git then reads no `.gitmodules` at all (`repo_read_gitmodules` asks `is_gitmodules_unmerged` first), so its init dies at the first gitlink, exit 128. `gitmodules_sections` read the work tree's file, found every section with its url, and the run ended ok. `gitmodules_sections` now asks `git ls-files -u -z -- .gitmodules` first. Where the index holds the file at a conflict stage, it returns nothing, with `Gitmodules.unmerged` set. `unfetchable` asks that first: every row not checked out gets "git reads no .gitmodules while a merge leaves it unmerged", and the line says to resolve the merge in .gitmodules and `git add .gitmodules`, then run this again. Added, the same run checks both members out. Test: rv648's MM1 in tests/test_family_bootstrap_gate_order.py, the two sides of the merge each giving OTHER's section another `branch`; the cure typed, and the run ends ok. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 44 +++++++++++---- tests/test_family_bootstrap_gate_order.py | 62 +++++++++++++++++++++- 2 files changed, 95 insertions(+), 11 deletions(-) diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 25893717..6dce353f 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -40,11 +40,14 @@ for (a stray case variant, a member mounted by hand), since git's init dies at the first such gitlink it asks about, before it checks out any member. Which section and which url git reads is its own - parser's answer, not `git config`'s (`gitmodules_sections`). Git - exits 0 over the second where a recursive clone ran first, and 128 - (`No url found for submodule path`) in a plain clone; a missing - credential exits non-zero too, so what the holder records is the - test, never git's words. The line says what cures each (`cures`): + parser's answer, not `git config`'s (`gitmodules_sections`), and + while a merge leaves `.gitmodules` unmerged in the index git reads + none at all: every such member is said to be that, and the cure is + the file resolved and added. Git exits 0 over the second where a + recursive clone ran first, and 128 (`No url found for submodule + path`) in a plain clone; a missing credential exits non-zero too, + so what the holder records is the test, never git's words. The line + says what cures each (`cures`): `validate-family.py` names the commit for the first two; for the fourth it prints the `git submodule update --init --checkout` that checks the member out whatever the clone chose. For the third and @@ -98,6 +101,7 @@ #: Why (a) left a row's member NOT CHECKED OUT where no credential would #: check it out (#648), in the order `unfetchable` asks. `STOPPED` is #: filled with the gitlink git's init died at (`init_stop`). +UNMERGED = f"git reads no {GITMODULES} while a merge leaves it unmerged" NO_GITLINK = "no gitlink is recorded at that path" NO_SECTION = f"no section of {GITMODULES} gives that path" NO_URL = "git has no url for that path" @@ -172,6 +176,8 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, """What (a)'s line says cures the rows `left` names, one sentence for each kind of cause among them (#648): + - A `.gitmodules` a merge left unmerged in the index, which git reads + nothing from (`Gitmodules.unmerged`): the file resolved and added. - A gitlink or a section the holder lacks at the row's path: `validate-family.py`, which names it and the commit that cures it, `member-gitlink-absent` (with the move, for a gitlink recorded only @@ -195,6 +201,9 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, clone left out, and `--checkout` overrides an `update = none`.""" whys = {why for _, _, why in left} said = [] + if UNMERGED in whys: + said.append(f"Resolve the merge in {GITMODULES} and `git add " + f"{GITMODULES}`, then run this again.") if whys & {NO_GITLINK, NO_SECTION}: said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " "and the commit that cures it.") @@ -242,18 +251,23 @@ class Gitmodules(NamedTuple): `member-gitmodules-absent` and `gitmodules-path-twice`. `read` is `{path: name}`, the section git reads for each path, and `urls` is `{name: url}`, the url git takes from each, as git's own parser has - them.""" + them. `unmerged` says a merge left the file unmerged in the index, and + then git reads nothing from it, and the three are empty.""" given: Counter read: dict[str, str] urls: dict[str, str] + unmerged: bool = False def gitmodules_sections(root: Path) -> Gitmodules: """`.gitmodules` as `git submodule update` reads it: the work tree's file, which a fresh clone takes from HEAD. Nothing where there is no such file, or one git cannot read: git has no url for any submodule - then either. + then either. Nothing, and `unmerged`, where the index holds the file + only at a merge's conflict stages (`git ls-files -u`): git reads no + `.gitmodules` at all then, whatever the work tree holds + (`repo_read_gitmodules`), and its init dies at the first gitlink. `read` and `urls` keep git 2.43's own rules (submodule-config.c), which drop what `git config --list` keeps. Entry by entry, in file order: @@ -268,6 +282,14 @@ def gitmodules_sections(root: Path) -> Gitmodules: gives a path again lets go of the one it gave, even where a later section gave that one too: git then reads no section there.""" nothing = Gitmodules(Counter(), {}, {}) + try: + stages = git_out(["ls-files", "-u", "-z", "--", GITMODULES], + cwd=root, binary=True) + except Refusal: + stages = b"" + if any(record.partition(b"\t")[2] == GITMODULES.encode() + for record in stages.split(b"\0")): + return nothing._replace(unmerged=True) if not os.path.lexists(root / GITMODULES): return nothing try: @@ -364,7 +386,9 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool gitlink git's init died at (`init_stop`), or None; and `.gitmodules` as it was read, for `cures`. - Asked of this holder's own records, never of git's words: a row at + Asked of this holder's own records, never of git's words: any row at + all while a merge leaves `.gitmodules` unmerged in the index, since git + then reads no section and asks for no credential; a row at whose path the holder records no gitlink (`recorded_gitlink`, as `validate-family.py`'s `member-gitlink-absent` reads one), or whose gitlink no section of `.gitmodules` gives a path (as its @@ -387,7 +411,9 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool stopper = None if exited_0 else init_stop(root, modules, config) found: list[tuple[dict, str, str]] = [] for path, row in left.items(): - if recorded_gitlink(root, path) is None: + if modules.unmerged: + found.append((row, path, UNMERGED)) + elif recorded_gitlink(root, path) is None: found.append((row, path, NO_GITLINK)) elif not modules.given[path]: found.append((row, path, NO_SECTION)) diff --git a/tests/test_family_bootstrap_gate_order.py b/tests/test_family_bootstrap_gate_order.py index 06598e5a..3bd49d54 100644 --- a/tests/test_family_bootstrap_gate_order.py +++ b/tests/test_family_bootstrap_gate_order.py @@ -19,8 +19,10 @@ path after its own, a section under a name git ignores and a url git drops (D1, Q1 and O1). So it does where git exits 0 over a member a subset clone or `update = none` leaves out, whose line prints the update - that checks it out, unless git has no url for it (A1). A missing - credential is still the skip it was, after a recursive clone too. + that checks it out, unless git has no url for it (A1), and where a + merge leaves `.gitmodules` unmerged, which git then reads nothing from + (MM1). A missing credential is still the skip it was, after a recursive + clone too. - (b) In a plain clone of #363's state followed by `git submodule update --init --recursive`, as CI and `make bootstrap` run it, the update dies at `Members/

` before it checks out any member, so where that row is not @@ -78,6 +80,7 @@ NO_SECTION = "no section of .gitmodules gives that path" NO_GITLINK = "no gitlink is recorded at that path" NO_URL = "git has no url for that path" +UNMERGED = "git reads no .gitmodules while a merge leaves it unmerged" EXITED_0 = "git exited 0 and checked nothing out there" STOPPED = "git stopped at {path}, a gitlink it has no url for" @@ -100,6 +103,8 @@ "to take the others out.") OUTSIDE_CURE = ("No row names {path} and no clone can check it out: `git rm " "--cached -- {path}` takes its gitlink out, then commit that.") +UNMERGED_CURE = ("Resolve the merge in .gitmodules and `git add .gitmodules`, " + "then run this again.") CHECKOUT_CURE = ("`git submodule update --init --checkout --recursive -- " "{path}` checks it out.") @@ -685,6 +690,59 @@ def test_a_hand_mount_outside_members_is_taken_out_of_the_index( assert_passes_in_every_mode(clone, 2) +def with_branch(value: str): + """An edit giving OTHER's section a `branch` key of `value`, after its + url.""" + return lambda data: url_edited( + data, OTHER, lambda line: line + f"\tbranch = {value}\n".encode()) + + +def test_a_gitmodules_a_merge_left_unmerged_says_to_add_it( + family, holder, offline, tmp_path): + """rv648's MM1. In a plain clone, a merge conflicts in `.gitmodules` + (each side gave OTHER's section another `branch`), and the file is + resolved in the work tree but not added: the index holds it only at + the conflict's three stages. Git reads no `.gitmodules` at all then, + so its init dies at MEMBER's gitlink, the first, and the run said ok, + reading the work tree's file. It names both rows for that and says to + add the file once it is resolved; added, the same run checks both + members out.""" + env = fetching(isolated(offline, tmp_path)) + remote = family["remotes"] / f"{NAME}.git" + side = f"w648/{next(BRANCHES)}" + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, with_branch("dev")(data)) + raw(holder, env, "push", "-q", str(remote), f"HEAD:refs/heads/{side}") + raw(holder, env, "update-index", "-q", "--refresh", check=False) + raw(holder, env, "reset", "-q", "--keep", "HEAD~1") + clone, _ = edited_clone(holder, env, family, tmp_path, + with_branch("main")) + raw(clone, env, "fetch", "-q", "origin", side) + merged = raw(clone, env, "merge", "-q", "--no-edit", "FETCH_HEAD", + check=False) + assert merged.returncode == 1, merged.stdout + merged.stderr + stages = raw(clone, env, "ls-files", "-u", "--", GITMODULES).stdout + assert len(stages.splitlines()) == 3, "premise: .gitmodules is unmerged" + (clone / GITMODULES).write_bytes( + raw(clone, env, "show", f":2:{GITMODULES}").stdout) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", UNMERGED), + (f"members/{OTHER}", UNMERGED), cure=UNMERGED_CURE), + failed_on(MEMBER, OTHER)], result.stderr + assert (f"No url found for submodule path 'members/{MEMBER}'" + in result.stderr) + raw(clone, env, "add", "--", GITMODULES) + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + @pytest.mark.parametrize("set_up", UNNAMED_STOPS.values(), ids=UNNAMED_STOPS) def test_a_recursive_clone_with_no_credential_is_still_the_skip( family, holder, offline, tmp_path, set_up): From fa69ead905ee5d11f50bb91c9cfd71fab84bfcda Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 11:40:37 +0000 Subject: [PATCH 09/20] bootstrap.py and validate-family.py: a `.gitmodules` value git dies on as it reads the file fails `make bootstrap` naming the key, and the family gate names it in every mode, so `update = None` no longer ends `family bootstrap ok` (#768) Git 2.43 dies reading a `.gitmodules` that gives a submodule a `path`, `url`, `update`, `ignore` or `branch` with no value (a `branch` with none crashes it, exit 139), an `update` other than `none`, `checkout`, `rebase` or `merge`, a `shallow` that is no boolean, or a `fetchRecurseSubmodules` that is neither a boolean nor `on-demand`, before `git submodule update` checks out any member. The family holder's bootstrap listed the file with `git config --list`, which reads all of them, and ended `family bootstrap ok`; the gate refused `member-uninitialized` members checked, naming the update that dies, and said `family ok` under `--no-members`. bootstrap.py (the family root's, pinned): - `gitmodules_sections` checks each entry against git's rules (`refused_value`; a boolean is read by git itself, `git config --type=bool`). Where one would make git die it returns those entries and no sections, and `unfetchable` gives every row not checked out the reason "git cannot read .gitmodules: `` is ``" (or "has no value"). The cure names each key and what git takes for it. Where every member was checked out, the run fails on `.gitmodules` all the same, since git updated none of them. - Where the work tree has no `.gitmodules`, it reads the index's, then HEAD's, with `git config --blob`, as git does. A key with no value ends the reading of such a blob, where git calls the line bad and goes on. - `init_stop` reads a section's own `submodule..active` before `submodule.active`'s patterns, and matches a plain path pattern, as `git clone --recurse-submodules=` writes one, to that path and what is under it. A pattern with pathspec magic or a glob keeps the old reading. validate-family.py (the family root's, pinned): - A new finding, `gitmodules-value-refused`, for each such key in the work tree's `.gitmodules` and in HEAD's, asked before any member check and in every mode. Its remedy prints `git config -f .gitmodules --unset-all submodule..`, one command to a line, writes a member's `path` or `url` back where taking it out would leave none, says which values git takes, and ends with the commit. It says which file gives the value; where HEAD's alone does, it says to commit the work tree's, and where the work tree has no `.gitmodules`, it puts HEAD's back first. - Where the work tree's file gives such a value, `gitmodules-path-twice` and `gitmodules-path-unnamed` read nothing from it, as git reads nothing, so no command they print meets one of its lines. Where either file gives one, a member that is not checked out is not refused `member-uninitialized`: the finding fails the run. tests/test_family_gitmodules_values.py: every row of the issue's table and `branch` with no value, in a plain clone, through bootstrap and the gate in three modes, the printed commands typed; the no-misfire values; the index's and HEAD's file; `submodule..active` and the subset's patterns; and which file gives the value. tests/test_validate_family_without_git.py: with no git and no work tree `.gitmodules`, the first check to run git is now the reading of HEAD's `.gitmodules`, so the `git-not-runnable` refusal names that read, where it named the gitlink listing. Exit 2 and the remedy are as they were. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 270 ++++++-- .../family-root/scripts/validate-family.py | 244 +++++++- tests/test_family_gitmodules_values.py | 588 ++++++++++++++++++ tests/test_validate_family_without_git.py | 14 +- 4 files changed, 1049 insertions(+), 67 deletions(-) create mode 100644 tests/test_family_gitmodules_values.py diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 6dce353f..92ad4cc8 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -40,10 +40,16 @@ for (a stray case variant, a member mounted by hand), since git's init dies at the first such gitlink it asks about, before it checks out any member. Which section and which url git reads is its own - parser's answer, not `git config`'s (`gitmodules_sections`), and - while a merge leaves `.gitmodules` unmerged in the index git reads - none at all: every such member is said to be that, and the cure is - the file resolved and added. Git exits 0 over the second where a + parser's answer, not `git config`'s (`gitmodules_sections`), from + the work tree's file or, where the work tree has none, the index's, + then HEAD's, and while a merge leaves `.gitmodules` unmerged in the + index git reads none at all: every such member is said to be that, + and the cure is the file resolved and added. Where the file gives a + submodule a value git dies on as it reads it (`refused_value`, #768), + every such member is said to be that, naming each key and its value, + and the cure is the key given a value git takes or its line taken + out; where every member was checked out the run fails all the same, + since git updated none of them. Git exits 0 over the second where a recursive clone ran first, and 128 (`No url found for submodule path`) in a plain clone; a missing credential exits non-zero too, so what the holder records is the test, never git's words. The line @@ -66,7 +72,8 @@ the family Makefile's `validate` target uses. (c) EXITS 0 unless a member's own command failed, or (a) left a member not - checked out that no credential would check out. Any other member that + checked out that no credential would check out, or found a value in + `.gitmodules` that git cannot read (#768). Any other member that is not checked out is REPORTED and skipped, not a failure: it is the state a missing credential leaves, and it is already reported in (a). @@ -100,14 +107,30 @@ #: Why (a) left a row's member NOT CHECKED OUT where no credential would #: check it out (#648), in the order `unfetchable` asks. `STOPPED` is -#: filled with the gitlink git's init died at (`init_stop`). +#: filled with the gitlink git's init died at (`init_stop`), and `REFUSED` +#: with each entry of `.gitmodules` git refused (`reason`). UNMERGED = f"git reads no {GITMODULES} while a merge leaves it unmerged" +REFUSED = f"git cannot read {GITMODULES}: {{}}" NO_GITLINK = "no gitlink is recorded at that path" NO_SECTION = f"no section of {GITMODULES} gives that path" NO_URL = "git has no url for that path" EXITED_0 = "git exited 0 and checked nothing out there" STOPPED = "git stopped at {}, a gitlink it has no url for" +#: What git 2.43 dies on in a `.gitmodules` it reads (#768), as +#: `parse_config` in its submodule-config.c reads a submodule's key: these +#: keys with no value (a `branch` with none crashes it), and an `update` +#: other than these four. A `shallow` must be a boolean and a +#: `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`). +NO_VALUE_REFUSED = frozenset({"path", "url", "update", "ignore", "branch"}) +UPDATE_VALUES = ("none", "checkout", "rebase", "merge") +#: What git takes for a key whose value it refused, as (a)'s cure says it. +TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", + "shallow": "a boolean, `true` or `false`", + "fetchrecursesubmodules": "a boolean or `on-demand`", + "ignore": "one of `untracked`, `dirty`, `all` or `none`", + "path": "a path", "url": "a url", "branch": "a branch name"} + #: The two environment variables a leg/member credential may arrive in, in the #: order the `validate` workflow resolves them. An App-minted installation #: token is short-lived and per-owner; the PAT is the standing fallback. @@ -130,6 +153,10 @@ def credential() -> tuple[str, str]: def fetch_members(root: Path, rows: list[dict]) -> list[str]: """(a) `git submodule update --init --recursive`, authenticated or not. + Where every member is checked out and git died on a value in + `.gitmodules`, so it updated none of them, the one failure line it + returns is for `.gitmodules` (#768). + Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for the run to end FAILED on (#648); every member and leg is said to be @@ -160,11 +187,14 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: left, stopper, modules = unfetchable(root, rows, proc.returncode == 0) if left: print(" NOT CHECKED OUT, and not for want of a credential: " - + ", ".join(f"{spelled(path)} (" - + (STOPPED.format(spelled(stopper)) - if why == STOPPED else why) + ")" + + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" for _, path, why in left) + ". " + cures(left, stopper, rows, modules), file=sys.stderr) + elif modules.refused: + print(" NO MEMBER UPDATED, and not for want of a credential: " + + reason(REFUSED, stopper, modules) + ". " + + cures(left, stopper, rows, modules), file=sys.stderr) + return [f"{GITMODULES} (git cannot read it)"] elif proc.returncode == 0: print(" every member and leg fetched") return [f"{spelled(row.get('project') or '?')} (not checked out)" @@ -178,6 +208,10 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, - A `.gitmodules` a merge left unmerged in the index, which git reads nothing from (`Gitmodules.unmerged`): the file resolved and added. + - A `.gitmodules` that gives a value git dies on (`Gitmodules.refused`, + #768): each key given a value git takes (`TAKES`), or its line taken + out, and the file committed, with `validate-family.py --no-members`, + whose `gitmodules-value-refused` prints the commands. - A gitlink or a section the holder lacks at the row's path: `validate-family.py`, which names it and the commit that cures it, `member-gitlink-absent` (with the move, for a gitlink recorded only @@ -204,6 +238,15 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, if UNMERGED in whys: said.append(f"Resolve the merge in {GITMODULES} and `git add " f"{GITMODULES}`, then run this again.") + if modules.refused: + given = "; ".join( + f"give `{spelled(key)}` a value git takes, " + f"{TAKES[key.rpartition('.')[2]]}, or take that line out" + for key in dict.fromkeys(key for key, _ in modules.refused)) + said.append(f"{given[:1].upper()}{given[1:]}, and commit " + f"{GITMODULES}. `{PYTHON} " + "scripts/validate-family.py --no-members` prints the " + "commands.") if whys & {NO_GITLINK, NO_SECTION}: said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " "and the commit that cures it.") @@ -243,6 +286,20 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, return " ".join(said) +def reason(why: str, stopper: str | None, modules: Gitmodules) -> str: + """`why` as (a)'s line says it: `STOPPED` names the gitlink git stopped + at, and `REFUSED` each entry of `.gitmodules` git refused, as "`` + is ``", or "has no value" (#768).""" + if why == STOPPED: + return STOPPED.format(spelled(stopper)) + if why == REFUSED: + return REFUSED.format(" and ".join( + f"`{spelled(key)}` " + ("has no value" if value is None + else f"is `{spelled(value)}`") + for key, value in modules.refused)) + return why + + class Gitmodules(NamedTuple): """`.gitmodules` as (a) asks it (#648), read by `gitmodules_sections`. @@ -252,29 +309,39 @@ class Gitmodules(NamedTuple): `{path: name}`, the section git reads for each path, and `urls` is `{name: url}`, the url git takes from each, as git's own parser has them. `unmerged` says a merge left the file unmerged in the index, and - then git reads nothing from it, and the three are empty.""" + `refused` holds each `(submodule.., value)` git dies on as + it reads the file, None for a key with no value (#768): git reads + nothing from it then, and the three are empty.""" given: Counter read: dict[str, str] urls: dict[str, str] unmerged: bool = False + refused: tuple[tuple[str, str | None], ...] = () def gitmodules_sections(root: Path) -> Gitmodules: """`.gitmodules` as `git submodule update` reads it: the work tree's - file, which a fresh clone takes from HEAD. Nothing where there is no - such file, or one git cannot read: git has no url for any submodule - then either. Nothing, and `unmerged`, where the index holds the file - only at a merge's conflict stages (`git ls-files -u`): git reads no - `.gitmodules` at all then, whatever the work tree holds + file, which a fresh clone takes from HEAD, or, where the work tree has + none, as a deletion not yet committed or a sparse checkout leaves it, + the index's, then HEAD's (`config_from_gitmodules`, #768). Nothing + where none holds one, or git cannot read it: git has no url for any + submodule then either. Nothing, and `unmerged`, where the index holds + the file only at a merge's conflict stages (`git ls-files -s`): git + reads no `.gitmodules` at all then, whatever the work tree holds (`repo_read_gitmodules`), and its init dies at the first gitlink. + Nothing, and `refused`, where git dies on an entry as it reads the + file (`refused_value`, #768), before it checks out any member. `read` and `urls` keep git 2.43's own rules (submodule-config.c), which drop what `git config --list` keeps. Entry by entry, in file order: - a section whose name is empty or has a `..` component, split on `/` and on a backslash, is ignored whole (git warns `ignoring suspicious - submodule name`); + submodule name`), a value git would die on included; + - a key git refuses with no value ends the reading of the index's or + HEAD's file, a blob, where `bad config line` is an error and no die: + git reads nothing after it; - a `path` or `url` that begins with `-` is dropped, and the value before it stands (`may be interpreted as a command-line option`); - where two sections give one path, the later in the file decides, as @@ -283,61 +350,120 @@ def gitmodules_sections(root: Path) -> Gitmodules: section gave that one too: git then reads no section there.""" nothing = Gitmodules(Counter(), {}, {}) try: - stages = git_out(["ls-files", "-u", "-z", "--", GITMODULES], + stages = git_out(["ls-files", "-s", "-z", "--", GITMODULES], cwd=root, binary=True) except Refusal: stages = b"" - if any(record.partition(b"\t")[2] == GITMODULES.encode() - for record in stages.split(b"\0")): - return nothing._replace(unmerged=True) - if not os.path.lexists(root / GITMODULES): - return nothing + source = ["--blob", f"HEAD:{GITMODULES}"] + for record in stages.split(b"\0"): + meta, _, name = record.partition(b"\t") + fields = meta.split() + if name != GITMODULES.encode() or len(fields) != 3: + continue + if fields[2] != b"0": + return nothing._replace(unmerged=True) + source = ["--blob", fields[1].decode("ascii")] + if os.path.lexists(root / GITMODULES): + source = ["--file", GITMODULES] try: - listed = git_out(["config", "-z", "--file", GITMODULES, "--list"], - cwd=root, binary=True) + listed = git_out(["config", "-z", *source, "--list"], cwd=root, + binary=True) except Refusal: return nothing sections: dict[str, dict[str, str]] = {} read: dict[str, str] = {} held: dict[str, str] = {} urls: dict[str, str] = {} + refused: dict[tuple[str, str | None], None] = {} for record in listed.split(b"\0"): - key, _, value = record.decode("utf-8", - "surrogateescape").partition("\n") + key, valued, value = record.decode("utf-8", + "surrogateescape").partition("\n") section, _, variable = key.rpartition(".") if not section.startswith("submodule."): continue name = section[len("submodule."):] + ignored = not name or ".." in name.replace("\\", "/").split("/") + if (not ignored and not valued and source[0] == "--blob" + and variable in NO_VALUE_REFUSED - {"branch"}): + break sections.setdefault(name, {})[variable] = value - if (not name or ".." in name.replace("\\", "/").split("/") - or value.startswith("-")): + if ignored: continue - if variable == "path": + if refused_value(root, variable, value if valued else None): + refused[(key, value if valued else None)] = None + elif value.startswith("-"): + continue + elif variable == "path": if name in held: read.pop(held[name], None) held[name] = value read[value] = name elif variable == "url": urls[name] = value + if refused: + return nothing._replace(refused=tuple(refused)) return Gitmodules(Counter(keys["path"] for keys in sections.values() if "path" in keys), read, urls) -def submodule_config(root: Path) -> set[str]: +def refused_value(root: Path, key: str, value: str | None) -> bool: + """Whether git 2.43 dies reading a `.gitmodules` that gives a submodule + `value` for `key`, None for no value (#768), as `parse_config` in its + submodule-config.c reads the key: a `path`, `url`, `update` or + `ignore` with no value (`bad config line`), and a `branch` with none + (it crashes); an `update` other than `none`, `checkout`, `rebase` or + `merge`, in that case, a `!command` included (`invalid value`); a + `shallow` that is no boolean (`bad boolean config value`); and a + `fetchrecursesubmodules` that is neither a boolean nor `on-demand`. An + `ignore` it does not know is a warning, and any other key takes any + value.""" + if value is None: + return key in NO_VALUE_REFUSED + if key == "update": + return value not in UPDATE_VALUES + if key == "shallow" or (key == "fetchrecursesubmodules" + and value != "on-demand"): + return git_bool(root, value) is None + return False + + +def git_bool(root: Path, value: str | None) -> bool | None: + """`value` read as a boolean by git itself, which reads one so for + every key (`git config --type=bool`, its `git_parse_maybe_bool`): + `true`, `yes`, `on` in any case, a number other than 0, or no value at + all are true, and `false`, `no`, `off`, 0 and an empty value false. + None where git takes it for no boolean, and dies.""" + if value is None: + return True + try: + answer = git_out(["-c", f"probe.value={value}", "config", + "--type=bool", "--get", "probe.value"], cwd=root) + except Refusal: + return None + return answer == "true" + + +def submodule_config(root: Path) -> dict[str, list[str | None]]: """Every `submodule.` key this repository's git config sets, in every - scope git reads, as git's init asks them: `submodule.active`, and - `submodule..url` as an earlier init writes it. None where git - lists none.""" + scope git reads, with each value it gives, None for no value, as git's + init asks them: `submodule.active` and its patterns, a section's own + `submodule..active` (#768), and `submodule..url` as an + earlier init writes it. Empty where git lists none.""" try: config = git_out(["config", "-z", "--get-regexp", r"^submodule\."], cwd=root, binary=True) except Refusal: - return set() - return {record.decode("utf-8", "surrogateescape").partition("\n")[0] - for record in config.split(b"\0")} + return {} + found: dict[str, list[str | None]] = {} + for record in filter(None, config.split(b"\0")): + key, valued, value = record.decode("utf-8", + "surrogateescape").partition("\n") + found.setdefault(key, []).append(value if valued else None) + return found -def has_url(modules: Gitmodules, config: set[str], path: str) -> bool: +def has_url(modules: Gitmodules, config: dict[str, list[str | None]], + path: str) -> bool: """Whether git's init finds a url for the gitlink at `path`: in the section git reads for that path (`Gitmodules.read`), or in the git config under that section's name, as an earlier init writes it. Where @@ -347,8 +473,8 @@ def has_url(modules: Gitmodules, config: set[str], path: str) -> bool: or f"submodule.{name}.url" in config) -def init_stop(root: Path, modules: Gitmodules, config: set[str] - ) -> str | None: +def init_stop(root: Path, modules: Gitmodules, + config: dict[str, list[str | None]]) -> str | None: """The gitlink `git submodule update --init` dies at, or None: the first in the index, in index order, that its init step asks about and has no url for (`No url found for submodule path`, exit 128), before it @@ -357,11 +483,11 @@ def init_stop(root: Path, modules: Gitmodules, config: set[str] ASKED AS GIT 2.43 ASKS IT, of the index, `modules` and `config`, this repository's git config. Every gitlink the index records is asked, but where the config sets `submodule.active`, as a recursive clone sets it - (`.`), only those a section git reads gives a path: git counts no other - active. A gitlink git has no url for (`has_url`) stops it. So a stray - case variant or a member mounted by hand stops a plain clone's update, - and leaves a recursive clone's to the credential. Read nothing, it - answers None.""" + (`.`), only those git counts active (`active`), and never one no + section git reads gives a path. A gitlink git has no url for + (`has_url`) stops it. So a stray case variant or a member mounted by + hand stops a plain clone's update, and leaves a recursive clone's to + the credential. Read nothing, it answers None.""" try: index = git_out(["ls-files", "-s", "-z"], cwd=root, binary=True) except Refusal: @@ -371,13 +497,54 @@ def init_stop(root: Path, modules: Gitmodules, config: set[str] if meta.split()[:1] != [GITLINK_MODE]: continue path = entry.decode("utf-8", "surrogateescape") - if path not in modules.read and "submodule.active" in config: + if ("submodule.active" in config + and not active(root, modules, config, path)): continue if not has_url(modules, config, path): return path return None +def active(root: Path, modules: Gitmodules, + config: dict[str, list[str | None]], path: str) -> bool: + """Whether git's init counts the gitlink at `path` active where the + config sets `submodule.active` (`is_tree_submodule_active` in git + 2.43's submodule.c): never where no section git reads gives that path; + else as the section's own `submodule..active` says, its last + value read as a boolean (`git_bool`), where the config sets one that + is (#768); else where one of `submodule.active`'s patterns names it + (`pattern_names`).""" + name = modules.read.get(path) + if name is None: + return False + own = config.get(f"submodule.{name}.active") + flag = git_bool(root, own[-1]) if own else None + if flag is not None: + return flag + return any(pattern_names(pattern, path) + for pattern in config["submodule.active"]) + + +def pattern_names(pattern: str | None, path: str) -> bool: + """Whether a `submodule.active` pattern names `path` (#768). A plain + path, as `git clone --recurse-submodules=` writes one, names + itself and every path under it, as git's pathspec does, and `.` names + every path. A pattern with pathspec magic (a leading `:`), a glob + character or a backslash, or with an empty, `.` or `..` component past + a leading `./`, is read as naming every path, as every pattern was + before: this does not match those as git does.""" + if (pattern is None or pattern.startswith(":") + or any(char in pattern for char in "*?[\\")): + return True + while pattern.startswith("./"): + pattern = pattern[2:] + pattern = pattern.rstrip("/") + if pattern in ("", ".") or any(part in ("", ".", "..") + for part in pattern.split("/")): + return True + return path == pattern or path.startswith(pattern + "/") + + def unfetchable(root: Path, rows: list[dict], exited_0: bool ) -> tuple[list[tuple[dict, str, str]], str | None, Gitmodules]: @@ -388,7 +555,8 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool Asked of this holder's own records, never of git's words: any row at all while a merge leaves `.gitmodules` unmerged in the index, since git - then reads no section and asks for no credential; a row at + then reads no section and asks for no credential, or while the file + gives a value git dies on (`Gitmodules.refused`, #768); a row at whose path the holder records no gitlink (`recorded_gitlink`, as `validate-family.py`'s `member-gitlink-absent` reads one), or whose gitlink no section of `.gitmodules` gives a path (as its @@ -399,20 +567,26 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool died at a gitlink it has no url for, any row at all, since git then checked out no member and asked for no credential. Any other row not checked out is the missing credential's, and stays the skip it always - was. A path two rows give is said once.""" + was. A path two rows give is said once. Where every row's member is + checked out and git did not exit 0, `.gitmodules` is read all the + same, for `fetch_members` to ask `refused` of.""" left: dict[str, dict] = {} for row in rows: if not (member_path(root, row) / ".git").exists(): left.setdefault(row_path(row), row) - if not left: + if not left and exited_0: return [], None, Gitmodules(Counter(), {}, {}) modules = gitmodules_sections(root) + if not left: + return [], None, modules config = submodule_config(root) stopper = None if exited_0 else init_stop(root, modules, config) found: list[tuple[dict, str, str]] = [] for path, row in left.items(): if modules.unmerged: found.append((row, path, UNMERGED)) + elif modules.refused: + found.append((row, path, REFUSED)) elif recorded_gitlink(root, path) is None: found.append((row, path, NO_GITLINK)) elif not modules.given[path]: diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index cee36ce5..1054f51a 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -55,6 +55,15 @@ clone fetches that path from only one of them, so the other is a url the next edit can make the one. It reads that file alone, so `--no-members` asks it too. + - that no section of `.gitmodules` gives a value git dies on as it reads + the file, before `git submodule update` checks out any member, the + `gitmodules-value-refused` finding (#768): a `path`, `url`, `update`, + `ignore` or `branch` with no value, an `update` other than `none`, + `checkout`, `rebase` or `merge`, a `shallow` that is no boolean, or a + `fetchRecurseSubmodules` that is neither a boolean nor `on-demand`. It + reads the work tree's file and HEAD's, so `--no-members` asks it too, + and where it is found no member that is not checked out is refused as + uninitialized: git checked none out. - the REVERSE of every member check (#361): that each gitlink this repository records in `members/` (the index at stage 0, else HEAD), and each path a `.gitmodules` section gives, is a path some row names. A @@ -90,9 +99,10 @@ 1 a FINDING: drift a reviewer can act on 2 a REFUSAL: the question could not be asked — no manifest, an uninitialized member (but for one no section of HEAD's `.gitmodules` - gives: that is `member-gitmodules-absent`, exit 1), an unreadable - pin, no git to run. An unanswerable question is never an implicit - pass. + gives, and any while `.gitmodules` gives a value git dies on: those + are `member-gitmodules-absent` and `gitmodules-value-refused`, exit + 1), an unreadable pin, no git to run. An unanswerable question is + never an implicit pass. """ from __future__ import annotations @@ -111,7 +121,8 @@ from repo_shape import ( # noqa: E402 COMMIT_RE, PROJECT_ID_RE, PYTHON, SHA256_RE, TREE_DIGEST_DEFINITION, VISIBILITY_CHOICES, NamingPolicy, Refusal, file_sha256, find_repo_root, - git_out, load_yaml, recorded_gitlink, repo_basename, tree_digest, + git_out, load_yaml, recorded_gitlink, repo_basename, spelled, + tree_digest, ) DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") @@ -994,12 +1005,20 @@ def _gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: not read: the missing git is one fact about the machine, and every check here that meets it says so under one code. """ + return _sections_listed(_gitmodules_listed(root)) + + +def _gitmodules_listed(root: Path) -> bytes: + """What `git config -z --file .gitmodules --list` prints of the work + tree's file, or nothing where there is none, read as + `_gitmodules_sections` says: what it and `_check_gitmodules_values` + read (#768).""" if not os.path.lexists(root / GITMODULES): - return {} + return b"" with _git_runs(f"{GITMODULES} could not be read as git reads it"): try: - listed = git_out(["config", "-z", "--file", GITMODULES, "--list"], - cwd=root, binary=True) + return git_out(["config", "-z", "--file", GITMODULES, "--list"], + cwd=root, binary=True) except Refusal as exc: raise Refusal( "gitmodules-unreadable", @@ -1009,7 +1028,6 @@ def _gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: "holder's submodules can be cloned from it as it stands. Mend " f"the file by hand (`git config -f {GITMODULES} --list` says " "where it breaks), then run this again.") from exc - return _sections_listed(listed) def _sections_listed(listed: bytes) -> dict[str, dict[str, str]]: @@ -1042,6 +1060,14 @@ def _head_gitmodules(root: Path no pathspec, as `_member_gitlinks` lists HEAD. One git cannot read is `gitmodules-unreadable`, as the work tree's is. """ + head = _head_listed(root) + return None if head is None else (head[0], _sections_listed(head[1])) + + +def _head_listed(root: Path) -> tuple[bool, bytes] | None: + """`(whether HEAD holds a .gitmodules, what `git config -z --blob + --list` prints of it)`, read as `_head_gitmodules` says: + what it and `_check_gitmodules_values` read (#768).""" with _git_runs(f"HEAD's {GITMODULES} could not be read as git reads it"): try: tree = git_out(["ls-tree", "-z", "HEAD"], cwd=root, binary=True) @@ -1053,10 +1079,10 @@ def _head_gitmodules(root: Path if name == GITMODULES.encode() and meta.split()[1:2] == [b"blob"]), None) if blob is None: - return False, {} + return False, b"" try: - listed = git_out(["config", "-z", "--blob", blob, "--list"], - cwd=root, binary=True) + return True, git_out(["config", "-z", "--blob", blob, "--list"], + cwd=root, binary=True) except Refusal as exc: raise Refusal( "gitmodules-unreadable", @@ -1066,7 +1092,185 @@ def _head_gitmodules(root: Path "holder's submodules can be cloned from it. Mend the file by " f"hand (`git config -f {GITMODULES} --list` says where it " "breaks), commit it, then run this again.") from exc - return True, _sections_listed(listed) + + +#: Where `gitmodules-value-refused` says a value is, by whether the work +#: tree's file and HEAD's give it (#768). +GIVEN_WHERE = {(True, True): "here and in HEAD, which a clone reads", + (True, False): "in the work tree, not in HEAD", + (False, True): "in HEAD, which a clone reads, not in the " + "work tree"} + +#: What git takes for a `submodule..` whose value it dies on as +#: it reads `.gitmodules` (#768), as `gitmodules-value-refused` says it. +GIT_TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", + "shallow": "a boolean, `true` or `false`", + "fetchrecursesubmodules": "a boolean or `on-demand`", + "ignore": "one of `untracked`, `dirty`, `all` or `none`", + "path": "a path", "url": "a url", "branch": "a branch name"} + + +def _check_gitmodules_values(root: Path, members: list, + report: Report) -> tuple[bool, bool]: + """No section of `.gitmodules` gives a value git dies on as it reads + the file, the `gitmodules-value-refused` finding (#768). Returns + whether the work tree's file gives one, and whether the work tree's + or HEAD's does. + + Git 2.43 dies reading such a file, before `git submodule update` + checks out any member, so `make bootstrap` checks none out either. In + a plain clone the members modes refused `member-uninitialized` for the + first row, naming that same update, and `--no-members` said `family + ok`. The rules are `parse_config`'s in git's submodule-config.c + (`_values_refused`). The work tree's file is read, which git reads + here, and HEAD's, which a clone reads, as `member-gitmodules-absent` + reads it: no member needs to be checked out, so every mode asks it. + One finding for each key, naming each value git dies on there. + + The remedy is by key, one command to a line, on the work tree's file, + as `gitmodules-path-unnamed` takes a key out: `--unset-all` takes out + every line of it, since git reads a section without one, and + `.gitmodules` is then committed where HEAD's gives it too. A member's + own section gets its `path` or `url` written back, as it last gave + one, or else as `family.py add` writes it, named after its path, with + `https://github.com/` and the row's `repository:` (`_written_back`). + Where HEAD's file alone gives the value, the work tree's is what to + commit, and where the work tree has no `.gitmodules`, the first such + finding puts HEAD's back first.""" + present = os.path.lexists(root / GITMODULES) + here = _submodule_entries(_gitmodules_listed(root)) + head = _head_listed(root) + there = _submodule_entries(head[1]) if head else [] + in_work = _values_refused(root, here) + in_head = _values_refused(root, there) + rows = {f"{DEFAULT_MEMBERS_DIR}/{row['project']}": row.get("repository") + for row in members if isinstance(row, dict) + and isinstance(row.get("project"), str) and row["project"]} + put_back = present + for name, key in dict.fromkeys([*in_work, *in_head]): + full = f"submodule.{name}.{key}" + values = dict.fromkeys([*in_work.get((name, key), []), + *in_head.get((name, key), [])]) + where = GIVEN_WHERE[(name, key) in in_work, (name, key) in in_head] + said = (f"{GITMODULES} gives `{spelled(full)}` " + " and ".join( + "no value" if value is None else f"the value `{spelled(value)}`" + for value in values) + f" {where}, and git dies reading a file " + "that does, before `git submodule update` checks out any " + "member, so neither that nor make bootstrap checks one out. Git " + f"takes {GIT_TAKES[key]} for that key.") + if present and (name, key) not in in_work: + report.finding("gitmodules-value-refused", + f"{said} Stage and commit the work tree's " + f"{GITMODULES}, which does not give it.") + continue + lines = [] if put_back else [f"git checkout HEAD -- {GITMODULES}"] + lines.append(f"git config -f {GITMODULES} --unset-all " + f"{shlex.quote(full)}") + back = _written_back(name, key, here if present else there, rows) + if back is not None: + lines.append(f"git config -f {GITMODULES} {shlex.quote(full)} " + f"{back}") + report.finding( + "gitmodules-value-refused", + f"{said} " + + ("" if put_back else f"The work tree holds no {GITMODULES}, " + "so the first line below puts HEAD's back. ") + + "Take the key out" + + (f" and write its {key} back" if back is not None else "") + + " by hand, in a POSIX shell" + + (f", then stage and commit {GITMODULES}" + if (name, key) in in_head else "") + + ":" + "".join(f"\n {line}" for line in lines)) + put_back = True + return bool(in_work), bool(in_work or in_head) + + +def _submodule_entries(listed: bytes) -> list[tuple[str, str, str | None]]: + """`(name, key, value)` for each `submodule..` in what `git + config -z --list` printed, in file order, None for a key with no value + (#768). A section git ignores whole, whose name is empty or has a `..` + component, split on `/` and on a backslash, gives none.""" + entries: list[tuple[str, str, str | None]] = [] + for record in listed.split(b"\0"): + key, valued, value = record.decode("utf-8", + "surrogateescape").partition("\n") + section, _, variable = key.rpartition(".") + name = section[len("submodule."):] + if (section.startswith("submodule.") and name + and ".." not in name.replace("\\", "/").split("/")): + entries.append((name, variable, value if valued else None)) + return entries + + +def _values_refused(root: Path, entries: list[tuple[str, str, str | None]] + ) -> dict[tuple[str, str], list[str | None]]: + """`{(name, key): [each value]}` of `entries` (`_submodule_entries`) + that git 2.43 dies on as it reads `.gitmodules` (#768), as + `parse_config` in its submodule-config.c reads a key: a `path`, `url`, + `update` or `ignore` with no value (`bad config line`), a `branch` + with none (it crashes), an `update` other than `none`, `checkout`, + `rebase` or `merge`, in that case, a `!command` included (`invalid + value`), a `shallow` that is no boolean, and a `fetchrecursesubmodules` + that is neither a boolean nor `on-demand`, as git itself reads a + boolean (`git config --type=bool`). An `ignore` git does not know is + a warning, and any other key takes any value.""" + refused: dict[tuple[str, str], list[str | None]] = {} + for name, key, value in entries: + if value is None: + dies = key in {"path", "url", "update", "ignore", "branch"} + elif key == "update": + dies = value not in {"none", "checkout", "rebase", "merge"} + else: + dies = (key == "shallow" or (key == "fetchrecursesubmodules" + and value != "on-demand") + ) and not _git_takes_bool(root, value) + if dies: + refused.setdefault((name, key), []).append(value) + return refused + + +def _git_takes_bool(root: Path, value: str) -> bool: + """Whether git reads `value` as a boolean, asked of git itself (`git + config --type=bool`), which reads `shallow` and + `fetchRecurseSubmodules` so (#768).""" + with _git_runs(f"a value in {GITMODULES} could not be read as git " + "reads a boolean"): + try: + git_out(["-c", f"probe.value={value}", "config", "--type=bool", + "--get", "probe.value"], cwd=root) + except Refusal: + return False + return True + + +def _written_back(name: str, key: str, + entries: list[tuple[str, str, str | None]], + rows: dict[str, object]) -> str | None: + """What `gitmodules-value-refused` writes back for the `path` or `url` + of the section `name` once its key is out (#768), POSIX-quoted, or None + for any other key, and for a section whose path no row's + `members/` is: the value the section last gave it that git + takes, or else, as `family.py add` writes a section named after its + path, that path, or `https://github.com/` and the row's `repository:`, + `` where that is not `/`. The section's path is the one + it last gave, or its name where it gave none.""" + if key not in {"path", "url"}: + return None + gave = {wanted: [value for section, variable, value in entries + if section == name and variable == wanted and value + and not value.startswith("-")] + for wanted in ("path", "url")} + path = gave["path"][-1] if gave["path"] else name + if path not in rows: + return None + if gave[key]: + return shlex.quote(gave[key][-1]) + if key == "path": + return shlex.quote(path) + repository = rows[path] + return (shlex.quote(f"https://github.com/{repository}.git") + if isinstance(repository, str) and QUALIFIED_RE.match(repository) + else "") def _check_gitmodules_paths(sections: dict[str, dict[str, str]], @@ -1560,10 +1764,23 @@ def _check_members(root: Path, members: list, no_members: bool, `member-uninitialized`, naming that same update, and the later row's `member-gitlink-absent`, which names the move that cures both, was never reached. + + A VALUE GIT DIES ON IN `.gitmodules` (#768) is asked first of the + checks that read that file, and where the work tree's file gives one, + `gitmodules-path-twice` and `gitmodules-path-unnamed` read nothing from + it, as git reads nothing: no command they would print for a section + then meets one of its own lines. Where + either file gives one, git checks out no member, so a member that is + not checked out is asked nothing, as one `member-gitmodules-absent` + names: the finding fails the run, where `member-uninitialized` named + the update that dies. """ _check_member_names_unique(members, report) _check_member_ids_unique(root, members, report) sections = _gitmodules_sections(root) + refused_here, refused = _check_gitmodules_values(root, members, report) + if refused_here: + sections = {} _check_gitmodules_paths(sections, report) named = _row_paths(members) gitlinks = _member_gitlinks(root) @@ -1587,7 +1804,8 @@ def _check_members(root: Path, members: list, no_members: bool, if asked is not None: gitlinked.append(asked) for asked in gitlinked: - _check_member_checkout(root, asked, report, unregistered) + _check_member_checkout(root, asked, report, + set(gitlinks) if refused else unregistered) if not members: report.note("no members yet: a family with none is empty, not " "wrong") diff --git a/tests/test_family_gitmodules_values.py b/tests/test_family_gitmodules_values.py new file mode 100644 index 00000000..26e52e38 --- /dev/null +++ b/tests/test_family_gitmodules_values.py @@ -0,0 +1,588 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A `.gitmodules` value git dies on as it reads the file, in a family +holder's own bootstrap and gate, and where git reads the file from (#768). + +THE ISSUE'S THREE LINES, each reproduced on #682's head f74f640c first: + +- Bootstrap said `family bootstrap ok`, exit 0, where git dies reading + `.gitmodules` before it checks out any member: an `update = None` (the + typo of `update = none`), an `update = !true`, a `url`, `update`, + `ignore` or `branch` with no value (git 2.43 crashes on the last), a + `shallow = maybe`, a `fetchRecurseSubmodules = bogus`. It names every + row with the key and its value now, and fails, where every member was + checked out too. +- The family gate named none of them: members checked and `--pins` + refused `member-uninitialized`, whose remedy is the update that dies, + and `--no-members` said `family ok`. Every mode finds + `gitmodules-value-refused` now, read from the work tree's file and + HEAD's, and its commands take the key out. +- What #682 left approximate: where the work tree has no `.gitmodules`, + git reads the index's, then HEAD's; a section's own + `submodule..active` decides before `submodule.active`'s patterns; + and a plain path pattern names only itself and what is under it. + +THE SAME FAMILY, OFFLINE RULES AND STATES as +`test_family_bootstrap_gate_order.py` (#648), whose helpers are imported: +a plain clone of the holder, `git submodule update` given the fixture's +local members (`fetching`) where a credential is said to be there and not +given them where it is not. Every printed command is typed as printed. +""" + +from __future__ import annotations + +import re + +import pytest + +from repo_shape import PYTHON +from test_family import family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bootstrap_gate_order import ( + CHECKOUT_CURE, EXITED_0, FETCHED, NO_URL, NOT_ALL_FETCHED, SECTION_CURE, + STOPPED, after_first_line, bootstrap, cloned, edited_clone, failed_on, + fetching, not_fetched, type_line, url_edited, without_url, +) +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( + GITMODULES, assert_passes_in_every_mode, commit_all, committed_gitmodules, +) +from test_family_member_path import MEMBER, OTHER, raw +from test_validate_family_unnamed_mounts import commands, findings + +#: The issue's table, `update = None` first, then `branch` with no value, +#: which git 2.43 crashes on (w768's probe): each a line added to OTHER's +#: section after its url, as rv648 (p11) and w648c (p12) added it, the key +#: git lists for it, the value, None for none, and git's update's exit. +REFUSED_LINES = { + "update-None": ("update = None", "update", "None", 128), + "update-command": ("update = !true", "update", "!true", 128), + "url-no-value": ("url", "url", None, 128), + "update-no-value": ("update", "update", None, 128), + "ignore-no-value": ("ignore", "ignore", None, 128), + "shallow-maybe": ("shallow = maybe", "shallow", "maybe", 128), + "fetch-bogus": ("fetchRecurseSubmodules = bogus", + "fetchrecursesubmodules", "bogus", 128), + "branch-no-value": ("branch", "branch", None, 139), +} + +#: What both tools say git takes for each key. +TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", + "shallow": "a boolean, `true` or `false`", + "fetchrecursesubmodules": "a boolean or `on-demand`", + "ignore": "one of `untracked`, `dirty`, `all` or `none`", + "path": "a path", "url": "a url", "branch": "a branch name"} + +#: Where the finding says the value is. +HERE_AND_HEAD = "here and in HEAD, which a clone reads" +WORK_TREE_ONLY = "in the work tree, not in HEAD" +HEAD_ONLY = "in HEAD, which a clone reads, not in the work tree" + + +def with_line(line: str): + """An edit giving OTHER's section `line` after its url.""" + return lambda data: url_edited( + data, OTHER, lambda url: url + f"\t{line}\n".encode()) + + +def full(key: str) -> str: + return f"submodule.members/{OTHER}.{key}" + + +def refused_why(key: str, value: str | None) -> str: + """Why (a) says each row was not checked out.""" + return (f"git cannot read .gitmodules: `{full(key)}` " + + ("has no value" if value is None else f"is `{value}`")) + + +def value_cure(key: str) -> str: + """What (a) says cures it.""" + return (f"Give `{full(key)}` a value git takes, {TAKES[key]}, or take " + f"that line out, and commit .gitmodules. `{PYTHON} " + "scripts/validate-family.py --no-members` prints the commands.") + + +def said(key: str, value: str | None, where: str) -> str: + """`gitmodules-value-refused` up to its remedy.""" + given = "no value" if value is None else f"the value `{value}`" + return (f"FINDING gitmodules-value-refused: .gitmodules gives " + f"`{full(key)}` {given} {where}, and git dies reading a file " + "that does, before `git submodule update` checks out any " + "member, so neither that nor make bootstrap checks one out. Git " + f"takes {TAKES[key]} for that key. ") + + +def taken_out(key: str, value: str | None, *, back: str | None = None, + where: str = HERE_AND_HEAD, put_back: bool = False) -> str: + """`gitmodules-value-refused` WHOLE, with the commands that take the key + out of the work tree's file, after the one that puts HEAD's back where + `put_back`, and write `back` in where it is given.""" + lines = (["git checkout HEAD -- .gitmodules"] if put_back else []) + [ + f"git config -f .gitmodules --unset-all {full(key)}"] + if back is not None: + lines.append(f"git config -f .gitmodules {full(key)} {back}") + return (said(key, value, where) + + ("The work tree holds no .gitmodules, so the first line below " + "puts HEAD's back. " if put_back else "") + + "Take the key out" + + (f" and write its {key} back" if back is not None else "") + + " by hand, in a POSIX shell" + + ("" if where == WORK_TREE_ONLY + else ", then stage and commit .gitmodules") + + ":" + "".join(f"\n {line}" for line in lines)) + + +def url_of(data: bytes) -> str: + """The url `family.py add` wrote in OTHER's section of `data`.""" + (url,) = re.findall(rb'\[submodule "members/' + OTHER.encode() + + rb'"\]\r?\n(?:\t[^\n]*\n)*?\turl = ([^\r\n]*)', data) + return url.decode("utf-8") + + +def assert_the_credential_s_skip(clone, env) -> None: + """Bootstrap with no credential for the members: git fails on their + fetch, not on `.gitmodules`, so they are skipped, and the run is ok.""" + result = bootstrap(clone, env, "--skip-members") + assert result.returncode == 0, result.stdout + result.stderr + assert "FETCH INCOMPLETE" in result.stderr + assert "not for want of a credential" not in result.stderr + + +def typed_and_committed(root, env, finding: str) -> None: + """Every command `finding` prints, typed as printed, then `.gitmodules` + staged and committed, as the finding says.""" + for command in commands(finding): + type_line(root, env, command) + raw(root, env, "add", "--", GITMODULES) + commit_all(root, env, "The key taken out as printed") + + +# --- lines 1 and 2: bootstrap and the gate ---------------------------------- + +@pytest.mark.parametrize("line, key, value, code", REFUSED_LINES.values(), + ids=REFUSED_LINES) +def test_a_value_git_dies_on_fails_bootstrap_and_every_mode_of_the_gate( + family, holder, offline, tmp_path, line, key, value, code): + """THE ISSUE'S LINES 1 AND 2. The line committed, a plain clone with the + credential: git's update dies and checks nothing out, and bootstrap + said ok. It names both rows, the key and its value, and fails. The + gate refused `member-uninitialized` members checked, with the update + that dies for remedy, and said `family ok` under `--no-members`; it + finds `gitmodules-value-refused` in all three modes now. Typed as + printed and committed, which writes back the url a `url` line took + with it, the file is the one `family.py add` wrote, and the same run + checks both members out and passes the gate in every mode.""" + env = fetching(isolated(offline, tmp_path)) + clone, data = edited_clone(holder, env, family, tmp_path, with_line(line)) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode == code, update.stderr + assert not (clone / "members" / MEMBER / ".git").exists(), ( + "premise: git checked nothing out") + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert after_first_line(result) == NOT_ALL_FETCHED + why = refused_why(key, value) + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure(key)), + failed_on(MEMBER, OTHER)], result.stderr + expected = taken_out(key, value, + back=url_of(data) if key == "url" else None) + for extra in ((), ("--pins",), ("--no-members",)): + gate = validate(clone, *extra) + assert gate.returncode == 1, (extra, gate.stdout + gate.stderr) + assert findings(gate) == [expected], extra + typed_and_committed(clone, env, expected) + assert raw(clone, env, "show", f"HEAD:{GITMODULES}").stdout == data + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + assert_passes_in_every_mode(clone, 2) + + +#: Values git takes, which no line or finding names (the issue's no-misfire +#: cases but `update = none`, whose run is the exit-0 skip as before). +TAKEN = {"update-checkout": "update = checkout", + "shallow-true": "shallow = true", + "fetch-on-demand": "fetchRecurseSubmodules = on-demand", + "ignore-bogus": "ignore = bogus"} + + +@pytest.mark.parametrize("line", TAKEN.values(), ids=TAKEN) +def test_a_value_git_takes_is_a_good_holder_s_run(family, holder, offline, + tmp_path, line): + """No misfire. Git takes each (an `ignore` it does not know is a + warning only). With no credential, git fails on the members' fetch and + they are the credential's: skipped, and ok. With it, the plain clone's + update checks both members out, bootstrap is a good holder's, and the + gate passes in every mode, in the clone and in the holder.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, with_line(line)) + assert_the_credential_s_skip(clone, env) + + result = bootstrap(clone, fetching(env), "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert after_first_line(result) == FETCHED + assert result.stderr == "" + assert_passes_in_every_mode(clone, 2) + assert_passes_in_every_mode(holder, 2) + + +def test_update_none_is_still_the_exit_0_skip(family, holder, offline, + tmp_path): + """No misfire. `update = none` is a value git takes: git exits 0 and + skips OTHER, and bootstrap names it with the update that checks it out, + as before (#648's N1), and no line or finding says git cannot read the + file.""" + env = fetching(isolated(offline, tmp_path)) + clone, _ = edited_clone(holder, env, family, tmp_path, + with_line("update = none")) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines() == [ + not_fetched((f"members/{OTHER}", EXITED_0), + cure=CHECKOUT_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)] + assert validate(clone, "--no-members").returncode == 0 + assert_passes_in_every_mode(holder, 2) + + +#: A section under a name git ignores whole, its `update = None` included +#: (`ignoring suspicious submodule name`): no row names its path. +IGNORED = b'[submodule "../x"]\n\tpath = x\n\tupdate = None\n' + + +def test_a_value_in_a_section_git_ignores_whole_is_no_finding( + family, holder, offline, tmp_path): + """No misfire. A section whose name has a `..` component is ignored + whole by git, a value it would die on included. With no credential, + the members are the credential's: skipped, and ok. With it, the plain + clone's update warns and checks both members out, bootstrap is a good + holder's, and the gate names the section as one no row names, not as a + value git refuses.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: data + IGNORED) + assert_the_credential_s_skip(clone, env) + + result = bootstrap(clone, fetching(env), "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert after_first_line(result) == FETCHED + gate = validate(clone, "--no-members") + assert [found.split(":")[0] for found in findings(gate)] == [ + "FINDING gitmodules-path-unnamed"], gate.stderr + + +def test_a_holder_with_every_member_checked_out_fails_on_it_too( + holder, offline, tmp_path): + """Every member checked out, as in the holder itself, and `update = + None` committed: git dies before it updates any of them, so no row is + left to name, and bootstrap said ok over git's `fatal:` line. It says + no member was updated, and why, and fails on `.gitmodules`; typed as + the gate prints it and committed, the run is a good holder's.""" + env = fetching(isolated(offline, tmp_path)) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, with_line("update = None")(data)) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + " NO MEMBER UPDATED, and not for want of a credential: " + f"{refused_why('update', 'None')}. {value_cure('update')}", + "family bootstrap FAILED: .gitmodules (git cannot read it)"] + gate = validate(holder) + assert gate.returncode == 1, gate.stdout + gate.stderr + assert findings(gate) == [taken_out("update", "None")] + typed_and_committed(holder, env, findings(gate)[0]) + + again = bootstrap(holder, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +#: The work tree's `.gitmodules` lost and not committed, its index entry +#: left (git reads that) or staged out too (git reads HEAD's). +def lose_work_tree_file(root, env) -> None: + (root / GITMODULES).unlink() + + +def lose_work_tree_and_index_files(root, env) -> None: + raw(root, env, "rm", "-q", "--cached", "--", GITMODULES) + (root / GITMODULES).unlink() + + +LOST = {"index-read": lose_work_tree_file, + "head-read": lose_work_tree_and_index_files} + + +@pytest.mark.parametrize("lose", LOST.values(), ids=LOST) +def test_where_the_work_tree_has_no_gitmodules_git_reads_the_index_s_or_head_s( + family, holder, offline, tmp_path, lose): + """THE ISSUE'S LINE 3, its first item. A plain clone whose work tree + lost `.gitmodules`, not committed, with no credential: git reads the + index's file, or HEAD's where the index lost it too, and fails on the + members' fetch. Bootstrap read the work tree's alone, said "no section + of .gitmodules gives that path" for each, and failed; the members are + the credential's now, skipped, and the run ends ok.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, recurse=False) + lose(clone, env) + + assert_the_credential_s_skip(clone, env) + + +def test_a_key_with_no_value_ends_git_s_reading_of_the_index_s_file( + family, holder, offline, tmp_path): + """No misfire. A `url` with no value, the last line of the index's + file, which git reads where the work tree has none: in a blob git + calls the line bad and stops reading there, an error and no die, so it + reads every section before it and fails on the fetch for want of a + credential. The members are the credential's: skipped, and ok.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, with_line("url")) + lose_work_tree_file(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert b"bad config line" in update.stderr, update.stderr + + assert_the_credential_s_skip(clone, env) + + +def test_a_value_git_dies_on_in_the_index_s_file_is_named_and_mended( + family, holder, offline, tmp_path): + """`update = None` committed, and the work tree's file lost, not + committed: git reads the index's and dies all the same, and bootstrap + names it. The gate reads HEAD's, and, the work tree holding none, its + first line puts HEAD's back before the key is taken out. Typed as + printed and committed, the same run checks both members out and the + gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + clone, _ = edited_clone(holder, env, family, tmp_path, + with_line("update = None")) + lose_work_tree_file(clone, env) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = refused_why("update", "None") + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure("update")), + failed_on(MEMBER, OTHER)], result.stderr + expected = taken_out("update", "None", where=HEAD_ONLY, put_back=True) + for extra in ((), ("--pins",), ("--no-members",)): + gate = validate(clone, *extra) + assert gate.returncode == 1, (extra, gate.stdout + gate.stderr) + assert findings(gate) == [expected], extra + typed_and_committed(clone, env, expected) + + again = bootstrap(clone, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + assert_passes_in_every_mode(clone, 2) + + +def test_two_keys_with_no_gitmodules_in_the_work_tree_put_head_s_back_once( + holder, offline, tmp_path): + """Two values git dies on committed, and the work tree's file lost: one + finding for each key, and only the first puts HEAD's back, since a + second `git checkout` would put back the key the first took out. Typed + as printed and committed, the holder passes in every mode.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + # Each line goes in right after the url, so `update` comes first. + committed_gitmodules(holder, env, with_line("update = None")( + with_line("shallow = maybe")(data))) + lose_work_tree_file(holder, env) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == [ + taken_out("update", "None", where=HEAD_ONLY, put_back=True), + taken_out("shallow", "maybe", where=HEAD_ONLY)] + for finding in findings(result): + for command in commands(finding): + type_line(holder, env, command) + raw(holder, env, "add", "--", GITMODULES) + commit_all(holder, env, "Both keys taken out as printed") + assert raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout == data + assert_passes_in_every_mode(holder, 2) + + +# --- line 3: submodule..active and submodule.active's patterns --------- + +def test_a_member_whose_own_active_is_false_is_not_where_git_stops( + family, holder, offline, tmp_path): + """THE ISSUE'S LINE 3, its second item. OTHER's section lost its url, + committed; in a plain clone `submodule.active` is `.`, as a recursive + clone sets it, so every member is in the subset, and OTHER's own + `submodule.members/

.active` is false. Git asks that first, inits + MEMBER alone and fails on its fetch for want of a credential, and stops + nowhere; bootstrap said git stopped at OTHER. MEMBER is the + credential's now, and OTHER, which git has no url for, is named with + the section written back.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: without_url(data, OTHER)) + raw(clone, env, "config", "submodule.active", ".") + raw(clone, env, "config", f"submodule.members/{OTHER}.active", "false") + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode != 0, "premise: the fetch failed" + assert b"No url found" not in update.stderr, update.stderr + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{OTHER}", NO_URL), + cure=SECTION_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)], result.stderr + + +#: `submodule.active` patterns: `(the pattern, whether git's init then asks +#: OTHER and dies, (a)'s rows, the rows it fails on)`. A plain path, however +#: spelt, names MEMBER alone, as `git clone --recurse-submodules=` +#: writes it; a leading directory names both. +STOPS_AT_OTHER = (f"members/{MEMBER}", STOPPED.format(path=f"members/{OTHER}")) +PATTERNS = { + "a-path": (f"members/{MEMBER}", False, + ((f"members/{OTHER}", NO_URL),), (OTHER,)), + "a-path-spelt-another-way": (f"./members/{MEMBER}/", False, + ((f"members/{OTHER}", NO_URL),), (OTHER,)), + "a-leading-directory": ("members", True, + (STOPS_AT_OTHER, (f"members/{OTHER}", NO_URL)), + (MEMBER, OTHER)), +} + + +@pytest.mark.parametrize("pattern, stops, left, failed", PATTERNS.values(), + ids=PATTERNS) +def test_a_subset_s_pattern_says_whether_git_stops_at_a_member_outside_it( + family, holder, offline, tmp_path, pattern, stops, left, failed): + """THE ISSUE'S LINE 3, its third item. OTHER's section lost its url, + committed, and the clone's subset is `pattern`, with no credential. + Where the pattern leaves OTHER out, git's init skips it and fails on + MEMBER's fetch, and bootstrap said git stopped at OTHER; MEMBER is the + credential's now. Where it names OTHER, git does die there, and that + is still said.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: without_url(data, OTHER)) + raw(clone, env, "config", "submodule.active", pattern) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert (f"No url found for submodule path 'members/{OTHER}'".encode() + in update.stderr) is stops, update.stderr + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched(*left, cure=SECTION_CURE.format(path=f"members/{OTHER}")), + failed_on(*failed)], result.stderr + + +# --- the gate: which file gives it, and what is written back ----------------- + +def mended_not_committed(root, env, data: bytes) -> None: + """`update = None` committed, the work tree's file mended by hand.""" + committed_gitmodules(root, env, with_line("update = None")(data)) + (root / GITMODULES).write_bytes(data) + + +def edited_not_committed(root, env, data: bytes) -> None: + """`update = None` in the work tree's file alone.""" + (root / GITMODULES).write_bytes(with_line("update = None")(data)) + + +WHERE = { + "head-only": (mended_not_committed, + said("update", "None", HEAD_ONLY) + "Stage and commit the " + "work tree's .gitmodules, which does not give it."), + "work-tree-only": (edited_not_committed, + taken_out("update", "None", where=WORK_TREE_ONLY)), +} + + +@pytest.mark.parametrize("set_up, expected", WHERE.values(), ids=WHERE) +def test_the_gate_says_which_file_gives_it(holder, offline, tmp_path, set_up, + expected): + """In the holder, members checked out: HEAD's file alone gives the + value, the work tree's mended and not committed, and the finding says + to commit that, printing no command; or the work tree's alone, which + git reads here and a clone does not, and the command takes it out with + nothing to commit. Done as it says, the holder passes in every mode.""" + env = isolated(offline, tmp_path) + set_up(holder, env, raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == [expected] + for command in commands(expected): + type_line(holder, env, command) + raw(holder, env, "add", "--", GITMODULES) + raw(holder, env, "commit", "-q", "--allow-empty", "-m", "As it says") + assert_passes_in_every_mode(holder, 2) + + +#: OTHER's `url` or `path` line replaced by the key with no value, so the +#: section gives none git takes: `(edit, key, the value written back, the +#: findings after it)`. The url is `family.py add`'s for the row; the path +#: is the section's name. With no path, no section of HEAD's file gives +#: OTHER's, which `member-gitmodules-absent` says too, and its line writes +#: the same path again. +def replaced(key: str): + pattern = re.compile(rb"(\[submodule \"members/" + OTHER.encode() + + rb"\"\]\r?\n(?:\t[^\n]*\n)*?)\t" + key.encode() + + rb" = [^\n]*\n") + return lambda data: pattern.sub( + lambda found: found.group(1) + f"\t{key}\n".encode(), data, count=1) + + +WRITTEN_BACK = { + "url": ("url", f"https://github.com/InkRouter/{OTHER}.git", ()), + "path": ("path", f"members/{OTHER}", ("member-gitmodules-absent",)), +} + + +@pytest.mark.parametrize("key, back, beside", WRITTEN_BACK.values(), + ids=WRITTEN_BACK) +def test_a_url_or_path_left_with_no_value_is_written_back_as_add_writes_it( + holder, offline, tmp_path, key, back, beside): + """The section's only `url` or `path` line has no value, committed: + taking the key out would leave the member none, so the finding writes + it back after, as `family.py add` writes it. The work tree's file + gives the value too, so no other check of it is asked, and no + `gitmodules-path-unnamed` takes the member's section out by key. + Typed as printed and committed, the holder passes in every mode.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, replaced(key)(data)) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + found = findings(result) + assert found[0] == taken_out(key, None, back=back) + assert [code.split(":")[0][len("FINDING "):] + for code in found[1:]] == list(beside) + for finding in found: + for command in commands(finding): + type_line(holder, env, command) + raw(holder, env, "add", "--", GITMODULES) + commit_all(holder, env, "Written back as printed") + assert_passes_in_every_mode(holder, 2) diff --git a/tests/test_validate_family_without_git.py b/tests/test_validate_family_without_git.py index 1ca0e9ca..3d5ad928 100644 --- a/tests/test_validate_family_without_git.py +++ b/tests/test_validate_family_without_git.py @@ -121,18 +121,20 @@ def test_with_no_gitmodules_the_gitlink_listing_is_named(holder, monkeypatch, capsys, extra): """THE ISSUE'S RESIDUE: a holder with member rows and its `.gitmodules` gone from the working tree, where main still ended in the traceback in - the default and `--pins` modes. The first check to run git is now #361's - listing of `members/`, which every mode runs, `--no-members` included: - where it passed before, having run no git at all, it now refuses.""" + the default and `--pins` modes. The first check to run git was #361's + listing of `members/`; since #768 it is the reading of HEAD's + `.gitmodules` for a value git dies on, which every mode runs, + `--no-members` included: where it passed before, having run no git at + all, it now refuses.""" (holder / ".gitmodules").unlink() module = holder_validator(holder, monkeypatch) without_git(module, monkeypatch, validator=True, helpers=True) code, err = run(module, holder, capsys, *extra) assert code == 2, err assert err == [ - "REFUSED git-not-runnable: the gitlinks this repository records in " - "members/ could not be listed: git could not be run (No such file " - "or directory)", REMEDIATION] + "REFUSED git-not-runnable: HEAD's .gitmodules could not be read as " + "git reads it: git could not be run (No such file or directory)", + REMEDIATION] @MEMBER_MODES From e1379a9d44f8a0c3114a20628ac9f354b0dab353 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 17:13:15 +0000 Subject: [PATCH 10/20] bootstrap.py, validate-family.py and tests/test_family_gitmodules_values.py: a `branch` with no value is read as the git that runs reads it, a work tree `.gitmodules` git cannot parse fails the run with every member checked out, and the boolean probe reads no config of the person's, so CI's git 2.55 passes the row (#768, rv819) CI's run 39031235254 (git 2.55.0) failed `test_a_value_git_dies_on_fails_bootstrap_and_every_mode_of_the_gate[branch-no-value]`, and lane 2's review rv819 of c12b5f8a (NOT READY) found one MUST, four SHOULDs and five NITs. This commit answers them. MUST 1. The row pinned git's own update at exit 139, the crash w768 measured on git 2.43.0. Git 2.55 refuses the line, `error: missing value for 'submodule.members/IRSS.branch'` and `fatal: bad config line 7`, exit 128; the same assertion fails on git 2.47.1 (`assert 128 == 139`). The table's exit column is gone: the premise is that git's update died and checked nothing out, and every other assertion is the tools' own. SHOULD 2. Measured on git 2.43.0 and 2.47.1, a `branch` with no value after a submodule's url, then `git submodule update --init --recursive` in a plain clone: from the work tree's file, 2.43.0 crashes (exit 139) and 2.47.1 dies (exit 128); from the index's file or HEAD's, a blob, 2.43.0 crashes and 2.47.1 says `error: bad config line 4 in blob`, reads every section before it, checks the submodule out and exits 0, as for a `url` with no value. git's submodule-config.c gained `if (!value) ret = config_error_nonbool(var);` for `branch` in v2.43.1 and v2.44.0; v2.43.0, v2.42.4 and v2.39.5 lack it. Bootstrap kept 2.43.0's rule for every git, so on 2.43.1 and later such a blob had every member not checked out named "git cannot read .gitmodules" and the run failed, where git failed only for want of a credential (measured on 2.47.1 at c12b5f8a). `crashes_on_branch` now reads `git version`: before 2.43.1, or a version it cannot read, the key is refused as before; from 2.43.1 on, the line ends the reading of a blob (`ends_blob_reading`). The gate is unchanged: it reads the work tree's file and HEAD's, which a clone checks out as its work tree's, and every git dies on the key there. The comments that said git crashes say which git does. SHOULD 3. Where every member was checked out, a work tree `.gitmodules` `git config --list` cannot parse (a conflict marker, an open quote) ended `family bootstrap ok` over git's `fatal: bad config line`. `Gitmodules` now says `unreadable`, and the run says NO MEMBER UPDATED and fails on `.gitmodules`, as for a refused value; a row not checked out is said to be that (`git cannot parse .gitmodules`), where it was "no section gives that path", and the cure is the file mended by hand. SHOULD 4. New tests catch each survivor rv819 named: the index's file staged apart from HEAD's; `update = rebase` and `update = merge`; a key with no value inside an ignored section of a blob; a `submodule.active` pattern that is only the start of a path; `submodule..active` set twice; a url beginning with `-` before the one with no value; the `` placeholder; a section name with a backslash; and HEAD's value alone leaving the work tree's file to `gitmodules-path-twice`. SHOULD 5. `gitmodules_sections` is split into `gitmodules_source`, `gitmodules_entries`, `ends_blob_reading` and `read_sections`; `unfetchable` gives each row's reason to `row_reason`; `cures` gives the `.gitmodules` sentences to `gitmodules_cures`; `_check_gitmodules_values` gives its text to `_value_said` and `_value_taken_out`. NIT 6: the gate-in-three-modes block and the good-run block are helpers. NIT 8: where HEAD's file alone gives the value, the finding says git dies on it in a clone. NIT 9: both boolean probes ask git for the `--default` of a key in an empty file (`git config --file `), so no `probe.value` in the person's config is read; one in it made both tools refuse `shallow = true`. NIT 7 (one copy of the rules) and NIT 10 (the body's count) are not code changes here. tests/test_family_gitmodules_values.py: 52 tests, from 28. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 299 ++++++++---- .../family-root/scripts/validate-family.py | 93 ++-- tests/test_family_gitmodules_values.py | 436 ++++++++++++++---- 3 files changed, 624 insertions(+), 204 deletions(-) diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 92ad4cc8..42d7d046 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -49,11 +49,13 @@ every such member is said to be that, naming each key and its value, and the cure is the key given a value git takes or its line taken out; where every member was checked out the run fails all the same, - since git updated none of them. Git exits 0 over the second where a - recursive clone ran first, and 128 (`No url found for submodule - path`) in a plain clone; a missing credential exits non-zero too, - so what the holder records is the test, never git's words. The line - says what cures each (`cures`): + since git updated none of them. A work tree file git cannot parse at + all, a conflict marker or an open quote left in it, is said and + failed on the same way, and its cure is the file mended by hand. Git + exits 0 over the second where a recursive clone ran first, and 128 + (`No url found for submodule path`) in a plain clone; a missing + credential exits non-zero too, so what the holder records is the + test, never git's words. The line says what cures each (`cures`): `validate-family.py` names the commit for the first two; for the fourth it prints the `git submodule update --init --checkout` that checks the member out whatever the clone chose. For the third and @@ -72,8 +74,8 @@ the family Makefile's `validate` target uses. (c) EXITS 0 unless a member's own command failed, or (a) left a member not - checked out that no credential would check out, or found a value in - `.gitmodules` that git cannot read (#768). Any other member that + checked out that no credential would check out, or found a line or a + value in `.gitmodules` that git cannot read (#768). Any other member that is not checked out is REPORTED and skipped, not a failure: it is the state a missing credential leaves, and it is already reported in (a). @@ -85,6 +87,7 @@ import argparse import os +import re import shlex import shutil import subprocess @@ -108,9 +111,11 @@ #: Why (a) left a row's member NOT CHECKED OUT where no credential would #: check it out (#648), in the order `unfetchable` asks. `STOPPED` is #: filled with the gitlink git's init died at (`init_stop`), and `REFUSED` -#: with each entry of `.gitmodules` git refused (`reason`). +#: with each entry of `.gitmodules` git refused (`reason`). `UNREADABLE` is +#: a work tree file git cannot parse at all (#768). UNMERGED = f"git reads no {GITMODULES} while a merge leaves it unmerged" REFUSED = f"git cannot read {GITMODULES}: {{}}" +UNREADABLE = f"git cannot parse {GITMODULES}" NO_GITLINK = "no gitlink is recorded at that path" NO_SECTION = f"no section of {GITMODULES} gives that path" NO_URL = "git has no url for that path" @@ -119,10 +124,22 @@ #: What git 2.43 dies on in a `.gitmodules` it reads (#768), as #: `parse_config` in its submodule-config.c reads a submodule's key: these -#: keys with no value (a `branch` with none crashes it), and an `update` -#: other than these four. A `shallow` must be a boolean and a -#: `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`). +#: keys with no value (a `branch` with none crashes it, `BRANCH_READ_SINCE`), +#: and an `update` other than these four. A `shallow` must be a boolean and +#: a `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`). NO_VALUE_REFUSED = frozenset({"path", "url", "update", "ignore", "branch"}) +#: The first git that reads a `branch` with no value as it reads a `url` +#: with none, `missing value` and `bad config line`, which dies in the work +#: tree's file and ends the reading of the index's or HEAD's, a blob: 2.43.1, +#: and 2.44.0 on its main line, by their submodule-config.c, and 2.47.1 as +#: measured. A git before it crashes reading any `.gitmodules` that gives +#: one, a blob too: 2.43.0 as measured (exit 139), and 2.42.4 and 2.39.5 by +#: their source, which has the same `xstrdup` of no value +#: (`crashes_on_branch`). +BRANCH_READ_SINCE = (2, 43, 1) +#: `git version 2.43.0`, `git version 2.45.1.windows.1`, `git version +#: 2.39.3 (Apple Git-146)`: three numbers, since the third decides here. +GIT_VERSION_RE = re.compile(r"git version (\d+)\.(\d+)\.(\d+)") UPDATE_VALUES = ("none", "checkout", "rebase", "merge") #: What git takes for a key whose value it refused, as (a)'s cure says it. TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", @@ -154,8 +171,9 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: """(a) `git submodule update --init --recursive`, authenticated or not. Where every member is checked out and git died on a value in - `.gitmodules`, so it updated none of them, the one failure line it - returns is for `.gitmodules` (#768). + `.gitmodules`, or on a work tree file it cannot parse, so it updated + none of them, the one failure line it returns is for `.gitmodules` + (#768). Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for @@ -190,9 +208,10 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" for _, path, why in left) + ". " + cures(left, stopper, rows, modules), file=sys.stderr) - elif modules.refused: + elif modules.refused or modules.unreadable: + why = REFUSED if modules.refused else UNREADABLE print(" NO MEMBER UPDATED, and not for want of a credential: " - + reason(REFUSED, stopper, modules) + ". " + + reason(why, stopper, modules) + ". " + cures(left, stopper, rows, modules), file=sys.stderr) return [f"{GITMODULES} (git cannot read it)"] elif proc.returncode == 0: @@ -212,6 +231,8 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, #768): each key given a value git takes (`TAKES`), or its line taken out, and the file committed, with `validate-family.py --no-members`, whose `gitmodules-value-refused` prints the commands. + - A work tree `.gitmodules` git cannot parse (`Gitmodules.unreadable`): + the file mended by hand and committed. - A gitlink or a section the holder lacks at the row's path: `validate-family.py`, which names it and the commit that cures it, `member-gitlink-absent` (with the move, for a gitlink recorded only @@ -234,19 +255,7 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, clone chose. `--init` with the path registers a member a subset clone left out, and `--checkout` overrides an `update = none`.""" whys = {why for _, _, why in left} - said = [] - if UNMERGED in whys: - said.append(f"Resolve the merge in {GITMODULES} and `git add " - f"{GITMODULES}`, then run this again.") - if modules.refused: - given = "; ".join( - f"give `{spelled(key)}` a value git takes, " - f"{TAKES[key.rpartition('.')[2]]}, or take that line out" - for key in dict.fromkeys(key for key, _ in modules.refused)) - said.append(f"{given[:1].upper()}{given[1:]}, and commit " - f"{GITMODULES}. `{PYTHON} " - "scripts/validate-family.py --no-members` prints the " - "commands.") + said = gitmodules_cures(whys, modules) if whys & {NO_GITLINK, NO_SECTION}: said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " "and the commit that cures it.") @@ -286,6 +295,31 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, return " ".join(said) +def gitmodules_cures(whys: set[str], modules: Gitmodules) -> list[str]: + """`cures`' sentences for a `.gitmodules` git reads nothing from, each + in the order `cures` says them (#648, #768): a merge left unmerged, + for a row `whys` names so; each value git dies on; a work tree file + git cannot parse.""" + said = [] + if UNMERGED in whys: + said.append(f"Resolve the merge in {GITMODULES} and `git add " + f"{GITMODULES}`, then run this again.") + if modules.refused: + given = "; ".join( + f"give `{spelled(key)}` a value git takes, " + f"{TAKES[key.rpartition('.')[2]]}, or take that line out" + for key in dict.fromkeys(key for key, _ in modules.refused)) + said.append(f"{given[:1].upper()}{given[1:]}, and commit " + f"{GITMODULES}. `{PYTHON} " + "scripts/validate-family.py --no-members` prints the " + "commands.") + if modules.unreadable: + said.append(f"Mend {GITMODULES} by hand (`git config -f " + f"{GITMODULES} --list` says where it breaks), and " + "commit it.") + return said + + def reason(why: str, stopper: str | None, modules: Gitmodules) -> str: """`why` as (a)'s line says it: `STOPPED` names the gitlink git stopped at, and `REFUSED` each entry of `.gitmodules` git refused, as "`` @@ -311,44 +345,61 @@ class Gitmodules(NamedTuple): them. `unmerged` says a merge left the file unmerged in the index, and `refused` holds each `(submodule.., value)` git dies on as it reads the file, None for a key with no value (#768): git reads - nothing from it then, and the three are empty.""" + nothing from it then, and the three are empty. `unreadable` says the + work tree's file is one `git config --list` cannot parse, which git + dies reading: the three are empty then too.""" given: Counter read: dict[str, str] urls: dict[str, str] unmerged: bool = False refused: tuple[tuple[str, str | None], ...] = () + unreadable: bool = False def gitmodules_sections(root: Path) -> Gitmodules: """`.gitmodules` as `git submodule update` reads it: the work tree's file, which a fresh clone takes from HEAD, or, where the work tree has none, as a deletion not yet committed or a sparse checkout leaves it, - the index's, then HEAD's (`config_from_gitmodules`, #768). Nothing - where none holds one, or git cannot read it: git has no url for any - submodule then either. Nothing, and `unmerged`, where the index holds - the file only at a merge's conflict stages (`git ls-files -s`): git - reads no `.gitmodules` at all then, whatever the work tree holds + the index's, then HEAD's (`gitmodules_source`, #768). Nothing where + none holds one, or git cannot read it: git has no url for any + submodule then either. Nothing, and `unreadable`, where that is the + work tree's file and `git config --list` cannot parse it: git dies + reading it (`bad config line`), before it checks out any member. + Nothing, and `unmerged`, where the index holds the file only at a + merge's conflict stages (`git ls-files -s`): git reads no + `.gitmodules` at all then, whatever the work tree holds (`repo_read_gitmodules`), and its init dies at the first gitlink. Nothing, and `refused`, where git dies on an entry as it reads the file (`refused_value`, #768), before it checks out any member. - `read` and `urls` keep git 2.43's own rules (submodule-config.c), which - drop what `git config --list` keeps. Entry by entry, in file order: - - - a section whose name is empty or has a `..` component, split on `/` - and on a backslash, is ignored whole (git warns `ignoring suspicious - submodule name`), a value git would die on included; - - a key git refuses with no value ends the reading of the index's or - HEAD's file, a blob, where `bad config line` is an error and no die: - git reads nothing after it; - - a `path` or `url` that begins with `-` is dropped, and the value - before it stands (`may be interpreted as a command-line option`); - - where two sections give one path, the later in the file decides, as - a later value in one section replaces an earlier. A section that - gives a path again lets go of the one it gave, even where a later - section gave that one too: git then reads no section there.""" + `read` and `urls` keep git's own rules (submodule-config.c), which + drop what `git config --list` keeps (`gitmodules_entries`, + `read_sections`).""" nothing = Gitmodules(Counter(), {}, {}) + source = gitmodules_source(root) + if source is None: + return nothing._replace(unmerged=True) + try: + listed = git_out(["config", "-z", *source, "--list"], cwd=root, + binary=True) + except Refusal: + return nothing._replace(unreadable=os.path.isfile(root / GITMODULES)) + entries = gitmodules_entries(root, listed, source[0] == "--blob") + refused = tuple(dict.fromkeys( + (f"submodule.{name}.{key}", value) + for name, key, value, ignored in entries + if not ignored and refused_value(root, key, value))) + if refused: + return nothing._replace(refused=refused) + return read_sections(entries) + + +def gitmodules_source(root: Path) -> list[str] | None: + """What `git config` is given to read the `.gitmodules` git reads + (`config_from_gitmodules`, #768): `--file` for the work tree's, where + there is one, else `--blob` for the index's at stage 0, else HEAD's. + None where the index holds it only at a merge's conflict stages.""" try: stages = git_out(["ls-files", "-s", "-z", "--", GITMODULES], cwd=root, binary=True) @@ -361,20 +412,24 @@ def gitmodules_sections(root: Path) -> Gitmodules: if name != GITMODULES.encode() or len(fields) != 3: continue if fields[2] != b"0": - return nothing._replace(unmerged=True) + return None source = ["--blob", fields[1].decode("ascii")] if os.path.lexists(root / GITMODULES): - source = ["--file", GITMODULES] - try: - listed = git_out(["config", "-z", *source, "--list"], cwd=root, - binary=True) - except Refusal: - return nothing - sections: dict[str, dict[str, str]] = {} - read: dict[str, str] = {} - held: dict[str, str] = {} - urls: dict[str, str] = {} - refused: dict[tuple[str, str | None], None] = {} + return ["--file", GITMODULES] + return source + + +def gitmodules_entries(root: Path, listed: bytes, blob: bool + ) -> list[tuple[str, str, str | None, bool]]: + """`(name, key, value, ignored)` for each `submodule..` that + `git config -z --list` printed in `listed`, in file order, as git's + submodule reader meets them: None for a key with no value, and + `ignored` for a section git ignores whole, whose name is empty or has + a `..` component, split on `/` and on a backslash (git warns `ignoring + suspicious submodule name`), a value git would die on included. In the + index's or HEAD's file, a `blob`, git reads nothing after a key it + refuses with no value (`ends_blob_reading`).""" + entries: list[tuple[str, str, str | None, bool]] = [] for record in listed.split(b"\0"): key, valued, value = record.decode("utf-8", "surrogateescape").partition("\n") @@ -383,25 +438,49 @@ def gitmodules_sections(root: Path) -> Gitmodules: continue name = section[len("submodule."):] ignored = not name or ".." in name.replace("\\", "/").split("/") - if (not ignored and not valued and source[0] == "--blob" - and variable in NO_VALUE_REFUSED - {"branch"}): + if blob and not ignored and not valued and ends_blob_reading( + root, variable): break - sections.setdefault(name, {})[variable] = value - if ignored: - continue - if refused_value(root, variable, value if valued else None): - refused[(key, value if valued else None)] = None - elif value.startswith("-"): + entries.append((name, variable, value if valued else None, ignored)) + return entries + + +def ends_blob_reading(root: Path, key: str) -> bool: + """Whether a `key` with no value ends git's reading of the index's or + HEAD's `.gitmodules`, a blob, where `bad config line` is an error and + no die (#768): each key git refuses with no value, but a `branch` only + where git does not crash on it (`crashes_on_branch`); before 2.43.1 + git crashes on it there too, and `refused_value` names it.""" + return key in NO_VALUE_REFUSED and (key != "branch" + or not crashes_on_branch(root)) + + +def read_sections(entries: list[tuple[str, str, str | None, bool]] + ) -> Gitmodules: + """`Gitmodules` of `entries` (`gitmodules_entries`) where git refuses + none. `given` counts every section that gives a path, an ignored one + too. `read` and `urls` keep git's rules: a section git ignores gives + neither; a `path` or `url` that begins with `-` is dropped, and the + value before it stands (`may be interpreted as a command-line + option`); where two sections give one path, the later in the file + decides, as a later value in one section replaces an earlier, and a + section that gives a path again lets go of the one it gave, even where + a later section gave that one too: git then reads no section there.""" + sections: dict[str, dict[str, str]] = {} + read: dict[str, str] = {} + held: dict[str, str] = {} + urls: dict[str, str] = {} + for name, key, value, ignored in entries: + sections.setdefault(name, {})[key] = "" if value is None else value + if ignored or value is None or value.startswith("-"): continue - elif variable == "path": + if key == "path": if name in held: read.pop(held[name], None) held[name] = value read[value] = name - elif variable == "url": + elif key == "url": urls[name] = value - if refused: - return nothing._replace(refused=tuple(refused)) return Gitmodules(Counter(keys["path"] for keys in sections.values() if "path" in keys), read, urls) @@ -411,7 +490,8 @@ def refused_value(root: Path, key: str, value: str | None) -> bool: `value` for `key`, None for no value (#768), as `parse_config` in its submodule-config.c reads the key: a `path`, `url`, `update` or `ignore` with no value (`bad config line`), and a `branch` with none - (it crashes); an `update` other than `none`, `checkout`, `rebase` or + (it crashes, and git 2.43.1 and later refuse it as they refuse a `url` + with none); an `update` other than `none`, `checkout`, `rebase` or `merge`, in that case, a `!command` included (`invalid value`); a `shallow` that is no boolean (`bad boolean config value`); and a `fetchrecursesubmodules` that is neither a boolean nor `on-demand`. An @@ -427,17 +507,39 @@ def refused_value(root: Path, key: str, value: str | None) -> bool: return False +def crashes_on_branch(root: Path, version: str | None = None) -> bool: + """Whether this git crashes reading a `.gitmodules` that gives a + submodule a `branch` with no value (#768): every git before 2.43.1 + does, in the index's or HEAD's file too, where a later git reads that + line as the end of the file (`BRANCH_READ_SINCE`). `version` is what + `git version` printed, for a test to give; None asks this git. A git + whose version cannot be read is taken for one that crashes: the run + then fails naming the key, a cure true on every git, rather than pass + a member git never asked for as the missing credential's.""" + if version is None: + try: + version = git_out(["version"], cwd=root) + except Refusal: + return True + found = GIT_VERSION_RE.match(version) + return (found is None + or tuple(map(int, found.groups())) < BRANCH_READ_SINCE) + + def git_bool(root: Path, value: str | None) -> bool | None: """`value` read as a boolean by git itself, which reads one so for every key (`git config --type=bool`, its `git_parse_maybe_bool`): `true`, `yes`, `on` in any case, a number other than 0, or no value at all are true, and `false`, `no`, `off`, 0 and an empty value false. - None where git takes it for no boolean, and dies.""" + None where git takes it for no boolean, and dies. It is asked as the + `--default` of a key looked up in an empty file, so no `probe.value` + in any of the person's own config scopes is read in its place.""" if value is None: return True try: - answer = git_out(["-c", f"probe.value={value}", "config", - "--type=bool", "--get", "probe.value"], cwd=root) + answer = git_out(["config", "--file", os.devnull, "--type=bool", + f"--default={value}", "--get", "probe.value"], + cwd=root) except Refusal: return None return answer == "true" @@ -556,7 +658,8 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool Asked of this holder's own records, never of git's words: any row at all while a merge leaves `.gitmodules` unmerged in the index, since git then reads no section and asks for no credential, or while the file - gives a value git dies on (`Gitmodules.refused`, #768); a row at + gives a value git dies on (`Gitmodules.refused`, #768), or is a work + tree file git cannot parse (`Gitmodules.unreadable`); a row at whose path the holder records no gitlink (`recorded_gitlink`, as `validate-family.py`'s `member-gitlink-absent` reads one), or whose gitlink no section of `.gitmodules` gives a path (as its @@ -583,23 +686,35 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool stopper = None if exited_0 else init_stop(root, modules, config) found: list[tuple[dict, str, str]] = [] for path, row in left.items(): - if modules.unmerged: - found.append((row, path, UNMERGED)) - elif modules.refused: - found.append((row, path, REFUSED)) - elif recorded_gitlink(root, path) is None: - found.append((row, path, NO_GITLINK)) - elif not modules.given[path]: - found.append((row, path, NO_SECTION)) - elif not has_url(modules, config, path): - found.append((row, path, NO_URL)) - elif exited_0: - found.append((row, path, EXITED_0)) - elif stopper is not None: - found.append((row, path, STOPPED)) + why = row_reason(root, path, modules, config, exited_0, stopper) + if why is not None: + found.append((row, path, why)) return found, stopper, modules +def row_reason(root: Path, path: str, modules: Gitmodules, + config: dict[str, list[str | None]], exited_0: bool, + stopper: str | None) -> str | None: + """Why `unfetchable` says the member at `path`, not checked out, is + none a credential would check out (#648, #768), in the order it asks, + or None where it is the missing credential's.""" + if modules.unmerged: + return UNMERGED + if modules.refused: + return REFUSED + if modules.unreadable: + return UNREADABLE + if recorded_gitlink(root, path) is None: + return NO_GITLINK + if not modules.given[path]: + return NO_SECTION + if not has_url(modules, config, path): + return NO_URL + if exited_0: + return EXITED_0 + return None if stopper is None else STOPPED + + def members_of(root: Path) -> tuple[str, list[dict]]: manifest_path = root / MANIFEST if not manifest_path.is_file(): diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 1054f51a..26f0e960 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -1117,8 +1117,8 @@ def _check_gitmodules_values(root: Path, members: list, whether the work tree's file gives one, and whether the work tree's or HEAD's does. - Git 2.43 dies reading such a file, before `git submodule update` - checks out any member, so `make bootstrap` checks none out either. In + Git dies reading such a file, before `git submodule update` checks + out any member, so `make bootstrap` checks none out either. In a plain clone the members modes refused `member-uninitialized` for the first row, naming that same update, and `--no-members` said `family ok`. The rules are `parse_config`'s in git's submodule-config.c @@ -1148,41 +1148,66 @@ def _check_gitmodules_values(root: Path, members: list, and isinstance(row.get("project"), str) and row["project"]} put_back = present for name, key in dict.fromkeys([*in_work, *in_head]): - full = f"submodule.{name}.{key}" - values = dict.fromkeys([*in_work.get((name, key), []), - *in_head.get((name, key), [])]) - where = GIVEN_WHERE[(name, key) in in_work, (name, key) in in_head] - said = (f"{GITMODULES} gives `{spelled(full)}` " + " and ".join( - "no value" if value is None else f"the value `{spelled(value)}`" - for value in values) + f" {where}, and git dies reading a file " - "that does, before `git submodule update` checks out any " - "member, so neither that nor make bootstrap checks one out. Git " - f"takes {GIT_TAKES[key]} for that key.") + said = _value_said(name, key, in_work, in_head) if present and (name, key) not in in_work: report.finding("gitmodules-value-refused", f"{said} Stage and commit the work tree's " f"{GITMODULES}, which does not give it.") continue - lines = [] if put_back else [f"git checkout HEAD -- {GITMODULES}"] - lines.append(f"git config -f {GITMODULES} --unset-all " - f"{shlex.quote(full)}") back = _written_back(name, key, here if present else there, rows) - if back is not None: - lines.append(f"git config -f {GITMODULES} {shlex.quote(full)} " - f"{back}") - report.finding( - "gitmodules-value-refused", - f"{said} " + report.finding("gitmodules-value-refused", _value_taken_out( + said, f"submodule.{name}.{key}", back, put_back=put_back, + committed=(name, key) in in_head)) + put_back = True + return bool(in_work), bool(in_work or in_head) + + +def _value_said(name: str, key: str, + in_work: dict[tuple[str, str], list[str | None]], + in_head: dict[tuple[str, str], list[str | None]]) -> str: + """`gitmodules-value-refused` up to its remedy, for the key `key` of + the section `name` (#768): each value the work tree's file and HEAD's + give it, where, and what git takes. Where HEAD's alone gives one, the + file git reads here may not, so it says git dies on it in a clone, + which reads HEAD's.""" + full = f"submodule.{name}.{key}" + values = dict.fromkeys([*in_work.get((name, key), []), + *in_head.get((name, key), [])]) + where = (name, key) in in_work, (name, key) in in_head + dies = ("so in a clone neither `git submodule update` nor make " + "bootstrap checks out any member" if where == (False, True) + else "before `git submodule update` checks out any member, so " + "neither that nor make bootstrap checks one out") + return (f"{GITMODULES} gives `{spelled(full)}` " + " and ".join( + "no value" if value is None else f"the value `{spelled(value)}`" + for value in values) + f" {GIVEN_WHERE[where]}, and git dies " + f"reading a file that does, {dies}. Git takes {GIT_TAKES[key]} for " + "that key.") + + +def _value_taken_out(said: str, full: str, back: str | None, *, + put_back: bool, committed: bool) -> str: + """`gitmodules-value-refused` whole, `said` (`_value_said`) and its + commands, one to a line, on the work tree's file (#768): HEAD's put + back first where the work tree has none and no finding before put it + back (`put_back` False), the key `full` taken out, and `back` written + in, where it is given, then the commit where HEAD's gives the value + (`committed`).""" + key = full.rpartition(".")[2] + lines = [] if put_back else [f"git checkout HEAD -- {GITMODULES}"] + lines.append(f"git config -f {GITMODULES} --unset-all " + f"{shlex.quote(full)}") + if back is not None: + lines.append(f"git config -f {GITMODULES} {shlex.quote(full)} " + f"{back}") + return (f"{said} " + ("" if put_back else f"The work tree holds no {GITMODULES}, " "so the first line below puts HEAD's back. ") + "Take the key out" + (f" and write its {key} back" if back is not None else "") + " by hand, in a POSIX shell" - + (f", then stage and commit {GITMODULES}" - if (name, key) in in_head else "") + + (f", then stage and commit {GITMODULES}" if committed else "") + ":" + "".join(f"\n {line}" for line in lines)) - put_back = True - return bool(in_work), bool(in_work or in_head) def _submodule_entries(listed: bytes) -> list[tuple[str, str, str | None]]: @@ -1205,10 +1230,11 @@ def _submodule_entries(listed: bytes) -> list[tuple[str, str, str | None]]: def _values_refused(root: Path, entries: list[tuple[str, str, str | None]] ) -> dict[tuple[str, str], list[str | None]]: """`{(name, key): [each value]}` of `entries` (`_submodule_entries`) - that git 2.43 dies on as it reads `.gitmodules` (#768), as - `parse_config` in its submodule-config.c reads a key: a `path`, `url`, - `update` or `ignore` with no value (`bad config line`), a `branch` - with none (it crashes), an `update` other than `none`, `checkout`, + that git dies on as it reads `.gitmodules` (#768), as `parse_config` + in its submodule-config.c reads a key: a `path`, `url`, `update` or + `ignore` with no value (`bad config line`), a `branch` with none (git + before 2.43.1 crashes on it, and a later one refuses it as it refuses + a `url` with none), an `update` other than `none`, `checkout`, `rebase` or `merge`, in that case, a `!command` included (`invalid value`), a `shallow` that is no boolean, and a `fetchrecursesubmodules` that is neither a boolean nor `on-demand`, as git itself reads a @@ -1232,12 +1258,15 @@ def _values_refused(root: Path, entries: list[tuple[str, str, str | None]] def _git_takes_bool(root: Path, value: str) -> bool: """Whether git reads `value` as a boolean, asked of git itself (`git config --type=bool`), which reads `shallow` and - `fetchRecurseSubmodules` so (#768).""" + `fetchRecurseSubmodules` so (#768). It is asked as the `--default` of + a key looked up in an empty file, so no `probe.value` in any of the + person's own config scopes is read in its place.""" with _git_runs(f"a value in {GITMODULES} could not be read as git " "reads a boolean"): try: - git_out(["-c", f"probe.value={value}", "config", "--type=bool", - "--get", "probe.value"], cwd=root) + git_out(["config", "--file", os.devnull, "--type=bool", + f"--default={value}", "--get", "probe.value"], + cwd=root) except Refusal: return False return True diff --git a/tests/test_family_gitmodules_values.py b/tests/test_family_gitmodules_values.py index 26e52e38..f0165ff8 100644 --- a/tests/test_family_gitmodules_values.py +++ b/tests/test_family_gitmodules_values.py @@ -7,10 +7,10 @@ - Bootstrap said `family bootstrap ok`, exit 0, where git dies reading `.gitmodules` before it checks out any member: an `update = None` (the typo of `update = none`), an `update = !true`, a `url`, `update`, - `ignore` or `branch` with no value (git 2.43 crashes on the last), a - `shallow = maybe`, a `fetchRecurseSubmodules = bogus`. It names every - row with the key and its value now, and fails, where every member was - checked out too. + `ignore` or `branch` with no value (git 2.43.0 crashes on the last, + where 2.43.1 and later refuse it as the others), a `shallow = maybe`, a + `fetchRecurseSubmodules = bogus`. It names every row with the key and + its value now, and fails, where every member was checked out too. - The family gate named none of them: members checked and `--pins` refused `member-uninitialized`, whose remedy is the update that dies, and `--no-members` said `family ok`. Every mode finds @@ -31,16 +31,19 @@ from __future__ import annotations import re +import sys import pytest +from conftest import REPO, load_without_bytecode, run_script from repo_shape import PYTHON -from test_family import family, validate # noqa: F401 +from test_family import ORG, family, validate # noqa: F401 from test_family_add_refusals import holder, offline # noqa: F401 from test_family_bootstrap_gate_order import ( - CHECKOUT_CURE, EXITED_0, FETCHED, NO_URL, NOT_ALL_FETCHED, SECTION_CURE, - STOPPED, after_first_line, bootstrap, cloned, edited_clone, failed_on, - fetching, not_fetched, type_line, url_edited, without_url, + CHECKOUT_CURE, DUPLICATE, EXITED_0, FETCHED, NO_URL, NOT_ALL_FETCHED, + SECTION_CURE, STOPPED, after_first_line, bootstrap, cloned, edited_clone, + failed_on, fetching, not_fetched, type_line, url_edited, with_config, + without_url, ) from test_family_dry_run import isolated from test_family_head_lost_gitmodules import ( @@ -49,20 +52,23 @@ from test_family_member_path import MEMBER, OTHER, raw from test_validate_family_unnamed_mounts import commands, findings -#: The issue's table, `update = None` first, then `branch` with no value, -#: which git 2.43 crashes on (w768's probe): each a line added to OTHER's -#: section after its url, as rv648 (p11) and w648c (p12) added it, the key -#: git lists for it, the value, None for none, and git's update's exit. +#: The issue's table, `update = None` first, then `branch` with no value: +#: each a line added to OTHER's section after its url, as rv648 (p11) and +#: w648c (p12) added it, the key git lists for it, and the value, None for +#: none. Git's update dies on each, exit 128, but for `branch` with no +#: value, which git 2.43.0 crashes on (exit 139, w768's probe) and 2.43.1 +#: and later refuse as the others (exit 128, CI's 2.55.0): the tool's +#: answer is the test, never how git dies. REFUSED_LINES = { - "update-None": ("update = None", "update", "None", 128), - "update-command": ("update = !true", "update", "!true", 128), - "url-no-value": ("url", "url", None, 128), - "update-no-value": ("update", "update", None, 128), - "ignore-no-value": ("ignore", "ignore", None, 128), - "shallow-maybe": ("shallow = maybe", "shallow", "maybe", 128), + "update-None": ("update = None", "update", "None"), + "update-command": ("update = !true", "update", "!true"), + "url-no-value": ("url", "url", None), + "update-no-value": ("update", "update", None), + "ignore-no-value": ("ignore", "ignore", None), + "shallow-maybe": ("shallow = maybe", "shallow", "maybe"), "fetch-bogus": ("fetchRecurseSubmodules = bogus", - "fetchrecursesubmodules", "bogus", 128), - "branch-no-value": ("branch", "branch", None, 139), + "fetchrecursesubmodules", "bogus"), + "branch-no-value": ("branch", "branch", None), } #: What both tools say git takes for each key. @@ -102,13 +108,17 @@ def value_cure(key: str) -> str: def said(key: str, value: str | None, where: str) -> str: - """`gitmodules-value-refused` up to its remedy.""" + """`gitmodules-value-refused` up to its remedy. Where HEAD's file + alone gives the value, git dies on it in a clone, which reads HEAD's, + and that is what it says (rv819's NIT 8).""" given = "no value" if value is None else f"the value `{value}`" + dies = ("so in a clone neither `git submodule update` nor make " + "bootstrap checks out any member" if where == HEAD_ONLY + else "before `git submodule update` checks out any member, so " + "neither that nor make bootstrap checks one out") return (f"FINDING gitmodules-value-refused: .gitmodules gives " f"`{full(key)}` {given} {where}, and git dies reading a file " - "that does, before `git submodule update` checks out any " - "member, so neither that nor make bootstrap checks one out. Git " - f"takes {TAKES[key]} for that key. ") + f"that does, {dies}. Git takes {TAKES[key]} for that key. ") def taken_out(key: str, value: str | None, *, back: str | None = None, @@ -156,12 +166,31 @@ def typed_and_committed(root, env, finding: str) -> None: commit_all(root, env, "The key taken out as printed") +def found_in_every_mode_and_typed(root, env, expected: str) -> None: + """The gate finds `expected` alone, exit 1, in all three modes, and + its commands are typed as printed and committed.""" + for extra in ((), ("--pins",), ("--no-members",)): + gate = validate(root, *extra) + assert gate.returncode == 1, (extra, gate.stdout + gate.stderr) + assert findings(gate) == [expected], extra + typed_and_committed(root, env, expected) + + +def assert_a_good_holder_s_run(root, env) -> None: + """Bootstrap checks every member out and is a good holder's, and the + gate passes in every mode.""" + again = bootstrap(root, env, "--skip-members") + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + assert_passes_in_every_mode(root, 2) + + # --- lines 1 and 2: bootstrap and the gate ---------------------------------- -@pytest.mark.parametrize("line, key, value, code", REFUSED_LINES.values(), +@pytest.mark.parametrize("line, key, value", REFUSED_LINES.values(), ids=REFUSED_LINES) def test_a_value_git_dies_on_fails_bootstrap_and_every_mode_of_the_gate( - family, holder, offline, tmp_path, line, key, value, code): + family, holder, offline, tmp_path, line, key, value): """THE ISSUE'S LINES 1 AND 2. The line committed, a plain clone with the credential: git's update dies and checks nothing out, and bootstrap said ok. It names both rows, the key and its value, and fails. The @@ -175,7 +204,7 @@ def test_a_value_git_dies_on_fails_bootstrap_and_every_mode_of_the_gate( clone, data = edited_clone(holder, env, family, tmp_path, with_line(line)) update = raw(clone, env, "submodule", "update", "--init", "--recursive", check=False) - assert update.returncode == code, update.stderr + assert update.returncode != 0, update.stderr assert not (clone / "members" / MEMBER / ".git").exists(), ( "premise: git checked nothing out") @@ -188,25 +217,19 @@ def test_a_value_git_dies_on_fails_bootstrap_and_every_mode_of_the_gate( not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), cure=value_cure(key)), failed_on(MEMBER, OTHER)], result.stderr - expected = taken_out(key, value, - back=url_of(data) if key == "url" else None) - for extra in ((), ("--pins",), ("--no-members",)): - gate = validate(clone, *extra) - assert gate.returncode == 1, (extra, gate.stdout + gate.stderr) - assert findings(gate) == [expected], extra - typed_and_committed(clone, env, expected) + found_in_every_mode_and_typed( + clone, env, taken_out(key, value, + back=url_of(data) if key == "url" else None)) assert raw(clone, env, "show", f"HEAD:{GITMODULES}").stdout == data - - again = bootstrap(clone, env, "--skip-members") - - assert again.returncode == 0, again.stdout + again.stderr - assert after_first_line(again) == FETCHED - assert_passes_in_every_mode(clone, 2) + assert_a_good_holder_s_run(clone, env) #: Values git takes, which no line or finding names (the issue's no-misfire -#: cases but `update = none`, whose run is the exit-0 skip as before). +#: cases but `update = none`, whose run is the exit-0 skip as before, and +#: each other `update` git takes, rv819's finding 4). TAKEN = {"update-checkout": "update = checkout", + "update-rebase": "update = rebase", + "update-merge": "update = merge", "shallow-true": "shallow = true", "fetch-on-demand": "fetchRecurseSubmodules = on-demand", "ignore-bogus": "ignore = bogus"} @@ -255,12 +278,17 @@ def test_update_none_is_still_the_exit_0_skip(family, holder, offline, #: A section under a name git ignores whole, its `update = None` included -#: (`ignoring suspicious submodule name`): no row names its path. +#: (`ignoring suspicious submodule name`): no row names its path. Git splits +#: the name on a backslash too, on every system (`check_submodule_name`); +#: in the file it is escaped, as git config escapes a subsection name. IGNORED = b'[submodule "../x"]\n\tpath = x\n\tupdate = None\n' +IGNORED_NAMES = {"slash": IGNORED, + "backslash": IGNORED.replace(b"../x", b"..\\\\x")} +@pytest.mark.parametrize("ignored", IGNORED_NAMES.values(), ids=IGNORED_NAMES) def test_a_value_in_a_section_git_ignores_whole_is_no_finding( - family, holder, offline, tmp_path): + family, holder, offline, tmp_path, ignored): """No misfire. A section whose name has a `..` component is ignored whole by git, a value it would die on included. With no credential, the members are the credential's: skipped, and ok. With it, the plain @@ -269,7 +297,7 @@ def test_a_value_in_a_section_git_ignores_whole_is_no_finding( value git refuses.""" env = isolated(offline, tmp_path) clone, _ = edited_clone(holder, env, family, tmp_path, - lambda data: data + IGNORED) + lambda data: data + ignored) assert_the_credential_s_skip(clone, env) result = bootstrap(clone, fetching(env), "--skip-members") @@ -310,6 +338,68 @@ def test_a_holder_with_every_member_checked_out_fails_on_it_too( assert after_first_line(again) == FETCHED +#: A `.gitmodules` git cannot parse at all, the line added at its end: a +#: merge's conflict marker left in it, and a quote left open (rv819's +#: finding 3). `git config --list` and git's update both die on it. +DAMAGED = {"conflict-marker": b"<<<<<<< HEAD\n", + "open-quote": b'\tupdate = "none\n'} + +#: What (a) says of such a file, and what cures it. +UNPARSED = "git cannot parse .gitmodules" +MEND_CURE = ("Mend .gitmodules by hand (`git config -f .gitmodules --list` " + "says where it breaks), and commit it.") + + +@pytest.mark.parametrize("damage", DAMAGED.values(), ids=DAMAGED) +def test_a_gitmodules_git_cannot_parse_fails_with_every_member_checked_out( + holder, offline, tmp_path, damage): + """Every member checked out, as in the holder itself, and a + `.gitmodules` git cannot parse committed: git's update dies before it + updates any member, and bootstrap said `family bootstrap ok`, where a + value git refuses failed the run. It says no member was updated, and + why, and fails on `.gitmodules`; mended and committed, the run is a + good holder's.""" + env = fetching(isolated(offline, tmp_path)) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, data + damage) + update = raw(holder, env, "submodule", "update", "--init", "--recursive", + check=False) + assert b"bad config line" in update.stderr, update.stderr + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + " NO MEMBER UPDATED, and not for want of a credential: " + f"{UNPARSED}. {MEND_CURE}", + "family bootstrap FAILED: .gitmodules (git cannot read it)"] + committed_gitmodules(holder, env, data) + + again = bootstrap(holder, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +def test_a_gitmodules_git_cannot_parse_is_named_for_every_row( + family, holder, offline, tmp_path): + """A plain clone of a holder whose committed `.gitmodules` git cannot + parse, with the credential: git's update dies and checks nothing out. + Bootstrap said no section gave either path, with the validator for + cure; it says git cannot parse the file, and how to mend it.""" + env = fetching(isolated(offline, tmp_path)) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: data + DAMAGED["conflict-marker"]) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", UNPARSED), + (f"members/{OTHER}", UNPARSED), cure=MEND_CURE), + failed_on(MEMBER, OTHER)], result.stderr + + #: The work tree's `.gitmodules` lost and not committed, its index entry #: left (git reads that) or staged out too (git reads HEAD's). def lose_work_tree_file(root, env) -> None: @@ -358,6 +448,120 @@ def test_a_key_with_no_value_ends_git_s_reading_of_the_index_s_file( assert_the_credential_s_skip(clone, env) +@pytest.mark.parametrize("lose", LOST.values(), ids=LOST) +def test_a_branch_with_no_value_in_a_blob_is_read_as_this_git_reads_it( + family, holder, offline, tmp_path, lose): + """A `branch` with no value, the last line of the index's file, or + HEAD's, which git reads where the work tree has none, and no + credential. Git 2.43.1 and later read it as a `url` with none, `bad + config line`, an error in a blob and no die: they read every section + before it and fail on the fetch for want of a credential, and + bootstrap said git cannot read the file and failed; the members are + the credential's now, skipped, and ok. Git 2.43.0 crashes on it there + too and checks nothing out, and bootstrap names it in every row, the + key and no value, and fails, as in the work tree's file.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, with_line("branch")) + lose(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode != 0, "premise: no credential, or a crash" + if b"bad config line" in update.stderr: + assert_the_credential_s_skip(clone, env) + return + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = refused_why("branch", None) + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure("branch")), + failed_on(MEMBER, OTHER)], result.stderr + + +def test_the_index_s_file_is_read_where_it_differs_from_head_s( + family, holder, offline, tmp_path): + """THE ISSUE'S LINE 3: git reads the index's `.gitmodules` before + HEAD's. A plain clone with the credential, `update = None` staged and + not committed, and the work tree's file lost: git reads the index's + and dies, HEAD's being clean, and bootstrap names it in every row + (rv819's finding 4).""" + env = fetching(isolated(offline, tmp_path)) + clone = cloned(holder, env, family, tmp_path, recurse=False) + data = raw(clone, env, "show", f"HEAD:{GITMODULES}").stdout + (clone / GITMODULES).write_bytes(with_line("update = None")(data)) + raw(clone, env, "add", "--", GITMODULES) + lose_work_tree_file(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode != 0, update.stderr + assert not (clone / "members" / MEMBER / ".git").exists(), ( + "premise: git checked nothing out") + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = refused_why("update", "None") + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure("update")), + failed_on(MEMBER, OTHER)], result.stderr + + +def test_a_key_with_no_value_in_a_section_git_ignores_reads_on( + family, holder, offline, tmp_path): + """No misfire. A section git ignores whole, first in the index's file, + gives `url` with no value: git warns and reads on, as in the work + tree's file, so it reads both members' sections and fails on the fetch + for want of a credential. The members are the credential's: skipped, + and ok (rv819's finding 4).""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone( + holder, env, family, tmp_path, + lambda data: b'[submodule "../x"]\n\tpath = x\n\turl\n' + data) + lose_work_tree_file(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert b"bad config line" not in update.stderr, update.stderr + + assert_the_credential_s_skip(clone, env) + + +#: `git version` lines, and whether `crashes_on_branch` says that git +#: crashes on a `branch` with no value: before 2.43.1, and where no version +#: can be read. +VERSIONS = { + "2.43.0": ("git version 2.43.0", True), + "2.43.1": ("git version 2.43.1", False), + "2.42.4": ("git version 2.42.4", True), + "apple-2.39.3": ("git version 2.39.3 (Apple Git-146)", True), + "windows-2.43.0": ("git version 2.43.0.windows.1", True), + "windows-2.45.1": ("git version 2.45.1.windows.1", False), + "2.55.0": ("git version 2.55.0", False), + "3.0.0": ("git version 3.0.0", False), + "unread": ("git version unknown", True), +} + + +@pytest.mark.parametrize("version, crashes", VERSIONS.values(), + ids=VERSIONS) +def test_which_git_crashes_on_a_branch_with_no_value(monkeypatch, tmp_path, + version, crashes): + """Which git the family root's bootstrap takes for one that crashes + on a `branch` with no value, where it reads the index's or HEAD's + file: every git before 2.43.1, and one whose version it cannot read. + The test above meets the git it runs on; this meets each side of the + line. Loading it puts its own folder on `sys.path`, which is put + back.""" + monkeypatch.setattr(sys, "path", list(sys.path)) + module = load_without_bytecode( + REPO / "templates" / "family-root" / "scripts" / "bootstrap.py", + "gitmodules_values_bootstrap") + + assert module.crashes_on_branch(tmp_path, version) is crashes + + def test_a_value_git_dies_on_in_the_index_s_file_is_named_and_mended( family, holder, offline, tmp_path): """`update = None` committed, and the work tree's file lost, not @@ -379,18 +583,10 @@ def test_a_value_git_dies_on_in_the_index_s_file_is_named_and_mended( not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), cure=value_cure("update")), failed_on(MEMBER, OTHER)], result.stderr - expected = taken_out("update", "None", where=HEAD_ONLY, put_back=True) - for extra in ((), ("--pins",), ("--no-members",)): - gate = validate(clone, *extra) - assert gate.returncode == 1, (extra, gate.stdout + gate.stderr) - assert findings(gate) == [expected], extra - typed_and_committed(clone, env, expected) - - again = bootstrap(clone, env, "--skip-members") - - assert again.returncode == 0, again.stdout + again.stderr - assert after_first_line(again) == FETCHED - assert_passes_in_every_mode(clone, 2) + found_in_every_mode_and_typed( + clone, env, + taken_out("update", "None", where=HEAD_ONLY, put_back=True)) + assert_a_good_holder_s_run(clone, env) def test_two_keys_with_no_gitmodules_in_the_work_tree_put_head_s_back_once( @@ -428,7 +624,8 @@ def test_a_member_whose_own_active_is_false_is_not_where_git_stops( """THE ISSUE'S LINE 3, its second item. OTHER's section lost its url, committed; in a plain clone `submodule.active` is `.`, as a recursive clone sets it, so every member is in the subset, and OTHER's own - `submodule.members/

.active` is false. Git asks that first, inits + `submodule.members/

.active` is set twice, true then false, of which + git reads the last. Git asks that first, inits MEMBER alone and fails on its fetch for want of a credential, and stops nowhere; bootstrap said git stopped at OTHER. MEMBER is the credential's now, and OTHER, which git has no url for, is named with @@ -437,7 +634,9 @@ def test_a_member_whose_own_active_is_false_is_not_where_git_stops( clone, _ = edited_clone(holder, env, family, tmp_path, lambda data: without_url(data, OTHER)) raw(clone, env, "config", "submodule.active", ".") - raw(clone, env, "config", f"submodule.members/{OTHER}.active", "false") + for flag in ("true", "false"): + raw(clone, env, "config", "--add", + f"submodule.members/{OTHER}.active", flag) update = raw(clone, env, "submodule", "update", "--init", "--recursive", check=False) assert update.returncode != 0, "premise: the fetch failed" @@ -452,28 +651,31 @@ def test_a_member_whose_own_active_is_false_is_not_where_git_stops( failed_on(OTHER)], result.stderr -#: `submodule.active` patterns: `(the pattern, whether git's init then asks -#: OTHER and dies, (a)'s rows, the rows it fails on)`. A plain path, however -#: spelt, names MEMBER alone, as `git clone --recurse-submodules=` -#: writes it; a leading directory names both. +#: `submodule.active` patterns: `(the patterns, whether git's init then +#: asks OTHER and dies, (a)'s rows, the rows it fails on)`. A plain path, +#: however spelt, names MEMBER alone, as `git clone +#: --recurse-submodules=` writes it; a leading directory names both; +#: the start of OTHER's path that is no directory of it names nothing. STOPS_AT_OTHER = (f"members/{MEMBER}", STOPPED.format(path=f"members/{OTHER}")) PATTERNS = { - "a-path": (f"members/{MEMBER}", False, + "a-path": ((f"members/{MEMBER}",), False, ((f"members/{OTHER}", NO_URL),), (OTHER,)), - "a-path-spelt-another-way": (f"./members/{MEMBER}/", False, + "a-path-spelt-another-way": ((f"./members/{MEMBER}/",), False, ((f"members/{OTHER}", NO_URL),), (OTHER,)), - "a-leading-directory": ("members", True, + "a-leading-directory": (("members",), True, (STOPS_AT_OTHER, (f"members/{OTHER}", NO_URL)), (MEMBER, OTHER)), + "a-string-prefix": ((f"members/{MEMBER}", f"members/{OTHER[:-1]}"), + False, ((f"members/{OTHER}", NO_URL),), (OTHER,)), } -@pytest.mark.parametrize("pattern, stops, left, failed", PATTERNS.values(), +@pytest.mark.parametrize("patterns, stops, left, failed", PATTERNS.values(), ids=PATTERNS) def test_a_subset_s_pattern_says_whether_git_stops_at_a_member_outside_it( - family, holder, offline, tmp_path, pattern, stops, left, failed): + family, holder, offline, tmp_path, patterns, stops, left, failed): """THE ISSUE'S LINE 3, its third item. OTHER's section lost its url, - committed, and the clone's subset is `pattern`, with no credential. + committed, and the clone's subset is `patterns`, with no credential. Where the pattern leaves OTHER out, git's init skips it and fails on MEMBER's fetch, and bootstrap said git stopped at OTHER; MEMBER is the credential's now. Where it names OTHER, git does die there, and that @@ -481,7 +683,8 @@ def test_a_subset_s_pattern_says_whether_git_stops_at_a_member_outside_it( env = isolated(offline, tmp_path) clone, _ = edited_clone(holder, env, family, tmp_path, lambda data: without_url(data, OTHER)) - raw(clone, env, "config", "submodule.active", pattern) + for pattern in patterns: + raw(clone, env, "config", "--add", "submodule.active", pattern) update = raw(clone, env, "submodule", "update", "--init", "--recursive", check=False) assert (f"No url found for submodule path 'members/{OTHER}'".encode() @@ -539,39 +742,93 @@ def test_the_gate_says_which_file_gives_it(holder, offline, tmp_path, set_up, assert_passes_in_every_mode(holder, 2) +def test_a_url_no_row_gives_one_for_is_written_back_as_a_placeholder( + holder, offline, tmp_path): + """OTHER's row gives no `/` repository, and its section's + only `url` line has no value, both committed: there is no url as + `family.py add` writes one, so the finding writes back `` for a + person to fill in, as `member-gitmodules-absent` does (rv819's finding + 4).""" + env = isolated(offline, tmp_path) + rows = (holder / "family.yaml").read_bytes() + qualified = f"repository: {ORG}/{OTHER}".encode() + assert rows.count(qualified) == 1, rows + (holder / "family.yaml").write_bytes( + rows.replace(qualified, f"repository: {OTHER}".encode())) + raw(holder, env, "add", "--", "family.yaml") + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, replaced("url")(data)) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result)[0] == taken_out("url", None, back="") + + +def test_a_value_head_s_file_alone_gives_leaves_the_work_tree_s_read( + holder, offline, tmp_path): + """HEAD's file alone gives `update = None`, and the work tree's, + mended and not committed, gives OTHER's path in a second section. Git + reads the work tree's here, so `gitmodules-path-twice` still reads it + and names the two: only a value in the work tree's file stops that + reading (rv819's finding 4).""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, with_line("update = None")(data)) + (holder / GITMODULES).write_bytes(data + DUPLICATE) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert [found.split(":")[0] for found in findings(result)] == [ + "FINDING gitmodules-value-refused", "FINDING gitmodules-path-twice"] + + #: OTHER's `url` or `path` line replaced by the key with no value, so the #: section gives none git takes: `(edit, key, the value written back, the #: findings after it)`. The url is `family.py add`'s for the row; the path #: is the section's name. With no path, no section of HEAD's file gives #: OTHER's, which `member-gitmodules-absent` says too, and its line writes #: the same path again. -def replaced(key: str): +def replaced(key: str, before: bytes = b""): + """An edit replacing OTHER's `key` line with `before`, then the key + with no value.""" pattern = re.compile(rb"(\[submodule \"members/" + OTHER.encode() + rb"\"\]\r?\n(?:\t[^\n]*\n)*?)\t" + key.encode() + rb" = [^\n]*\n") return lambda data: pattern.sub( - lambda found: found.group(1) + f"\t{key}\n".encode(), data, count=1) + lambda found: found.group(1) + before + f"\t{key}\n".encode(), data, + count=1) +#: `(edit, key, the value written back, the findings after it)`. A url git +#: drops, one that begins with `-`, is no url to write back (rv819's +#: finding 4). WRITTEN_BACK = { - "url": ("url", f"https://github.com/InkRouter/{OTHER}.git", ()), - "path": ("path", f"members/{OTHER}", ("member-gitmodules-absent",)), + "url": (replaced("url"), "url", + f"https://github.com/InkRouter/{OTHER}.git", ()), + "path": (replaced("path"), "path", f"members/{OTHER}", + ("member-gitmodules-absent",)), + "url-after-a-dashed-one": ( + replaced("url", b"\turl = -bad\n"), "url", + f"https://github.com/InkRouter/{OTHER}.git", ()), } -@pytest.mark.parametrize("key, back, beside", WRITTEN_BACK.values(), +@pytest.mark.parametrize("edit, key, back, beside", WRITTEN_BACK.values(), ids=WRITTEN_BACK) def test_a_url_or_path_left_with_no_value_is_written_back_as_add_writes_it( - holder, offline, tmp_path, key, back, beside): + holder, offline, tmp_path, edit, key, back, beside): """The section's only `url` or `path` line has no value, committed: taking the key out would leave the member none, so the finding writes - it back after, as `family.py add` writes it. The work tree's file - gives the value too, so no other check of it is asked, and no - `gitmodules-path-unnamed` takes the member's section out by key. - Typed as printed and committed, the holder passes in every mode.""" + it back after, as `family.py add` writes it, never a url git drops. + The work tree's file gives the value too, so no other check of it is + asked, and no `gitmodules-path-unnamed` takes the member's section out + by key. Typed as printed and committed, the holder passes in every + mode.""" env = isolated(offline, tmp_path) data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout - committed_gitmodules(holder, env, replaced(key)(data)) + committed_gitmodules(holder, env, edit(data)) result = validate(holder, "--no-members") @@ -586,3 +843,22 @@ def test_a_url_or_path_left_with_no_value_is_written_back_as_add_writes_it( raw(holder, env, "add", "--", GITMODULES) commit_all(holder, env, "Written back as printed") assert_passes_in_every_mode(holder, 2) + + + +def test_a_probe_value_in_the_person_s_own_config_changes_no_answer( + family, holder, offline, tmp_path): + """Both tools ask git whether a value is a boolean through a key of + their own, `probe.value`. One the person's own config gives, here no + boolean, made both refuse `shallow = true` (rv819's NIT 9): bootstrap + named every row and failed, and the gate found + `gitmodules-value-refused`. Neither reads it now: the members are the + credential's, skipped, and ok, and the gate passes.""" + env = with_config(isolated(offline, tmp_path), "probe.value", "notabool") + clone, _ = edited_clone(holder, env, family, tmp_path, + with_line("shallow = true")) + + assert_the_credential_s_skip(clone, env) + gate = run_script(clone / "scripts" / "validate-family.py", + "--no-members", cwd=clone, env=env) + assert gate.returncode == 0, gate.stdout + gate.stderr From 233a334e7c56236ffcc334485d41034d9960d0d3 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 00:40:46 +0000 Subject: [PATCH 11/20] tests/test_family_gitmodules_values.py: a staged `git rm --cached` beside a value git dies on in the work tree's `.gitmodules` is a finding that prints the reset alone, so `_check_removals_staged` handed the emptied sections no longer puts `git checkout HEAD -- .gitmodules` in it (#768, point 3 of the merge with `b9b70692`) The merge of main `b9b70692` (#820, #762) into this branch hands `_check_removals_staged` the work tree's sections as read (`as_read`), not as #768 empties them where the work tree's file gives a value git dies on (rv820's note for whichever of #819 and #820 landed second). No test held that line: a mutant giving it `sections` again passed all 191 tests of `test_validate_family_typed_states.py`, `test_family_gitmodules_values.py`, `test_family_head_lost_gitmodules.py`, `test_validate_family_remedies_typed.py` and `test_validate_family_unnamed_mounts.py`. `test_a_staged_removal_beside_a_refused_value_puts_no_section_back` holds the state: the work tree's file gives OTHER's section `update = None`, and `git rm --cached -- members/IRRS` is staged. In all three modes the gate finds `gitmodules-value-refused` and `member-gitlink-removal-staged`, whole (`taken_out` and `removal_staged`), the second's commands are `git reset -q -- members/IRRS` alone, and `git checkout HEAD -- .gitmodules` is not in it. The test passes on `e32ed55a`'s validator. With the call changed back to `sections` it fails at the first mode: the finding's last line is the `git checkout HEAD -- .gitmodules` it should not print. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_family_gitmodules_values.py | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_family_gitmodules_values.py b/tests/test_family_gitmodules_values.py index f0165ff8..3a6610ac 100644 --- a/tests/test_family_gitmodules_values.py +++ b/tests/test_family_gitmodules_values.py @@ -48,8 +48,11 @@ from test_family_dry_run import isolated from test_family_head_lost_gitmodules import ( GITMODULES, assert_passes_in_every_mode, commit_all, committed_gitmodules, + pinned, ) from test_family_member_path import MEMBER, OTHER, raw +from test_family_tracking_branch import on_main # noqa: F401 +from test_validate_family_typed_states import ROW, removal_staged from test_validate_family_unnamed_mounts import commands, findings #: The issue's table, `update = None` first, then `branch` with no value: @@ -862,3 +865,33 @@ def test_a_probe_value_in_the_person_s_own_config_changes_no_answer( gate = run_script(clone / "scripts" / "validate-family.py", "--no-members", cwd=clone, env=env) assert gate.returncode == 0, gate.stdout + gate.stderr + + +# --- beside #762's staged removal -------------------------------------------- + +def test_a_staged_removal_beside_a_refused_value_puts_no_section_back( + on_main, offline, tmp_path): + """#768 BESIDE #762, the point rv820 asked the second PR to land + (resolution point 3 of the merge with main `b9b70692`). The work tree's + file gives OTHER's section `update = None`, which git dies on, and + `git rm --cached` has staged MEMBER's gitlink out, taking no section + from `.gitmodules`. The gate reads no section from a file it refuses + (`sections` is emptied), and the finding for the removal read that as + a section `git rm` took out: it printed `git checkout HEAD -- + .gitmodules` and said it put the section back. It is handed the + sections as read, so it prints the reset alone, in every mode.""" + env = isolated(offline, tmp_path) + commit = pinned(on_main, MEMBER) + raw(on_main, env, "rm", "-q", "--cached", "--", ROW) + edited_not_committed( + on_main, env, raw(on_main, env, "show", f"HEAD:{GITMODULES}").stdout) + expected = [taken_out("update", "None", where=WORK_TREE_ONLY), + removal_staged(on_main, commit, "git-rm-cached")] + + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(on_main, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + assert findings(result) == expected, extra + staged = findings(result)[1] + assert commands(staged) == [f"git reset -q -- {ROW}"], extra + assert f"git checkout HEAD -- {GITMODULES}" not in staged, extra From 20f74cbdd9ebee81fe550e880c2fe24d7b180c55 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 02:53:11 +0000 Subject: [PATCH 12/20] the family root's bootstrap.py and validate-family.py: the credential advice and a member's exit code say what happened, and the gate's `git rm --cached`, guarded `mv` and section lines type as printed where they failed (#839) Seven lines of #839, in the holder's two pinned scripts, on #819's head (233a334e): 1. `fetch_members` printed "If the members are PRIVATE, set SHAPE_LEGS_TOKEN ..." for every failed `git submodule update`, then "NOT CHECKED OUT, and not for want of a credential" for every row `unfetchable` names. `credential_advice` prints it only where git asked for a credential: not where its init died at a gitlink it has no url for, nor where it read nothing from `.gitmodules` (unmerged, a value git dies on, a file it cannot parse). Beside a denial it says it is for the members that line does not name. A leg left unfetched in a checked-out member keeps it. 2. `run_member_target` said `make exited 2` for a member's finding (1), its refusal (2) and make's own failure alike: GNU make exits 2 for each (measured, GNU Make 4.3). `run_make` passes make's stderr on as it comes and reads the code from its last `Error ` line: `make bootstrap: the member's command exited N`, or `make bootstrap exited 2 and printed no ... line` where make printed none. Make is still what runs, so `--members-make validate` gets it too. 3. In the gate's state after `git mv members/

Members/

`, staged, with a commit or an untracked file in the moved checkout, the printed `git rm --cached -- Members/

` (the STRAY finding) refused, exit 1. `_forced` reads where the index stages a variant at a commit HEAD does not record there over a checkout; the STRAY line and the wrong-case move's `git rm` then carry `-f` and say what it does, as family.py's move back says it (#828). 4. The wrong-case move's guard followed a link (`-ef`, `-e`), so a link at the variant to the row's own directory or to a clone was kept, and `family.py add` refused it; with nothing there the `mv` failed `cannot stat`. `_guarded_move` prints `[ ! -L V ] && { [ ! -e V ] || [ V -ef R ] || [ -e V/.git ]; } || mv -- V ../`, run in dash and bash on seven layouts. Comment 6237599648 is the same fault. 5. Comment 6237354577 (a): bootstrap already reads the work tree's, the index's, then HEAD's `.gitmodules` at 233a334e (#819); a test asks it. 6. Comment 6237354577 (b): `member-gitmodules-absent`'s `git config -f` lines made a one-section file where the work tree's `.gitmodules` is gone. The first finding that writes the file puts it back first (`_gitmodules_put_back`, `_back_first`), and no other, nor any after a `gitmodules-value-refused` that put HEAD's back; a second `member-gitmodules-url-absent` put the file back over the first's url. Tests: tests/test_family_holder_bootstrap_texts.py (48), and `variant_refused` in tests/test_family_head_lost_gitmodules.py and the make line in tests/test_bootstrap_spelling.py follow the new texts. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 143 +++- .../family-root/scripts/validate-family.py | 291 +++++-- tests/test_bootstrap_spelling.py | 5 +- tests/test_family_head_lost_gitmodules.py | 59 +- tests/test_family_holder_bootstrap_texts.py | 738 ++++++++++++++++++ 5 files changed, 1125 insertions(+), 111 deletions(-) create mode 100644 tests/test_family_holder_bootstrap_texts.py diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index cf8ba415..8fa2685b 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -25,7 +25,10 @@ A fetch that fails is REPORTED and does not stop the run: the members that did arrive are still worth bootstrapping, and a family that refused to do anything because one member is unreachable would be worse - than one that says which. + than one that says which. The advice to set a credential is given only + where git asked for one: not where its init died at a gitlink it has + no url for, nor where it read nothing from `.gitmodules`, both below, + since it fetched nothing then (#839). "every member and leg fetched" is said only where git exited 0 AND every row's member is checked out (#648). A member left NOT CHECKED @@ -72,7 +75,11 @@ review-authority readout. This command does not reimplement any of that; it walks the members `family.yaml` names and calls theirs. `--members-make validate` runs `make validate` instead, which is what - the family Makefile's `validate` target uses. + the family Makefile's `validate` target uses. A member whose command + fails is named with that command's exit code, read from make's + `Error ` line, since make exits 2 for any command that fails, + so a member's finding (1) and its refusal (2) are told apart, and + from a failure of make's own, which prints no such line (#839). (c) EXITS 0 unless a member's own command failed, or (a) left a member not checked out that no credential would check out, or found a line or a @@ -182,8 +189,10 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for the run to end FAILED on (#648); every member and leg is said to be - fetched only where there is none and git exited 0. Raises the - `git-not-runnable` refusal where git cannot be started (#694).""" + fetched only where there is none and git exited 0. The advice to set a + credential is printed only where git asked for one + (`credential_advice`, #839). Raises the `git-not-runnable` refusal + where git cannot be started (#694).""" source, token = credential() args = ["git"] if token: @@ -202,19 +211,10 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: if refusal is None: raise raise refusal from exc - if proc.returncode != 0: - # The token can appear in git's own error text (it is in the URL it - # tried), so the output is scrubbed before it is printed. - message = (proc.stderr + proc.stdout).strip() - if token: - message = message.replace(token, "") - print(f" FETCH INCOMPLETE (git exit {proc.returncode}) using " - f"credential source '{source}':\n{message}", file=sys.stderr) - print(" If the members are PRIVATE, set SHAPE_LEGS_TOKEN (a " - "fine-grained PAT with contents:read on them) or export an " - "App-minted SHAPE_LEGS_APP_TOKEN, then re-run. What did arrive " - "is bootstrapped below.", file=sys.stderr) left, stopper, modules = unfetchable(root, rows, proc.returncode == 0) + if proc.returncode != 0: + fetch_incomplete(proc, source, token, + credential_advice(bool(left), stopper, modules)) if left: print(" NOT CHECKED OUT, and not for want of a credential: " + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" @@ -232,6 +232,44 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: for row, _, _ in left] +def fetch_incomplete(proc: subprocess.CompletedProcess, source: str, + token: str, advice: str) -> None: + """(a)'s report of a `git submodule update` that exited non-zero: git's + own words, then `advice`, where there is any (`credential_advice`).""" + # The token can appear in git's own error text (it is in the URL it + # tried), so the output is scrubbed before it is printed. + message = (proc.stderr + proc.stdout).strip() + if token: + message = message.replace(token, "") + print(f" FETCH INCOMPLETE (git exit {proc.returncode}) using " + f"credential source '{source}':\n{message}", file=sys.stderr) + if advice: + print(advice, file=sys.stderr) + + +def credential_advice(denied: bool, stopper: str | None, + modules: Gitmodules) -> str: + """(a)'s advice to set a credential, or nothing where git asked for none + (#839): where its init died at a gitlink it has no url for (`stopper`), + or it read nothing from `.gitmodules`, a merge leaving it unmerged, a + value git dies on or a file it cannot parse, it fetched nothing. Every + row (a) then leaves not checked out is said to be no credential's, and + the advice beside that was its denial (w768's candidate 1 on #819). + Anywhere else a fetch git made failed, a member's, or a leg's in a + member checked out, and the advice stands; where a row is said to be no + credential's beside it (`denied`), it is for the members that line does + not name.""" + if (stopper is not None or modules.unmerged or modules.refused + or modules.unreadable): + return "" + which = ("the members the next line does not name" if denied + else "the members") + return (f" If {which} are PRIVATE, set SHAPE_LEGS_TOKEN (a fine-grained " + "PAT with contents:read on them) or export an App-minted " + "SHAPE_LEGS_APP_TOKEN, then re-run. What did arrive is " + "bootstrapped below.") + + def git_not_started(exc: OSError) -> Refusal | None: """The `git-not-runnable` refusal for an `OSError` that says git could not be STARTED (#694), else `None`, and the error goes on as it did. @@ -782,7 +820,9 @@ def member_path(root: Path, row: dict) -> Path: def run_member_target(root: Path, row: dict, target: str) -> str | None: - """(b) One member's own `make `. Returns a failure line, or None.""" + """(b) One member's own `make `. Returns a failure line, or None: + one naming the exit code of the member's command where make names it, + else make's own (`run_make`, #839).""" # A member row is the family manifest's text (#391): what is PRINTED of it, # and of `--members-make`, is spelled. The path is built from the raw row. project = spelled(row.get("project") or "?") @@ -803,11 +843,72 @@ def run_member_target(root: Path, row: dict, target: str) -> str | None: return None print(f" --- {project}: make {named} ---") sys.stdout.flush() - proc = subprocess.run([make, target], cwd=str(path), check=False) + returncode, own = run_make(make, target, path) sys.stdout.flush() - if proc.returncode != 0: - return f"{project} (make {named} exited {proc.returncode})" - return None + if returncode == 0: + return None + if own is not None: + return f"{project} (make {named}: the member's command exited {own})" + return (f"{project} (make {named} exited {returncode} and printed no " + "`Error ` line for a command of the member's: make's own " + "message above says what failed)") + + +#: The line GNU make prints where a command of the target exits non-zero, +#: `make: *** [Makefile:44: bootstrap] Error 1` (`make[1]:` in a make run by +#: make, `*** [bootstrap] Error 1` before make 4.0): the command's own exit +#: code, which make's does not carry, since make exits 2 for any command +#: that fails (#839). +MAKE_ERROR_RE = re.compile(rb"\*\*\* \[[^\n]*\] Error (\d+)\s*$") + + +def run_make(make: str, target: str, cwd: Path) -> tuple[int, int | None]: + """(b) `make ` in a member, as `(make's exit code, the exit code + of the member's command it names)`, the second None where make printed + no `MAKE_ERROR_RE` line (#839). + + MAKE'S OWN CODE TELLS NOTHING APART. GNU make exits 2 for any command + that fails, measured (GNU Make 4.3) for a command exiting 1, 2, 3 or + 126, and 2 for a failure of its own (`No rule to make target`), so the + line said `make bootstrap exited 2` for the member's own `bootstrap + FAILED` (1, a finding), for a refusal (2) and for make's own failure + alike (w764b, for lane 4). The command's code is the one make names + in its `Error ` line, the last on stderr, so make's stderr is + passed on to this run's own as it comes and that line read from the + end of it; a make that prints none, one of its own failures, a command + killed by a signal, or a make speaking another language, leaves make's + code alone. The member's command is never run around make: another + target than `bootstrap` (`--members-make validate`) has no script of + its own to call, as `make siblings` calls a member's + `scripts/bootstrap.py`.""" + proc = subprocess.Popen([make, target], cwd=str(cwd), + stderr=subprocess.PIPE) + tail = b"" + for chunk in iter(lambda: proc.stderr.read1(65536), b""): + passed_on(chunk) + tail = (tail + chunk)[-4096:] + proc.stderr.close() + returncode = proc.wait() + found = [MAKE_ERROR_RE.search(line) for line in tail.splitlines()] + named = [int(match.group(1)) for match in found if match] + return returncode, (named[-1] if named else None) + + +def passed_on(chunk: bytes) -> None: + """`chunk` of what a member's make wrote to its stderr, written to this + run's as it came (#839): as its bytes, after what this run wrote there + before, or as text where stderr has no byte buffer.""" + stream = sys.stderr + if stream is None: + return + stream.flush() + sink = getattr(stream, "buffer", None) + if sink is None: + stream.write(chunk.decode("utf-8", "replace")) + stream.flush() + return + sink.write(chunk) + sink.flush() def main(argv: list[str] | None = None) -> int: diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index d95a8636..b37bb850 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -593,7 +593,8 @@ def _gitlink_in_another_case(root: Path, path: str, commit: str, move it out of the holder, by an `mv` with nothing to fill in, which a shell test skips where the disk folds the two names into one directory, the row's own, and where the directory is a checkout - (#681). A checkout is deleted by hand once nothing only it holds is + (#681), and where nothing is there, and never skips for a symbolic + link (#839). A checkout is deleted by hand once nothing only it holds is needed. Where nothing is checked out at the row's path, as in a fresh clone, `git submodule update --init --recursive` then checks the member out there, LAST (#762): a checkout `git mv` moved to the variant names @@ -612,7 +613,8 @@ def _gitlink_in_another_case(root: Path, path: str, commit: str, f"at {path}.", "Remediation: the member is recorded in the wrong case. " + _wrong_case_move(path, commit, variants, - _move_context(root, path, variants, repository))) + _move_context(root, path, variants, repository, + gitlinks))) class MoveContext(NamedTuple): @@ -621,18 +623,22 @@ class MoveContext(NamedTuple): checkout keeps off disk; `in_the_way`, every entry the index holds at or under the row's path; and `section`, the lines that make `.gitmodules` give the row's path, with what they do, or empty where a - section of HEAD's already gives it.""" + section of HEAD's already gives it. `forced`, the variants whose `git + rm --cached` takes `-f` (`_forced`, #839).""" sparse: tuple[str, ...] = () in_the_way: tuple[str, ...] = () section: str = "" section_said: str = "" + forced: tuple[str, ...] = () def _move_context(root: Path, path: str, variants: list[str], - repository) -> MoveContext: + repository, gitlinks: dict[str, str]) -> MoveContext: """What the index and HEAD's `.gitmodules` hold around the move of a - member recorded only at `variants` to its row's `path` (#762). + member recorded only at `variants` to its row's `path` (#762), and + which variants the move's `git rm --cached` takes out only with `-f` + (`_forced`, by `gitlinks`, `_member_gitlinks`, #839). `--sparse` (the issue's (f)): a variant a sparse checkout keeps off disk (`git ls-files -t`'s `S`, as `family.py`'s `_skip_worktree` reads @@ -697,7 +703,39 @@ def _move_context(root: Path, path: str, variants: list[str], "variant's path") section += f"\n git add -- {GITMODULES}" return MoveContext(tuple(sorted(sparse)), tuple(in_the_way), section, - said) + said, _forced_variants(root, variants, gitlinks)) + + +def _forced_variants(root: Path, variants: list[str], + gitlinks: dict[str, str]) -> tuple[str, ...]: + """Each of `variants` whose `git rm --cached` takes `-f` (`_forced`), + in their order (#839).""" + return tuple(variant for variant in variants + if _forced(root, variant, gitlinks[variant]) is not None) + + +def _forced(root: Path, path: str, commit: str) -> str | None: + """None where `git rm --cached` takes the gitlink at `path`, which + `_member_gitlinks` reads at `commit`, out of the index as it is, else + what HEAD records at `path`, its commit or `""` for nothing, where that + takes `-f` (#839): the index stages the gitlink at a commit HEAD does + not record there, and a checkout is on disk at it, a `.git` entry. Git + refuses it then wherever that checkout has moved on or holds a file it + does not track (`the following file has staged content different from + both the file and the HEAD`, exit 1), measured on git 2.43 after `git + mv members/

Members/

`, staged, and a commit or an untracked file + in the checkout it moved (w761's candidate 2 on #828), and `-f`, with + `--cached`, touches no file. A gitlink HEAD alone records, at a staged + removal or a conflict, is read at HEAD's commit and takes none.""" + if not os.path.lexists(root / path / ".git"): + return None + with _git_runs(f"the gitlink HEAD records at {path} could not be read"): + try: + head = git_out(["rev-parse", "--verify", "--quiet", + f"HEAD:{path}"], cwd=root).lower() + except Refusal: + head = "" + return None if head == commit else head def _wrong_case_move(path: str, commit: str, variants: list[str], @@ -708,39 +746,45 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], `member-gitlink-unnamed` under `--no-members` (#648), in one text. What else it meets is `context` (`_move_context`, #762). - EACH `mv` IS GUARDED BY TWO TESTS A POSIX SHELL RUNS (#681, rv611c's - N-1 and N-2), not by a system's name: this text is read in CI too, on - another disk than the one it is typed on. `[ -ef ]` is - true where the disk takes the two names for one directory, the row's - own, as a Linux mount that folds case does too (WSL's `/mnt/c`), where - the `mv` moved the member's own checkout out. `[ -e /.git ]` - is true for a checkout, and a submodule's `.git` is a file naming this - holder's `.git/modules/` by a relative path, which a move leaves - pointing nowhere (`fatal: not a git repository`). So the `mv` runs only - for a directory apart from the row's with no `.git` entry, the empty one - a fresh clone leaves or a plain one, and the text says where a - submodule checkout's commits are kept, in the store its `.git` file - names (not always `.git/modules/`: a checkout a plain `mv` - brought there names the old one, and git cannot open it), what deleting - it would lose, and to delete it by hand once nothing only it holds is - needed. A clone's own `.git` DIRECTORY holds its commits and nothing - else does, so the text names the `git log` that lists the ones not - pushed and says to move it out whole, not delete it (rv681's S2 and - N1, Direction B: the `-e` guard Brett Heap's word took is kept). + EACH `mv` IS GUARDED BY TESTS A POSIX SHELL RUNS (#681, rv611c's N-1 + and N-2; `_guarded_move`), not by a system's name: this text is read in + CI too, on another disk than the one it is typed on. `[ -ef + ]` is true where the disk takes the two names for one directory, + the row's own, as a Linux mount that folds case does too (WSL's + `/mnt/c`), where the `mv` moved the member's own checkout out. `[ -e + /.git ]` is true for a checkout, and a submodule's `.git` is a + file naming this holder's `.git/modules/` by a relative path, which a + move leaves pointing nowhere (`fatal: not a git repository`). `[ ! -e + ]` is true where nothing is there, where the bare `mv` failed + (#839). A symbolic link there is asked first, `[ ! -L ]`, and + always moved, since the other tests read through it (#839). So the + `mv` runs only for a link, or a directory apart from the row's with no + `.git` entry, the empty one a fresh clone leaves or a plain one, and + the text says where a submodule checkout's commits are kept, in the + store its `.git` file names (not always `.git/modules/`: a + checkout a plain `mv` brought there names the old one, and git cannot + open it), what deleting it would lose, and to delete it by hand once + nothing only it holds is needed. A clone's own `.git` DIRECTORY holds + its commits and nothing else does, so the text names the `git log` that + lists the ones not pushed and says to move it out whole, not delete it + (rv681's S2 and N1, Direction B: the `-e` guard Brett Heap's word took + is kept). + + THE `git rm` TAKES `-f` where the index stages a variant at a commit + HEAD does not record there and a checkout is on disk at it + (`MoveContext.forced`, `_forced`, #839): git refuses it there wherever + that checkout has moved on or holds a file it does not track, and the + text says what `-f` does (`FORCED_SAID`). """ taken_out = " ".join(shlex.quote(variant) for variant in variants) - moved = " and ".join( - f"`[ {shlex.quote(variant)} -ef {shlex.quote(path)} ] || " - f"[ -e {shlex.quote(variant + '/.git')} ] || " - f"mv -- {shlex.quote(variant)} " - f"{shlex.quote('../' + variant.replace('/', '-'))}`" - for variant in variants) + moved = " and ".join(_guarded_move(variant, path) for variant in variants) status = " or ".join(f"`git -C {shlex.quote(variant)} status`" for variant in variants) unpushed = " or ".join(f"`git -C {shlex.quote(variant)} log --oneline " "HEAD --branches --not --remotes`" for variant in variants) - flag = " --sparse" if context.sparse else "" + flag = (" --sparse" if context.sparse else "") + ( + " -f" if context.forced else "") return ( _entries_aside(path, context.in_the_way) + ("Then move" if context.in_the_way else "Move") @@ -748,10 +792,7 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], "the index alone, which touches no file, with the " + ("first two lines below" if context.section else "two lines below") - + (f" (the first with --sparse, which needs git 2.34 or later: a " - f"sparse checkout keeps {' and '.join(context.sparse)} off disk, " - "and without it git rm passes over what that keeps off disk, exit " - "0, and removes nothing)" if context.sparse else "") + + _first_line_flags(context) + (f"; the lines after them {context.section_said}, and stage " f"{GITMODULES}. Commit that." if context.section else ", and commit that.") @@ -761,9 +802,11 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], "(WSL's /mnt/c). Where the disk holds the two apart it stays there, " "untracked, and family.py add refuses it as " "family-members-dir-variant: move each one that is there out of the " - f"holder, in a POSIX shell, {moved}, which moves nothing where the " - "two names are one directory or where it holds a .git entry, a " - "checkout. A checkout git submodule add made holds a .git file " + f"holder, in a POSIX shell (Git Bash on Windows), {moved}, which " + "moves nothing where nothing is there, where the two names are one " + "directory or where it holds a .git entry, a checkout, and moves a " + "symbolic link there wherever it points, the link and never what it " + "points at. A checkout git submodule add made holds a .git file " "naming its store by a path relative to it, which a move would " "leave pointing nowhere. Its commits are kept under this holder's " ".git/modules/, in the directory its .git file names, whatever " @@ -786,6 +829,58 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], f"{shlex.quote(f'160000,{commit},{path}')}{context.section}") +#: What `-f` on a `git rm --cached` of a gitlink does, said where one is +#: printed with it (`_forced`, #839), as `family.py`'s move back says it +#: (#761). +FORCED_SAID = ("-f takes out a staged gitlink whose checkout has moved on or " + "holds a file git does not track, which git rm otherwise " + "refuses, and with --cached touches no file") + + +def _guarded_move(variant: str, path: str) -> str: + """The backticked `mv` that moves a directory left at `variant`, a case + variant of the row's `path`, out of the holder, each test a POSIX shell + runs, dash and bash alike (#681, #839). + + `[ ! -L ]` FIRST, so a symbolic link there is always moved, + the link and never what it points at: `-ef` and `-e` read through a + link, so the guard kept one that points at the row's own directory, or + at a clone elsewhere, and `family.py add` then refused it as + `family-members-dir-variant`, a link on disk and untracked (w840's + candidate 1). Then `[ ! -e ]`, true where nothing is there, + a fresh clone's, where the bare `mv` failed `cannot stat`, exit 1, and + stopped what was typed after it (Codex's P2 2 on #828, met again here); + a link pointing nowhere, which `-e` takes for nothing, is a link, and + moved. `[ -ef ]`, true where the disk takes the two + names for one directory, the row's own; `-L` asks the last component + alone, so a `Members` that is itself a link to `members`, which + `tests/test_validate_family_remedies_typed.py` emulates a folding disk + with, still moves nothing. `[ -e /.git ]`, true for a + checkout.""" + quoted = shlex.quote(variant) + return (f"`[ ! -L {quoted} ] && {{ [ ! -e {quoted} ] || " + f"[ {quoted} -ef {shlex.quote(path)} ] || " + f"[ -e {shlex.quote(variant + '/.git')} ]; }} || " + f"mv -- {quoted} " + f"{shlex.quote('../' + variant.replace('/', '-'))}`") + + +def _first_line_flags(context: MoveContext) -> str: + """What the wrong-case move says of the flags its first line, the `git + rm`, carries, or nothing: `--sparse` (#762) and `-f` (#839).""" + said = [] + if context.sparse: + said.append("--sparse, which needs git 2.34 or later: a sparse " + f"checkout keeps {' and '.join(context.sparse)} off " + "disk, and without it git rm passes over what that keeps " + "off disk, exit 0, and removes nothing") + if context.forced: + said.append(f"-f, since the index stages " + f"{' and '.join(context.forced)} at a commit HEAD does " + f"not record there, over a checkout: {FORCED_SAID}") + return f" (the first with {', and with '.join(said)})" if said else "" + + def _entries_aside(path: str, entries: tuple[str, ...]) -> str: """What the wrong-case move says first where the index holds `entries` at or under the row's `path` (#762's (g)), or nothing: they go aside @@ -1947,7 +2042,8 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], + _wrong_case_move(row_path, row.pin, variants, _move_context(root, row_path, variants, - row.repository))) + row.repository, + gitlinks))) continue if kind != STRAY: report.finding( @@ -1958,7 +2054,7 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], report.finding( "member-gitlink-unnamed", f"{said}{_variant_said(row_path, row)} " - + _second_path_out(path, path in removed)) + + _second_path_out(root, path, commit, path in removed)) continue parts = path.split("/") project = parts[1] if len(parts) == 2 and parts[1] else None @@ -1976,14 +2072,31 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], "did not match any files).") -def _second_path_out(path: str, staged: bool) -> str: - """What a `STRAY` gitlink, a second path to a member's own directory, - says to do: take it out of the index alone, or, where that is staged - already (#681), commit it.""" +def _second_path_out(root: Path, path: str, commit: str, + staged: bool) -> str: + """What a `STRAY` gitlink at `path`, a second path to a member's own + directory, which `_member_gitlinks` reads at `commit`, says to do: take + it out of the index alone, with `-f` where git refuses it without + (`_forced`, #839), or, where that is staged already (#681), commit it. + + A `git mv members/

Members/

`, staged, is this state: the index + holds the variant, and HEAD the row's own gitlink, whose removal is + staged (`member-gitlink-removal-staged`). A commit or an untracked file + in the checkout `git mv` moved, and the plain `git rm --cached` refused + (w761's candidate 2 on #828, measured on the gate's state).""" if staged: return f"{STAGED_REMOVAL} Commit that removal." - return ("Take it out of the index alone, which touches no file, and " - f"commit that:\n git rm --cached -- {shlex.quote(path)}") + head = _forced(root, path, commit) + if head is None: + return ("Take it out of the index alone, which touches no file, and " + f"commit that:\n git rm --cached -- {shlex.quote(path)}") + return ("Take it out of the index alone, which touches no file" + + (", and commit that. The index stages it at a commit HEAD does " + "not record there" if head else + ": HEAD records nothing there, so nothing of it is left to " + "commit. The index stages it") + + f" over a checkout, so the line takes -f: {FORCED_SAID}:" + f"\n git rm --cached -f -- {shlex.quote(path)}") def _mounted_by_hand(root: Path, path: str, project: str | None, @@ -2130,7 +2243,8 @@ def _section_lines(path: str, repository, def _check_gitlinks_registered(root: Path, members: list, gitlinks: dict[str, str], - report: Report) -> set[str]: + report: Report, + put_back: bool = False) -> set[str]: """Every row's gitlink is at a path a section of HEAD's `.gitmodules` gives — the `member-gitmodules-absent` finding (#611, #407's decision 3). Returns the paths it was made for, of which `_check_member` asks no @@ -2164,6 +2278,15 @@ def _check_gitlinks_registered(root: Path, members: list, submodule update --init` dies there (`No url found for submodule path`, exit 128) before it checks out any member, and no mode named it. A path more than one section gives is `gitmodules-path-twice`'s. + + WHERE THE WORK TREE'S FILE IS GONE FROM DISK, the first of these + findings puts it back first (`_gitmodules_put_back`), and no other + does, nor any where an earlier finding put HEAD's back (`put_back`, + `gitmodules-value-refused`'s, #768): `git config -f` makes a file of + the lines it is given, so `member-gitmodules-absent`'s lines, staged and + committed, dropped every other section (rv820's review, comment + 6237354577 on #839), and a second `member-gitmodules-url-absent`'s put + the file back over the url the first had written (#839). """ head = _head_gitmodules(root) if head is None: @@ -2171,6 +2294,7 @@ def _check_gitlinks_registered(root: Path, members: list, held, sections = head given = Counter(keys["path"] for keys in sections.values() if "path" in keys) + back = ("", "") if put_back else _gitmodules_put_back(root)[1:] unregistered: set[str] = set() for row in members: project = row.get("project") if isinstance(row, dict) else None @@ -2182,35 +2306,59 @@ def _check_gitlinks_registered(root: Path, members: list, continue if path in given: if given[path] == 1 and _url_absent(root, path, gitlinks[path], - row, sections, report): + row, sections, report, back): unregistered.add(path) + back = ("", "") continue unregistered.add(path) - lines = _section_lines(path, row.get("repository"), sections) - report.finding( - "member-gitmodules-absent", - f"{path} is a gitlink at {gitlinks[path]} that a row of members: " - "names, and " - + (f"no section of HEAD's {GITMODULES} gives that path" if held - else f"HEAD holds no {GITMODULES}") - + ", so a recursive clone of this holder checks out nothing " - "there, and neither does make bootstrap in that clone, whatever " - "checkout this holder has of it. Write its section back by hand, " - "in a POSIX shell, as family.py add writes one, then stage and " - f"commit {GITMODULES}:{lines}") + report.finding("member-gitmodules-absent", _absent_said( + path, gitlinks[path], held, + _section_lines(path, row.get("repository"), sections), back)) + back = ("", "") return unregistered +def _absent_said(path: str, commit: str, held: bool, lines: str, + back: tuple[str, str]) -> str: + """`member-gitmodules-absent`'s text for the gitlink at `path`, at + `commit`, where `held` says whether HEAD holds a `.gitmodules`, with + `lines` (`_section_lines`) under it, after the line that puts the work + tree's file back where `back` gives one (`_back_first`, #839).""" + return ( + f"{path} is a gitlink at {commit} that a row of members: names, and " + + (f"no section of HEAD's {GITMODULES} gives that path" if held + else f"HEAD holds no {GITMODULES}") + + ", so a recursive clone of this holder checks out nothing there, " + "and neither does make bootstrap in that clone, whatever checkout " + "this holder has of it. Write its section back by hand, in a POSIX " + "shell, as family.py add writes one, then stage and commit " + f"{GITMODULES}{_back_first(back)}{lines}") + + +def _back_first(back: tuple[str, str]) -> str: + """The end of the sentence before a finding's `git config -f` lines, + with the line that puts the work tree's `.gitmodules` back first where + `back` is `(that line, whose file it puts back)`, or a colon where it is + empty (#762, #839).""" + put_back, whose = back + if not put_back: + return ":" + return (f", which is gone from the work tree: the first line puts back " + f"{whose}, so that every other section stays:\n {put_back}") + + def _url_absent(root: Path, path: str, commit: str, row: dict, - sections: dict[str, dict[str, str]], report: Report) -> bool: + sections: dict[str, dict[str, str]], report: Report, + back: tuple[str, str]) -> bool: """`member-gitmodules-url-absent` where the one section of HEAD's `.gitmodules` (`sections`) that gives `path` gives no url (#762); True where it was made. The url written is the one `family.py add` writes, `https://github.com/` and the row's `repository:`, or `` where that is not `/`, as `_section_lines` writes it. Where the - work tree's file is gone from disk, a line before it puts that back - (`_gitmodules_put_back`), since `git config -f` would make a file of - that one key, and staged it dropped every other section.""" + work tree's file is gone from disk, `back` is the line before it that + puts that back (`_gitmodules_put_back`), since `git config -f` would + make a file of that one key, and staged it dropped every other + section, and empty where an earlier finding put it back (#839).""" name = next(name for name, keys in sections.items() if keys.get("path") == path) if "url" in sections[name]: @@ -2219,7 +2367,6 @@ def _url_absent(root: Path, path: str, commit: str, row: dict, url = (shlex.quote(f"https://github.com/{repository}.git") if isinstance(repository, str) and QUALIFIED_RE.match(repository) else "") - _, put_back, whose = _gitmodules_put_back(root) report.finding( "member-gitmodules-url-absent", f"{path} is a gitlink at {commit} that a row of members: names, and " @@ -2229,11 +2376,8 @@ def _url_absent(root: Path, path: str, commit: str, row: dict, "path, exit 128) before it checks out any member, as make " "bootstrap's does, and a recursive clone checks out nothing there. " "Write its url back by hand, in a POSIX shell, as family.py add " - f"writes one, then stage and commit {GITMODULES}" - + (f", which is gone from the work tree: the first line puts back " - f"{whose}, so that every other section stays:\n {put_back}" - if put_back else ":") - + f"\n git config -f {GITMODULES} " + f"writes one, then stage and commit {GITMODULES}{_back_first(back)}" + f"\n git config -f {GITMODULES} " f"{shlex.quote(f'submodule.{name}.url')} {url}") return True @@ -2509,8 +2653,9 @@ def _check_members(root: Path, members: list, no_members: bool, removed = _staged_removals(root, gitlinks) _check_gitlinks_named(root, gitlinks, named, removed, report) _check_gitmodules_named(sections, gitlinks, named, removed, report) - unregistered = _check_gitlinks_registered(root, members, gitlinks, - report) + unregistered = _check_gitlinks_registered( + root, members, gitlinks, report, + refused and not os.path.lexists(root / GITMODULES)) unregistered |= _check_removals_staged(root, members, gitlinks, removed, as_read, report) if no_members: diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py index 715c05ad..347684df 100644 --- a/tests/test_bootstrap_spelling.py +++ b/tests/test_bootstrap_spelling.py @@ -652,8 +652,9 @@ def test_a_members_row_and_the_make_target_are_spelled(holder, tmp_path): assert (members / "Make" / "ran").exists(), \ "the member's command was handed the escaped target, not the raw one" assert err.strip().endswith( - r"family bootstrap FAILED: Fa\x1bil (make ta\u2028rget.py exited 2)" - ), err + r"family bootstrap FAILED: Fa\x1bil (make ta\u2028rget.py exited 2 " + "and printed no `Error ` line for a command of the member's: " + "make's own message above says what failed)"), err assert not {"\x1b", "\u2028"} & set(out), out diff --git a/tests/test_family_head_lost_gitmodules.py b/tests/test_family_head_lost_gitmodules.py index 50584dc9..c9983af9 100644 --- a/tests/test_family_head_lost_gitmodules.py +++ b/tests/test_family_head_lost_gitmodules.py @@ -658,21 +658,50 @@ def test_a_row_written_twice_is_one_finding_per_path(holder, offline, # --- 574/2: a gitlink recorded only at a case variant of the row's path ----- +#: What the move's first line says of its `-f` (#839), as `family.py`'s +#: move back says it (#761). +FORCED_SAID = ("-f takes out a staged gitlink whose checkout has moved on or " + "holds a file git does not track, which git rm otherwise " + "refuses, and with --cached touches no file") + + +def guarded_move(variant: str, path: str) -> str: + """The guarded `mv` the move prints for a directory left at `variant`: + a symbolic link there always moved, nothing there kept, and the row's + own directory or a checkout kept (#681, #839).""" + return (f"`[ ! -L {variant} ] && {{ [ ! -e {variant} ] || [ {variant} " + f"-ef {path} ] || [ -e {variant}/.git ]; }} || mv -- {variant} " + f"../{variant.replace('/', '-')}`") + + +def first_line_flags(sparse: tuple, forced: tuple) -> str: + """What the move says of its first line's `--sparse` (#762) and `-f` + (#839), or nothing.""" + said = [text for given, text in ( + (sparse, "--sparse, which needs git 2.34 or later: a sparse checkout " + f"keeps {' and '.join(sparse)} off disk, and without it git " + "rm passes over what that keeps off disk, exit 0, and " + "removes nothing"), + (forced, f"-f, since the index stages {' and '.join(forced)} at a " + f"commit HEAD does not record there, over a checkout: " + f"{FORCED_SAID}")) if given] + return f" (the first with {', and with '.join(said)})" if said else "" + + def variant_refused(path: str, commit: str, variants: list[str], gitlinks: dict[str, str], *, sparse: tuple = (), aside: str = "", section: tuple = (), - section_said: str = "") -> list[str]: + section_said: str = "", forced: tuple = ()) -> list[str]: """`member-gitlink-absent`'s lines for a row whose gitlink is recorded only at `variants`. What the move meets beside them (#762): `sparse`, the variants a sparse checkout keeps off disk; `aside`, the sentence - for an entry in the way; and `section`, the lines that make - `.gitmodules` give `path`, with `section_said`, what they do.""" + for an entry in the way; `section`, the lines that make `.gitmodules` + give `path`, with `section_said`, what they do; and `forced`, the + variants the first line takes out with `-f` (#839).""" project = path.split("/", 1)[1] found = ", ".join(f"{variant} (at {gitlinks[variant]})" for variant in variants) - moved = " and ".join(f"`[ {variant} -ef {path} ] || [ -e {variant}/.git ] " - f"|| mv -- {variant} ../{variant.replace('/', '-')}`" - for variant in variants) + moved = " and ".join(guarded_move(variant, path) for variant in variants) status = " or ".join(f"`git -C {variant} status`" for variant in variants) unpushed = " or ".join(f"`git -C {variant} log --oneline HEAD --branches " "--not --remotes`" for variant in variants) @@ -687,10 +716,7 @@ def variant_refused(path: str, commit: str, variants: list[str], "row's path, at the commit the row pins, in the index alone, which " "touches no file, with the " + ("first two lines below" if section else "two lines below") - + (" (the first with --sparse, which needs git 2.34 or later: a " - f"sparse checkout keeps {' and '.join(sparse)} off disk, and " - "without it git rm passes over what that keeps off disk, exit 0, " - "and removes nothing)" if sparse else "") + + first_line_flags(sparse, forced) + (f"; the lines after them {section_said}, and stage .gitmodules. " "Commit that." if section else ", and commit that.") + " A directory left at " @@ -699,9 +725,11 @@ def variant_refused(path: str, commit: str, variants: list[str], "usually do and a Linux mount can (WSL's /mnt/c). Where the disk " "holds the two apart it stays there, untracked, and family.py add " "refuses it as family-members-dir-variant: move each one that is " - f"there out of the holder, in a POSIX shell, {moved}, which moves " - "nothing where the two names are one directory or where it holds a " - ".git entry, a checkout. A checkout git submodule add made holds a " + "there out of the holder, in a POSIX shell (Git Bash on Windows), " + f"{moved}, which moves nothing where nothing is there, where the two " + "names are one directory or where it holds a .git entry, a checkout, " + "and moves a symbolic link there wherever it points, the link and " + "never what it points at. A checkout git submodule add made holds a " ".git file naming its store by a path relative to it, which a move " "would leave pointing nowhere. Its commits are kept under this " "holder's .git/modules/, in the directory its .git file names, " @@ -719,7 +747,8 @@ def variant_refused(path: str, commit: str, variants: list[str], f"which that update points at {path}, so {status} reads {path} after " "it. The " + ("lines" if section else "two lines") + ":", " git rm -r --cached" + (" --sparse" if sparse else "") - + " --ignore-unmatch -- " + " ".join(variants), + + (" -f" if forced else "") + " --ignore-unmatch -- " + + " ".join(variants), f" git update-index --add --cacheinfo 160000,{commit},{path}", *(f" {line}" for line in section)] @@ -819,7 +848,7 @@ def case_sensitive_disk(tmp_path): #: The guarded `mv` the variant refusal gives for each variant, backticked: -#: two shell tests, then the move (#681). +#: the shell tests, then the move (#681, #839). GUARDED_MOVE = re.compile(r"`(\[ [^`]* \|\| mv -- [^`]*)`") diff --git a/tests/test_family_holder_bootstrap_texts.py b/tests/test_family_holder_bootstrap_texts.py new file mode 100644 index 00000000..2faa6988 --- /dev/null +++ b/tests/test_family_holder_bootstrap_texts.py @@ -0,0 +1,738 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The family holder's pinned `scripts/bootstrap.py` and +`scripts/validate-family.py`: seven lines of #839, each in the state it +names, each remedy typed as printed. + +THE ISSUE'S SEVEN LINES, each reproduced on #819's head (233a334e) before +this change, but where one already held there: + +- 1. `make bootstrap` told a person to set `SHAPE_LEGS_TOKEN` and, on the + next line, that no credential would check the members out: the advice was + printed for every failed `git submodule update`. It is printed only where + git asked for a credential, not where its init died at a gitlink it has + no url for or it read nothing from `.gitmodules`; beside the denial, it + is for the members the denial does not name. +- 2. A member's refusal and its finding both read `make bootstrap exited + 2`: make exits 2 for any command that fails, and for a failure of its + own. The line names the code of the member's command, read from make's + `Error ` line, or says make printed none. +- 3. In the gate's state after `git mv members/

Members/

`, staged, + with a commit or an untracked file in the checkout it moved, the printed + `git rm --cached -- Members/

` refused, exit 1 (`staged content + different from both the file and the HEAD`). It carries `-f`, with what + `-f` does, there and in the wrong-case move's `git rm`. +- 4. The wrong-case move's guarded `mv` kept a symbolic link at the variant + that points at the row's own directory or at a clone, so `family.py add` + then refused it; and with nothing at the variant, a fresh clone's, the + `mv` failed `cannot stat`, exit 1 (comment 6237599648 is the same fault). + A link is always moved, the link and never what it points at, and nothing + there moves nothing. +- 5. Comment 6237354577 (a): bootstrap read `.gitmodules` only from the + work tree. That HELD at 233a334e (#819's `gitmodules_source` reads the + index's, then HEAD's): the credential's skip here asks it again where + the work tree lost the file. +- 6. Comment 6237354577 (b): `member-gitmodules-absent`'s `git config -f` + lines, where the work tree's `.gitmodules` is gone, made a file of one + section, which staged and committed dropped every other. The first + finding that writes the file puts it back first, and no other does, + which a second `member-gitmodules-url-absent` did over the url the first + had written. + +THE SAME FAMILY, OFFLINE RULES AND HELPERS as +`tests/test_family_bootstrap_gate_order.py` (#648) and +`tests/test_validate_family_remedies_typed.py` (#681). A missing credential +is git's refusal of the file transport to the fixture's local remotes, as +there. A state that needs a directory or a link at a case variant is +skipped where the disk cannot hold one (`case_sensitive_disk`). +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import REPO, load_without_bytecode, rmtree, run_script +from test_family import FAMILY, SPARE, family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bootstrap_gate_order import ( + NO_GITLINK, NO_SECTION, NO_URL, OUTSIDE_CURE, SECTION_CURE, STOPPED, + UNMERGED, UNMERGED_CURE, UNNAMED_CURE, UNNAMED_STOPS, VALIDATOR_CURE, + bootstrap, cloned, edited_clone, failed_on, fetching, not_fetched, + skip_by_subset, type_line, variant, without_url, +) +from test_family_dry_run import isolated +from test_family_gitmodules_values import ( + DAMAGED, LOST, MEND_CURE, UNPARSED, refused_why, value_cure, with_line, +) +from test_family_head_lost_gitmodules import ( # noqa: F401 + FORCED_SAID, GUARDED_MOVE, absent, assert_passes_in_every_mode, + case_sensitive_disk, commit_all, committed_gitmodules, head_lost, + moved_as_printed, pinned, recorded_at, refused_at, staged_and_committed, + variant_refused, +) +from test_family_member_path import MEMBER, OTHER, raw +from test_family_remove_gitmodules import verb_args +from test_validate_family_remedies_typed import ( + MOVED_TO, VARIANT_PATH, mounted_at_the_variant, +) +from test_validate_family_typed_states import GONE, ROW, URL_ABSENT, gone +from test_validate_family_unnamed_mounts import ( + GITLINK_SAID, VARIANT, codes, commands, findings, spare_commits, + stage_gitlink, take_out_as_printed, +) + +# --- line 1: the credential advice ----------------------------------------- + +#: The advice (a) gives where git asked for a credential, alone and beside +#: the line that names members no credential would check out. +ADVICE = (" If {which} are PRIVATE, set SHAPE_LEGS_TOKEN (a fine-grained " + "PAT with contents:read on them) or export an App-minted " + "SHAPE_LEGS_APP_TOKEN, then re-run. What did arrive is " + "bootstrapped below.") +ALONE = ADVICE.format(which="the members") +BESIDE = ADVICE.format(which="the members the next line does not name") + +#: How each line bootstrap writes to stderr itself begins; git's own lines, +#: which (a) passes on under the first, begin otherwise. +OURS = (" FETCH INCOMPLETE ", " If ", " NOT CHECKED OUT, ", + " NO MEMBER UPDATED, ", "family bootstrap ") +FETCH_LINE = re.compile(r" FETCH INCOMPLETE \(git exit \d+\) using " + r"credential source 'none':") + + +def after_fetch_line(result) -> list[str]: + """Bootstrap's own stderr lines, which must begin with the fetch's, + after it: git's lines between are its own.""" + ours = [line for line in result.stderr.splitlines() + if line.startswith(OURS)] + assert ours, result.stderr + assert FETCH_LINE.fullmatch(ours[0]), result.stderr + return ours[1:] + + +def unmerged_gitmodules(root: Path, env: dict) -> None: + """`.gitmodules` held in the index only at a merge's three conflict + stages, each the blob it held, as a conflicted merge leaves it.""" + blob = raw(root, env, "rev-parse", ":.gitmodules").stdout.decode( + "ascii").strip() + raw(root, env, "update-index", "--index-info", stdin=( + f"0 {'0' * 40}\t.gitmodules\n" + + "".join(f"100644 {blob} {stage}\t.gitmodules\n" + for stage in (1, 2, 3))).encode("ascii")) + + +def plain_clone_of(set_up): + """A state `set_up` makes in the holder, then a plain clone of it.""" + def made(holder: Path, env: dict, family, tmp_path: Path) -> Path: + set_up(holder, env) + return cloned(holder, env, family, tmp_path, recurse=False) + return made + + +def edited(edit): + """HEAD's `.gitmodules` put through `edit` and committed, then a plain + clone of it.""" + def made(holder: Path, env: dict, family, tmp_path: Path) -> Path: + return edited_clone(holder, env, family, tmp_path, edit)[0] + return made + + +def unmerged_in_a_clone(holder: Path, env: dict, family, + tmp_path: Path) -> Path: + clone = cloned(holder, env, family, tmp_path, recurse=False) + unmerged_gitmodules(clone, env) + return clone + + +def unnamed_stop(name: str): + """rv648's S1 or H1, then a plain clone of it.""" + def made(holder: Path, env: dict, family, tmp_path: Path) -> Path: + UNNAMED_STOPS[name](holder, env, family) + return cloned(holder, env, family, tmp_path, recurse=False) + return made + + +def outside(holder: Path, env: dict, family, tmp_path: Path) -> Path: + """rv648's H2: a gitlink outside members/, then a plain clone of it.""" + stage_gitlink(holder, f"vendor/{SPARE}", spare_commits(family)[1]) + commit_all(holder, env, "A gitlink mounted by hand outside members/") + return cloned(holder, env, family, tmp_path, recurse=False) + + +ROWS = (f"members/{MEMBER}", f"members/{OTHER}") +STOP_AT_OTHER = STOPPED.format(path=f"members/{OTHER}") + +#: `(a plain clone in the state, why (a) says each row is no credential's, +#: what it says cures them)`: every reason `unfetchable` gives where git +#: exits non-zero (#648, #768). Git asked for no credential in any. +NO_CREDENTIAL_ASKED = { + "head-lost": (plain_clone_of(head_lost), (NO_SECTION, NO_SECTION), + VALIDATOR_CURE), + "variant": (plain_clone_of(variant), + (STOPPED.format(path=f"Members/{OTHER}"), NO_GITLINK), + f"{VALIDATOR_CURE} {UNNAMED_CURE}"), + "no-url": (edited(lambda data: without_url(data, OTHER)), + (STOP_AT_OTHER, NO_URL), + SECTION_CURE.format(path=f"members/{OTHER}")), + "unmerged": (unmerged_in_a_clone, (UNMERGED, UNMERGED), UNMERGED_CURE), + "value-refused": (edited(with_line("update = None")), + (refused_why("update", "None"),) * 2, + value_cure("update")), + "unparsed": (edited(lambda data: data + DAMAGED["conflict-marker"]), + (UNPARSED, UNPARSED), MEND_CURE), + "stray": (unnamed_stop("stray"), + (STOPPED.format(path=f"Members/{OTHER}"),) * 2, UNNAMED_CURE), + "hand-mount": (unnamed_stop("hand-mount"), + (STOPPED.format(path=f"members/{SPARE}"),) * 2, + UNNAMED_CURE), + "outside": (outside, (STOPPED.format(path=f"vendor/{SPARE}"),) * 2, + OUTSIDE_CURE.format(path=f"vendor/{SPARE}")), +} + + +@pytest.mark.parametrize("make, whys, cure", NO_CREDENTIAL_ASKED.values(), + ids=NO_CREDENTIAL_ASKED) +def test_no_advice_is_printed_beside_its_denial(family, holder, offline, + tmp_path, make, whys, cure): + """LINE 1, each reason a row is no credential's. Git died in its init, + or reading `.gitmodules`, and fetched nothing, so (a)'s line says both + members are no credential's, and the advice to set one printed just + above it said the opposite. Bootstrap's own lines on stderr are now the + fetch's, the denial and the failure, exactly.""" + env = fetching(isolated(offline, tmp_path)) + clone = make(holder, env, family, tmp_path) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert after_fetch_line(result) == [ + not_fetched(*zip(ROWS, whys), cure=cure), failed_on(MEMBER, OTHER)] + + +def as_cloned(root: Path, env: dict) -> None: + """The plain clone, as it was cloned.""" + + +#: A plain clone, and one whose work tree lost `.gitmodules`, not +#: committed, where git reads the index's or HEAD's (comment (a)). +CREDENTIAL_S = {"as-cloned": as_cloned, **LOST} + + +@pytest.mark.parametrize("lose", CREDENTIAL_S.values(), ids=CREDENTIAL_S) +def test_the_credential_s_skip_prints_the_advice_alone(family, holder, + offline, tmp_path, + lose): + """LINE 1's other half, and comment (a): a plain clone with no + credential for the members, where git asked for one and was refused. + The advice stands, alone, and the run ends ok. Where the work tree lost + `.gitmodules`, git reads the index's, or HEAD's, and bootstrap reads the + file git reads (#819), so the members are still the credential's, not + `no section of .gitmodules gives that path`.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, recurse=False) + lose(clone, env) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert after_fetch_line(result) == [ALONE] + assert result.stdout.endswith("\nfamily bootstrap ok\n"), result.stdout + + +def test_a_leg_the_credential_s_keeps_the_advice_with_every_member_out( + family, holder, offline, tmp_path): + """No misfire: every member checked out, and their legs refused for + want of a credential. No row is left out, so no line denies anything, + and the advice, which is for the legs too, stands.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, recurse=False) + raw(clone, fetching(env), "submodule", "update", "--init") + assert not (clone / ROW / "spec" / ".git").exists(), "premise: no legs" + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert after_fetch_line(result) == [ALONE] + + +def test_beside_a_denial_the_advice_is_for_the_members_it_does_not_name( + family, holder, offline, tmp_path): + """LINE 1, both kinds at once: a clone that chose MEMBER alone, whose + fetch git refuses for want of a credential, and OTHER's section with no + url, which git skips there and no credential cures. The advice, then + the denial for OTHER, the advice saying it is for the members the next + line does not name.""" + env = isolated(offline, tmp_path) + clone = edited(lambda data: without_url(data, OTHER))( + holder, env, family, tmp_path) + env = skip_by_subset(clone, env) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert after_fetch_line(result) == [ + BESIDE, + not_fetched((f"members/{OTHER}", NO_URL), + cure=SECTION_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)] + + +# --- line 2: a member's finding, its refusal, and make's own failure -------- + +#: `{project: its Makefile}`, every recipe for both targets (#839's line 2). +MAKEFILES = { + "Find": b"bootstrap validate:\n\t@echo 'Find: a finding' >&2; exit 1\n", + "Fine": b"bootstrap validate:\n\t@true\n", + "Refuse": b"bootstrap validate:\n\t@echo 'Refuse: refused' >&2; exit 2\n", + "NoRule": b"other:\n\t@true\n", +} + + +@pytest.mark.parametrize("target", ["bootstrap", "validate"]) +def test_a_finding_a_refusal_and_make_s_own_failure_are_told_apart( + holder, tmp_path, target): + """LINE 2. Four members, each checked out, whose `make ` runs a + command that exits 1 (a finding, as a member's `bootstrap FAILED`), one + that exits 0, one that exits 2 (a refusal), and none at all, make + failing on its own (`No rule to make target`). Make exits 2 for all + three failures, and the line said `exited 2` for each; it names the + command's own code now, read from make's `Error ` line, and says + so where make printed none. Make's own lines, and the members', reach + stderr as they did. `--members-make validate`, which has no script of + its own to call, the same.""" + if shutil.which("make") is None: + pytest.skip("no make on PATH") + (holder / "family.yaml").write_bytes( + b"name: InkRouter\nmembers:\n" + b"".join( + f" - project: {project}\n path: members/{project}\n" + .encode("ascii") for project in MAKEFILES)) + for project, makefile in MAKEFILES.items(): + (holder / "members" / project / ".git").mkdir(parents=True) + (holder / "members" / project / "Makefile").write_bytes(makefile) + + result = run_script(holder / "scripts" / "bootstrap.py", "--no-fetch", + "--members-make", target, cwd=holder, + env={"MAKE": ""}) + + assert result.returncode == 1, result.stdout + result.stderr + errors = result.stderr.splitlines() + assert errors[-1] == ( + f"family bootstrap FAILED: Find (make {target}: the member's command " + f"exited 1), Refuse (make {target}: the member's command exited 2), " + f"NoRule (make {target} exited 2 and printed no `Error ` line " + "for a command of the member's: make's own message above says what " + "failed)"), result.stderr + for said in ("Find: a finding", "Refuse: refused", + f"No rule to make target '{target}'"): + assert any(said in line for line in errors[:-1]), (said, errors) + assert f" --- Fine: make {target} ---" in result.stdout.splitlines() + + +# --- line 3: `-f` where git rm --cached refuses ------------------------------ + +#: What lands in the checkout `git mv` moved, as w761 measured it on #828. +WORK = {"commit": lambda checkout, env: raw( + checkout, env, "commit", "-q", "--allow-empty", "-m", + "Work there"), + "untracked": lambda checkout, env: ( + checkout / "scratch.txt").write_bytes(b"mine\n")} + +#: The `STRAY` finding for `Members/

`, staged where HEAD records nothing. +STAGED_STRAY = ( + GITLINK_SAID + VARIANT + " Take it out of the index alone, which touches " + "no file: HEAD records nothing there, so nothing of it is left to commit. " + "The index stages it over a checkout, so the line takes -f: " + + FORCED_SAID + ":\n git rm --cached -f -- {path}") + + +@pytest.mark.parametrize("work", WORK.values(), ids=WORK) +def test_a_git_mv_staged_with_work_in_the_checkout_is_undone_as_printed( + family, holder, offline, tmp_path, case_sensitive_disk, work): + """LINE 3, reproduced on the gate's own state as the issue asks: `mkdir + Members && git mv members/IRRS Members/IRRS`, staged, then a commit, or + an untracked file, in the checkout it moved. The gate names the + variant, a second path to the member's directory, and the row's own + gitlink, whose removal is staged. Its `git rm --cached` refused there, + exit 1, in every mode; it carries `-f` now and says what it does, and + every line printed, typed as printed, puts the member back as HEAD has + it, nothing left to commit. The gate then passes in every mode, and + the checkout's commit is still in the member's store.""" + env = fetching(isolated(offline, tmp_path)) + commit = pinned(holder, MEMBER) + (holder / "Members").mkdir() + raw(holder, env, "mv", ROW, VARIANT_PATH) + work(holder / VARIANT_PATH, env) + stray = STAGED_STRAY.format(path=VARIANT_PATH, commit=commit, + row_path=ROW, project=MEMBER) + + for extra in ((), ("--pins",), ("--no-members",)): + found = validate(holder, *extra) + assert found.returncode == 1, (extra, found.stdout + found.stderr) + assert codes(found) == ["member-gitlink-unnamed", + "gitmodules-path-unnamed", + "member-gitlink-removal-staged"], extra + assert findings(found)[0] == stray, extra + every = [line for finding in findings(validate(holder)) + for line in commands(finding)] + head = raw(holder, env, "rev-parse", "--verify", "--quiet", + f"HEAD:{VARIANT_PATH}", check=False) + for line in every: + type_line(holder, env, line) + + assert head.returncode != 0, "premise: HEAD records nothing there" + assert raw(holder, env, "diff", "--cached", "--quiet", + check=False).returncode == 0, "nothing left to commit" + assert_passes_in_every_mode(holder, 2) + assert raw(holder, env, "rev-parse", f"HEAD:{ROW}").stdout.decode( + "ascii").strip() == commit + + +def test_a_staged_second_path_with_nothing_on_disk_takes_no_f( + holder, offline, tmp_path): + """No misfire: a second path to MEMBER's directory, `Members/IRRS`, + staged and not committed, and nothing on disk there. Git takes the + gitlink out with no `-f`, so the line prints none; typed as printed, + the gate passes in every mode.""" + env = isolated(offline, tmp_path) + commit = pinned(holder, MEMBER) + stage_gitlink(holder, VARIANT_PATH, commit) + + (finding,) = findings(validate(holder, "--no-members")) + + assert finding == ( + GITLINK_SAID.format(path=VARIANT_PATH, commit=commit) + + VARIANT.format(row_path=ROW, project=MEMBER) + + " Take it out of the index alone, which touches no file, and " + f"commit that:\n git rm --cached -- {VARIANT_PATH}") + type_line(holder, env, commands(finding)[0]) + assert_passes_in_every_mode(holder, 2) + + +def test_a_checkout_moved_on_at_the_variant_is_moved_with_f( + family, holder, offline, tmp_path, case_sensitive_disk): + """LINE 3, the wrong-case move's own `git rm`: MEMBER recorded only at + `Members/IRRS`, by a committed `git submodule add`, then a commit in + that checkout staged in the holder, and an untracked file left in it. + The index stages the variant at a commit HEAD does not record there, + over a checkout that holds a file git does not track, so the plain + `git rm -r --cached` refused, exit 1, and the move stopped there. It + carries `-f` and says why; typed as printed, with the commit, the + section `git submodule add` wrote taken out as printed, and the + checkout deleted by hand as the text says, the gate passes in every + mode.""" + env = fetching(isolated(offline, tmp_path)) + commit = pinned(holder, MEMBER) + checkout = holder / VARIANT_PATH + mounted_at_the_variant(family, holder, env, commit, "submodule") + raw(checkout, env, "commit", "-q", "--allow-empty", "-m", "Work there") + work = raw(checkout, env, "rev-parse", "HEAD").stdout.decode().strip() + raw(holder, env, "add", "--", VARIANT_PATH) + (checkout / "scratch.txt").write_bytes(b"mine\n") + + refused = validate(holder) + + assert refused.returncode == 2, refused.stdout + refused.stderr + lines = refused.stderr.splitlines() + assert lines == variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: work}, + forced=(VARIANT_PATH,)) + moved_as_printed(holder, env, lines, [VARIANT_PATH]) + left = validate(holder) + assert codes(left) == ["gitmodules-path-unnamed"], left.stderr + take_out_as_printed(holder, findings(left)[0]) + rmtree(checkout) + assert_passes_in_every_mode(holder, 2) + store = holder / ".git" / "modules" / "Members" / MEMBER + assert raw(holder, env, f"--git-dir={store}", f"--work-tree={holder}", + "cat-file", "-e", work, check=False).returncode == 0, ( + "the commit is kept") + + +# --- line 4: what the guarded `mv` moves ------------------------------------- + +def link_to_the_row(holder: Path, tmp_path: Path) -> Path | None: + """A link to the row's own directory. Returns what a link there + points at, which must be there after the move.""" + (holder / "Members").mkdir() + os.symlink(f"../{ROW}", holder / VARIANT_PATH, target_is_directory=True) + return holder / ROW + + +def link_to_a_clone(holder: Path, tmp_path: Path) -> Path | None: + """A link to a clone beside the holder, a directory holding a `.git` + file.""" + clone = tmp_path / f"{MEMBER}-clone" + clone.mkdir() + (clone / ".git").write_bytes(b"gitdir: ../nowhere\n") + (holder / "Members").mkdir() + os.symlink(str(clone), holder / VARIANT_PATH, target_is_directory=True) + return clone + + +def dangling_link(holder: Path, tmp_path: Path) -> Path | None: + (holder / "Members").mkdir() + os.symlink("nowhere", holder / VARIANT_PATH) + return None + + +def nothing(holder: Path, tmp_path: Path) -> Path | None: + """Nothing at the variant, as a fresh clone of a holder whose index + records only the variant leaves it after the move.""" + return None + + +LAYOUTS = {"link-to-the-row": link_to_the_row, + "link-to-a-clone": link_to_a_clone, + "dangling-link": dangling_link, + "nothing": nothing} + + +@pytest.mark.parametrize("shell", ["sh", "bash"]) +@pytest.mark.parametrize("layout", LAYOUTS.values(), ids=LAYOUTS) +def test_a_link_or_nothing_at_the_variant_is_typed_as_printed( + family, holder, offline, tmp_path, case_sensitive_disk, layout, + shell): + """LINE 4 and comment 6237599648, in w840's state: MEMBER's gitlink + held only at `Members/IRRS`, committed, its checkout at its row's path. + After the move's two lines and their commit, what is at the variant, + then the guarded `mv` typed as printed, in dash (this host's `sh`) and + in bash. A link there, to the row's own directory, to a clone, or to + nowhere, is moved, the link and never what it points at: the guard + kept the first two, reading through them, and `family.py add` then + refused the link, on disk and untracked. Nothing there moves nothing + and exits 0, where the bare `mv` failed `cannot stat`, exit 1. The gate + then passes in every mode, and SPARE is added.""" + if shutil.which(shell) is None: + pytest.skip(f"no {shell} on PATH") + env = fetching(isolated(offline, tmp_path)) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + lines = refused_at(holder, commit, [VARIANT_PATH]) + moved_as_printed(holder, env, lines, [VARIANT_PATH]) + try: + pointed = layout(holder, tmp_path) + except OSError as exc: + pytest.skip(f"no symbolic link here: {exc}") + link = os.path.islink(holder / VARIANT_PATH) + target = os.readlink(holder / VARIANT_PATH) if link else None + (move,) = GUARDED_MOVE.findall(lines[1]) + + typed = subprocess.run([shell, "-c", move], cwd=str(holder), + capture_output=True, check=False) + + assert typed.returncode == 0, typed.stderr + assert not os.path.lexists(holder / VARIANT_PATH) + moved = tmp_path / MOVED_TO + assert os.path.islink(moved) is link + if link: + assert os.readlink(moved) == target + assert pointed is None or pointed.is_dir() + assert (holder / ROW / "project.yaml").is_file() + assert_passes_in_every_mode(holder, 2) + added = run_script(FAMILY, *verb_args("add", holder, family), env=env) + assert added.returncode == 0, added.stdout + added.stderr + assert_passes_in_every_mode(holder, 3) + + +# --- comment (b): the work tree's .gitmodules put back, once --------------- + +def put_back_first(finding: str, how: str) -> str: + """`finding` with the line that puts the work tree's `.gitmodules` + back from where git reads it (`GONE`), as its first command.""" + whose, line = GONE[how] + said = "then stage and commit .gitmodules:\n" + assert finding.count(said) == 1, finding + return finding.replace( + said, "then stage and commit .gitmodules, which is gone from the " + f"work tree: the first line puts back {whose}, so that every other " + f"section stays:\n {line}\n") + + +def typed_and_committed(root: Path, env: dict, result) -> None: + """Every command every finding prints, typed as printed, then + `.gitmodules` staged and committed, as each says.""" + for finding in findings(result): + for line in commands(finding): + type_line(root, env, line) + staged_and_committed(root, env, "Written back as printed") + + +@pytest.mark.parametrize("how", GONE) +def test_a_section_absent_from_head_is_written_into_the_file_put_back( + holder, offline, tmp_path, how): + """COMMENT (b): OTHER's section taken out of HEAD's `.gitmodules`, and + the work tree's file gone, deleted or its deletion staged. The finding + printed the `git config -f` lines alone, which made a file of OTHER's + section, so staged and committed HEAD lost MEMBER's: a second round. + The line that puts the file back from where git reads it comes first + now, in every mode; typed as printed and committed, HEAD keeps + MEMBER's section, and the gate passes in every mode.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", "HEAD:.gitmodules").stdout + committed_gitmodules(holder, env, section_taken_out(data, OTHER)) + gone(holder, env, how) + expected = put_back_first( + absent(OTHER, pinned(holder, OTHER), held=True), how) + + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(holder, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + assert findings(result) == [expected], extra + + typed_and_committed(holder, env, validate(holder)) + assert f'[submodule "{ROW}"]'.encode() in raw( + holder, env, "show", "HEAD:.gitmodules").stdout + assert_passes_in_every_mode(holder, 2) + + +def section_taken_out(data: bytes, project: str) -> bytes: + """`.gitmodules` bytes with `project`'s section taken out by hand.""" + section = re.compile(rb'\[submodule "members/' + project.encode() + + rb'"\]\n(?:\t[^\n]*\n)*') + return section.sub(b"", data) + + +#: `(the edit to HEAD's .gitmodules, each finding as `(url-less?, project)` +#: in the order printed)`: two findings that each write the file. +TWO_WRITE = { + "two-url-less": (lambda data: without_url(without_url(data, MEMBER), + OTHER), + ((True, MEMBER), (True, OTHER))), + "absent-then-url-less": (lambda data: without_url( + section_taken_out(data, MEMBER), OTHER), + ((False, MEMBER), (True, OTHER))), + "url-less-then-absent": (lambda data: section_taken_out( + without_url(data, MEMBER), OTHER), + ((True, MEMBER), (False, OTHER))), +} + + +@pytest.mark.parametrize("edit, made", TWO_WRITE.values(), ids=TWO_WRITE) +def test_two_findings_that_write_the_file_put_it_back_once( + holder, offline, tmp_path, edit, made): + """COMMENT (b)'s ORDER: two findings that write `.gitmodules`, with the + work tree's gone. Each `member-gitmodules-url-absent` put the file back + first, so the second put it back over the url the first had written, + and a `member-gitmodules-absent` before one had its section put back + over the same way: a second round. Only the first puts it back now; + typed as printed and committed, the gate passes in every mode.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", "HEAD:.gitmodules").stdout + committed_gitmodules(holder, env, edit(data)) + gone(holder, env, "deleted") + expected = [ + (URL_ABSENT.replace(OTHER, project).format( + commit=pinned(holder, project)) if url_less + else absent(project, pinned(holder, project), held=True)) + for url_less, project in made] + expected[0] = put_back_first(expected[0], "deleted") + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == expected + typed_and_committed(holder, env, result) + assert_passes_in_every_mode(holder, 2) + + +def test_a_value_finding_that_put_head_s_back_is_not_put_back_over( + holder, offline, tmp_path): + """No second put back after `gitmodules-value-refused`'s: MEMBER's + section given `update = None`, OTHER's taken out, committed, and the + work tree's file deleted. The value's finding puts HEAD's back first + (#768) and takes the key out; OTHER's `member-gitmodules-absent` then + writes into that file, and puts nothing back over the key it took + out. Typed as printed and committed, the gate passes in every mode.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", "HEAD:.gitmodules").stdout + line = f"\tpath = members/{MEMBER}\n".encode() + assert data.count(line) == 1, data + committed_gitmodules(holder, env, section_taken_out( + data.replace(line, line + b"\tupdate = None\n"), OTHER)) + gone(holder, env, "deleted") + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert codes(result) == ["gitmodules-value-refused", + "member-gitmodules-absent"] + assert commands(findings(result)[0])[0] == ( + "git checkout HEAD -- .gitmodules") + assert findings(result)[1] == absent(OTHER, pinned(holder, OTHER), + held=True) + typed_and_committed(holder, env, result) + assert_passes_in_every_mode(holder, 2) + + +# --- the guard's text, typed in both shells on every layout ---------------- + +#: `(what is at the variant, whether the guard moves it)` for the guard +#: alone, in a scratch tree: the issue's seven layouts. +GUARD_LAYOUTS = { + "nothing": (lambda root: None, False), + "dangling-link": (lambda root: os.symlink( + "nowhere", root / VARIANT_PATH), True), + "link-to-the-row": (lambda root: os.symlink( + f"../{ROW}", root / VARIANT_PATH), True), + "link-to-a-checkout": (lambda root: os.symlink( + "../../clone", root / VARIANT_PATH), True), + "Members-a-link-to-members": (None, False), + "a-directory-apart": (lambda root: (root / VARIANT_PATH).mkdir(), True), + "a-checkout-apart": (lambda root: ( + (root / VARIANT_PATH).mkdir(), + (root / VARIANT_PATH / ".git").write_bytes(b"gitdir: x\n")), False), +} + + +@pytest.fixture(scope="module") +def guarded_move(): + """The gate's own `_guarded_move`, loaded by path; `sys.path` as it + was.""" + kept = list(sys.path) + gate = load_without_bytecode( + REPO / "templates" / "family-root" / "scripts" / "validate-family.py", + "validate_family_holder_texts") + sys.path[:] = kept + return gate._guarded_move + + +@pytest.mark.parametrize("shell", ["sh", "bash"]) +@pytest.mark.parametrize("place, moves", GUARD_LAYOUTS.values(), + ids=GUARD_LAYOUTS) +def test_the_guard_moves_a_link_always_and_nothing_where_nothing_is( + tmp_path, case_sensitive_disk, guarded_move, shell, place, moves): + """The guarded `mv` the variant refusal prints, on its own, on the + issue's seven layouts, in dash and in bash: exit 0 on each, a link + always moved, the link itself; nothing there, `Members` itself a link + to `members` (how `tests/test_validate_family_remedies_typed.py` + emulates a folding disk), and a checkout apart kept.""" + if shutil.which(shell) is None: + pytest.skip(f"no {shell} on PATH") + root = tmp_path / "holder" + (root / ROW).mkdir(parents=True) + (root / ROW / ".git").write_bytes(b"gitdir: x\n") + (tmp_path / "clone").mkdir() + (tmp_path / "clone" / ".git").write_bytes(b"gitdir: y\n") + (move,) = GUARDED_MOVE.findall(guarded_move(VARIANT_PATH, ROW)) + try: + if place is None: + os.symlink("members", root / "Members", target_is_directory=True) + else: + (root / "Members").mkdir() + place(root) + except OSError as exc: + pytest.skip(f"no symbolic link here: {exc}") + there = os.path.lexists(root / VARIANT_PATH) + + typed = subprocess.run([shell, "-c", move], cwd=str(root), + capture_output=True, check=False) + + assert typed.returncode == 0, typed.stderr + assert os.path.lexists(tmp_path / MOVED_TO) is moves + assert os.path.lexists(root / VARIANT_PATH) is (there and not moves) + assert (root / ROW / ".git").is_file() From 77f8a6c8946f619b743faa5694a92295a12a096b Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 02:57:05 +0000 Subject: [PATCH 13/20] validate-family.py: `_check_gitlinks_registered` is handed `refused` as `_check_gitmodules_values` returns it, so `_check_members` gains no condition for the put-back `gitmodules-value-refused` already printed (#839) Where the work tree holds `.gitmodules`, `_gitmodules_put_back` names no line to put it back either way, so `refused` alone says the same as `refused and not os.path.lexists(root / GITMODULES)`, and the cognitive complexity of `_check_members` stays at its base's. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- .../family-root/scripts/validate-family.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index b37bb850..0c2109c1 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -2244,7 +2244,7 @@ def _section_lines(path: str, repository, def _check_gitlinks_registered(root: Path, members: list, gitlinks: dict[str, str], report: Report, - put_back: bool = False) -> set[str]: + refused: bool = False) -> set[str]: """Every row's gitlink is at a path a section of HEAD's `.gitmodules` gives — the `member-gitmodules-absent` finding (#611, #407's decision 3). Returns the paths it was made for, of which `_check_member` asks no @@ -2281,12 +2281,13 @@ def _check_gitlinks_registered(root: Path, members: list, WHERE THE WORK TREE'S FILE IS GONE FROM DISK, the first of these findings puts it back first (`_gitmodules_put_back`), and no other - does, nor any where an earlier finding put HEAD's back (`put_back`, - `gitmodules-value-refused`'s, #768): `git config -f` makes a file of - the lines it is given, so `member-gitmodules-absent`'s lines, staged and - committed, dropped every other section (rv820's review, comment - 6237354577 on #839), and a second `member-gitmodules-url-absent`'s put - the file back over the url the first had written (#839). + does, nor any where `.gitmodules` gives a value git dies on + (`refused`), whose `gitmodules-value-refused` puts HEAD's back first + (#768): `git config -f` makes a file of the lines it is given, so + `member-gitmodules-absent`'s lines, staged and committed, dropped every + other section (rv820's review, comment 6237354577 on #839), and a + second `member-gitmodules-url-absent`'s put the file back over the url + the first had written (#839). """ head = _head_gitmodules(root) if head is None: @@ -2294,7 +2295,7 @@ def _check_gitlinks_registered(root: Path, members: list, held, sections = head given = Counter(keys["path"] for keys in sections.values() if "path" in keys) - back = ("", "") if put_back else _gitmodules_put_back(root)[1:] + back = ("", "") if refused else _gitmodules_put_back(root)[1:] unregistered: set[str] = set() for row in members: project = row.get("project") if isinstance(row, dict) else None @@ -2653,9 +2654,8 @@ def _check_members(root: Path, members: list, no_members: bool, removed = _staged_removals(root, gitlinks) _check_gitlinks_named(root, gitlinks, named, removed, report) _check_gitmodules_named(sections, gitlinks, named, removed, report) - unregistered = _check_gitlinks_registered( - root, members, gitlinks, report, - refused and not os.path.lexists(root / GITMODULES)) + unregistered = _check_gitlinks_registered(root, members, gitlinks, + report, refused) unregistered |= _check_removals_staged(root, members, gitlinks, removed, as_read, report) if no_members: From c3f8fa542461091f0704e6f0732c93e5e1dd58ee Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 07:18:55 +0000 Subject: [PATCH 14/20] validate-family.py: `gitmodules-path-unnamed` takes the gitlink at a section's path outside `members/` out of the index before the section, and a gitlink there that no section gives is `gitlink-unnamed`, so the remedy typed as printed no longer leaves a gitlink at which a plain clone's `git submodule update --init` exits 128 while the gate passes (#878) `_member_gitlinks` listed gitlinks in `members/` alone, so no mode asked about one elsewhere. `git submodule add tools/x`, committed, or a member added at `Members/IRRS` and moved with `git mv` to `renamed-1`: the gate printed only the section's `--unset-all` lines, and typed as printed they left the gitlink. The gate then passed in every mode while `git submodule status` and a plain clone's update exited 128 with no member checked out, which is how the holder's CI checks its members out. - `_member_gitlinks` lists every gitlink (`_in_members` says which are in `members/`); `member-gitlink-unnamed` still asks only those. - `gitmodules-path-unnamed` for a section at such a gitlink prints `git rm --cached -- ` and its commit first, then its own lines, once per path (`_gitlink_out_first`). Where HEAD's `.gitmodules` holds no section of its name, it says to stage the file and that nothing of it is left to commit. - `gitlink-unnamed` (`_check_gitlinks_elsewhere`): a gitlink outside `members/` that no section of the work tree's `.gitmodules` gives, with bootstrap's cure, `git rm --cached` and the commit, or the commit alone where its removal is staged (`_second_path_out`). - Each of those `git rm --cached` lines takes `-f`, with #828's clause, where the index stages the gitlink at a commit HEAD does not record and a checkout is on disk at it (`_forced`, #876), and comes after `git add -- .gitmodules` where lines printed before it may leave that file unstaged, or it is (`_stage_first`): git takes no gitlink out of the index then (`please stage your changes to .gitmodules`, exit 128), measured after a value finding's lines (#879's C3 order), an earlier unnamed section's, and a hand edit. Tests: tests/test_family_unnamed_gitlink.py (new, 17 cases), and two tests that pinned the old scope: `test_only_members_is_asked_about` and rv648's H2 test, whose `--no-members` now names bootstrap's own line. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- .../family-root/scripts/validate-family.py | 283 ++++++++- tests/test_family_bootstrap_gate_order.py | 15 +- tests/test_family_unnamed_gitlink.py | 545 ++++++++++++++++++ tests/test_validate_family_unnamed_mounts.py | 11 +- 4 files changed, 821 insertions(+), 33 deletions(-) create mode 100644 tests/test_family_unnamed_gitlink.py diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 0c2109c1..96ea4bc9 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -88,6 +88,19 @@ which would fail (#681). One no section of HEAD's `.gitmodules` gives a url is said to be checked out nowhere by a recursive clone, and to stop a plain clone's `git submodule update --init` (#762). + - that every gitlink OUTSIDE `members/`, which no row can name, is at a + path a section of `.gitmodules` gives, the `gitlink-unnamed` finding + (#878): a plain clone's `git submodule update --init` stops at one no + section gives (`No url found for submodule path`, exit 128) before it + checks out any member, as the holder's CI runs it, so it is taken out + of the index. One a section gives is that section's + `gitmodules-path-unnamed`, which takes the gitlink out of the index + first, then the section: taken out alone, the section left the + gitlink, and this gate passed it. Each `git rm --cached` comes after + `git add -- .gitmodules` where lines printed before it may leave that + file with changes not staged, or it holds some, since git takes no + gitlink out of the index then. Neither needs a member checked out, + so `--no-members` asks them too. - that a section of HEAD's `.gitmodules` gives the path of every row's gitlink, the `member-gitmodules-absent` finding (#611, #407's decision 3). A recursive clone reads HEAD's file, so a gitlink no section of it @@ -1743,10 +1756,21 @@ def _check_gitmodules_paths(sections: dict[str, dict[str, str]], f"inherits none of them, then commit {GITMODULES}.") +def _in_members(path: str) -> bool: + """Whether `path` is in `members/`, the directory spelt in any case + (#361): `Members/Zed` is in `members/` on macOS and Windows.""" + return path.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR + + def _member_gitlinks(root: Path) -> dict[str, str]: - """`{path: commit}` for every gitlink this repository records in - `members/`, the directory spelt in any case: what `_check_gitlinks_named` - asks of each (#361). + """`{path: commit}` for every gitlink this repository records: what + `_check_gitlinks_named` asks of each in `members/`, the directory spelt + in any case (#361, `_in_members`), and `_check_gitlinks_elsewhere` and + `_check_gitmodules_named` of each outside it (#878). It listed + `members/` alone, so no mode asked about a gitlink elsewhere, and + `gitmodules-path-unnamed`'s remedy for a section that gave one took + out the section and left the gitlink, at which a plain clone's `git + submodule update --init` stops. IN THE ORDER `recorded_gitlink` READS ONE: the index at stage 0 first, HEAD second. A path the index holds at stage 0 as a gitlink is answered @@ -1792,8 +1816,7 @@ def _member_gitlinks(root: Path) -> dict[str, str]: if stage_at is not None and fields[stage_at] != b"0": continue path = name.decode("utf-8", "surrogateescape") - if path.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR: - found[path] = fields[oid_at].decode("ascii").lower() + found[path] = fields[oid_at].decode("ascii").lower() return found @@ -2008,6 +2031,10 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], still a gitlink HEAD records, so it is still named, and its `git rm` typed again failed: it says to commit the removal instead and prints no `git rm` (#681). A hand mount's keeps the branch and the `add`. + + A gitlink outside `members/` is no member's mount, and no row can name + it: `_check_gitlinks_elsewhere` and `_check_gitmodules_named` ask + about it (#878). """ holds = set(gitlinks) head = _head_gitmodules(root) if set(gitlinks) - set(named) else None @@ -2015,7 +2042,7 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], if "path" in keys and "url" in keys} if head is not None else set()) for path, commit in sorted(gitlinks.items()): - if path in named: + if path in named or not _in_members(path): continue said = (f"{path} is a gitlink at {commit} that no row of members: " + ("names, so a recursive clone of this holder fetches it " @@ -2073,7 +2100,7 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], def _second_path_out(root: Path, path: str, commit: str, - staged: bool) -> str: + staged: bool, stage_first: bool = False) -> str: """What a `STRAY` gitlink at `path`, a second path to a member's own directory, which `_member_gitlinks` reads at `commit`, says to do: take it out of the index alone, with `-f` where git refuses it without @@ -2083,20 +2110,148 @@ def _second_path_out(root: Path, path: str, commit: str, holds the variant, and HEAD the row's own gitlink, whose removal is staged (`member-gitlink-removal-staged`). A commit or an untracked file in the checkout `git mv` moved, and the plain `git rm --cached` refused - (w761's candidate 2 on #828, measured on the gate's state).""" + (w761's candidate 2 on #828, measured on the gate's state). + + `gitlink-unnamed` says the same of a gitlink outside `members/` that no + section gives (#878), after `git add -- .gitmodules` where + `stage_first` (`_stage_first`).""" if staged: return f"{STAGED_REMOVAL} Commit that removal." head = _forced(root, path, commit) + return ((f"{STAGE_SAID[0].upper()}{STAGE_SAID[1:]}take" if stage_first + else "Take") + + " it out of the index alone, which touches no file" + + _forced_said(head, "the second line" if stage_first + else "the line") + + ":\n " + (f"{STAGE_LINE}\n " if stage_first else "") + + _cached_removal(path, head)) + + +#: The line the `git rm --cached` of a gitlink outside `members/` comes +#: after where `.gitmodules` may hold changes not staged (`_stage_first`, +#: #878), and what the text says of it, before what it says of the `git rm`. +STAGE_LINE = f"git add -- {GITMODULES}" +STAGE_SAID = (f"stage {GITMODULES} with the first line, since git rm refuses " + "to take any gitlink out of the index, -f or not, while " + f"{GITMODULES} holds changes not staged (please stage your " + f"changes to {GITMODULES} or stash them to proceed, exit 128), " + "as lines printed above may leave it or as it stands, then ") + + +def _stage_first(root: Path, edited: bool) -> bool: + """Whether the `git rm --cached` that `gitlink-unnamed` and + `gitmodules-path-unnamed` print for a gitlink outside `members/` comes + after `git add -- .gitmodules` (`STAGE_LINE`, #878). + + GIT TAKES NO GITLINK OUT OF THE INDEX, `--cached` and `-f` or not, + while the work tree's `.gitmodules` holds changes the index does not + (`please stage your changes to .gitmodules or stash them to proceed`, + exit 128, `is_staging_gitmodules_ok` in git's rm.c), measured on git + 2.43 after a `gitmodules-value-refused` finding's `--unset-all` lines + (the order #879's C3 names for a stray's `git rm --cached`), after an + earlier unnamed section's lines, and over a hand edit not staged. So it + does where `edited`, where lines printed before it may write the file, + and where the file on disk differs from the index's as it stands: its + bytes hashed as `git add` would store them (`git hash-object`) against + the blob the index holds, as git compares them, and never by stat, + which a file written in the same second as the index may fool. The + commit the text names next commits what it stages, which the lines + before it say to commit. Not where the work tree holds no + `.gitmodules` and no line before it writes one, where `git add` would + stage its deletion, nor where the index holds none at stage 0: git + asks nothing of the file then.""" + if edited: + return True + if not os.path.lexists(root / GITMODULES): + return False + with _git_runs(f"whether {GITMODULES} holds changes not staged could not " + "be read"): + try: + staged = git_out(["rev-parse", "--verify", "--quiet", + f":{GITMODULES}"], cwd=root) + except Refusal: + return False + return git_out(["hash-object", "--", GITMODULES], cwd=root) != staged + + +def _forced_said(head: str | None, line: str) -> str: + """What the text says after "take it out of the index alone, which + touches no file" of the `git rm --cached` it prints as `line`, where + `_forced` answered `head` (#839): that the removal is committed, and + where the line takes `-f`, why, and what `-f` does (`FORCED_SAID`), + and that nothing is left to commit where HEAD records nothing.""" if head is None: - return ("Take it out of the index alone, which touches no file, and " - f"commit that:\n git rm --cached -- {shlex.quote(path)}") - return ("Take it out of the index alone, which touches no file" - + (", and commit that. The index stages it at a commit HEAD does " - "not record there" if head else - ": HEAD records nothing there, so nothing of it is left to " - "commit. The index stages it") - + f" over a checkout, so the line takes -f: {FORCED_SAID}:" - f"\n git rm --cached -f -- {shlex.quote(path)}") + return ", and commit that" + return ((", and commit that. The index stages it at a commit HEAD does " + "not record there" if head else + ": HEAD records nothing there, so nothing of it is left to " + "commit. The index stages it") + + f" over a checkout, so {line} takes -f: {FORCED_SAID}") + + +def _cached_removal(path: str, head: str | None) -> str: + """`git rm --cached -- `, with `-f` where `_forced` answered + `head` (#839).""" + return ("git rm --cached" + ("" if head is None else " -f") + + f" -- {shlex.quote(path)}") + + +def _check_gitlinks_elsewhere(root: Path, gitlinks: dict[str, str], + named: dict[str, RowPath], + removed: frozenset[str], + report: Report) -> None: + """Every gitlink OUTSIDE `members/` that no row names is at a path a + section of `.gitmodules` gives, the `gitlink-unnamed` finding (#878). + + NO ROW CAN NAME ONE, since a member's path is `members/` + (`member-path`), and NO CLONE CAN CHECK OUT ONE NO SECTION GIVES: a + plain clone's `git submodule update --init` stops there (`No url found + for submodule path`, exit 128) before it checks out any member, which + is how the holder's CI checks its members out before it runs the + member half of this gate, a recursive clone checks out nothing at it, + and `git submodule status` exits 128 (`no submodule mapping found`). + No mode asked about one: `_member_gitlinks` listed `members/` alone, so + `gitmodules-path-unnamed`'s `--unset-all` lines, typed as printed, + left this state and the gate passed it. Bootstrap's `init_stop` names + it, with its cure (`cures`), and so does this, in its words: `git rm + --cached`, which touches no file, with `-f` where git refuses it + without (`_forced`, #839), and the commit, or, where its removal is + staged already (`removed`), the commit alone (`_second_path_out`). + + THE WORK TREE'S `.gitmodules`, read whole, as `_check_gitmodules_named` + reads it where it gives no value git dies on: a gitlink a section + there gives is that section's finding, which takes the gitlink out + first, so each such gitlink is one finding, never two. Where the file + gives a value git dies on, `gitmodules-path-unnamed` reads none of + its sections, and a gitlink one of them gives is said once that is + mended. + + `git add -- .gitmodules` COMES FIRST where a `gitmodules-value-refused` + finding, printed before this one, may leave the file with changes not + staged, or it holds some as it stands (`_stage_first`): git takes no + gitlink out of the index then, and the `git rm` exited 128.""" + unnamed = [path for path in sorted(gitlinks) + if path not in named and not _in_members(path)] + if not unnamed: + return + given = {keys["path"] for keys in _gitmodules_sections(root).values() + if "path" in keys} + stage_first = _stage_first(root, + "gitmodules-value-refused" in report.codes) + for path in unnamed: + if path in given: + continue + report.finding( + "gitlink-unnamed", + f"{path} is a gitlink at {gitlinks[path]}, outside " + f"{DEFAULT_MEMBERS_DIR}/, that no section of {GITMODULES} gives, " + "so in a plain clone of this holder git submodule update --init " + "stops there (No url found for submodule path, exit 128) before " + "it checks out any member, and a recursive clone checks out " + f"nothing at it. No row names {path} and no clone can check it " + "out. " + + _second_path_out(root, path, gitlinks[path], path in removed, + stage_first)) def _mounted_by_hand(root: Path, path: str, project: str | None, @@ -2143,7 +2298,7 @@ def _mounted_by_hand(root: Path, path: str, project: str | None, f" git rm -- {shlex.quote(path)}\n{add}") -def _check_gitmodules_named(sections: dict[str, dict[str, str]], +def _check_gitmodules_named(root: Path, sections: dict[str, dict[str, str]], gitlinks: dict[str, str], named: dict[str, RowPath], removed: frozenset[str], report: Report) -> None: @@ -2173,10 +2328,24 @@ def _check_gitmodules_named(sections: dict[str, dict[str, str]], sections give the path (`gitmodules-path-twice`) it takes out only the one git reads for it, the last in the file, so each of those keeps its own commands. + + A SECTION AT THE PATH OF A GITLINK OUTSIDE `members/` (#878), which no + row can name, and which the index holds, prints the `git rm --cached` + that takes the gitlink out first, and the commit, then its own lines + (`_gitlink_out_first`): its lines alone took out the section and left + the gitlink, which no section then gave, and the gate passed it while + a plain clone's `git submodule update --init` stopped there. Once for + each path, by the first section that gives it: the `git rm` typed + again fails. A gitlink whose removal is staged prints none, as above. + After a finding whose lines write `.gitmodules`, a value's or an + earlier section's (`edited`), the `git rm` comes after `git add -- + .gitmodules`, which it needs (`_stage_first`). """ holds = {keys["path"] for keys in sections.values() if "path" in keys} given = Counter(keys["path"] for keys in sections.values() if "path" in keys) + taken: set[str] = set() + edited = "gitmodules-value-refused" in report.codes for name, keys in sections.items(): path = keys.get("path") if path is None or path in named: @@ -2202,13 +2371,22 @@ def _check_gitmodules_named(sections: dict[str, dict[str, str]], said += (" If its row was deleted or its path: edited by hand, " "put the row back as git last had it.") lead = "Otherwise take" - if path in gitlinks and path not in removed and given[path] == 1: + if (path in gitlinks and path not in removed and given[path] == 1 + and _in_members(path)): report.finding( "gitmodules-path-unnamed", f"{said} Otherwise the git rm printed above for the " f"gitlink at {path} takes this section out too, so no " "command is printed here.") continue + first = (_gitlink_out_first(root, name, path, gitlinks, removed, + taken, edited) + if variant is None else None) + then, line = f"then commit {GITMODULES}", "" + if first is not None: + first_said, line, then = first + said += first_said + lead = "Then, with the lines after it, take" removal = "".join( f"\n git config -f {GITMODULES} --unset-all " f"{shlex.quote(f'submodule.{name}.{key}')}" for key in keys) @@ -2216,7 +2394,64 @@ def _check_gitmodules_named(sections: dict[str, dict[str, str]], "gitmodules-path-unnamed", f"{said} {lead} the section out by hand, in a POSIX shell, every " "key it holds, so a submodule later given its name inherits none " - f"of them, then commit {GITMODULES}:{removal}") + f"of them, {then}:{line}{removal}") + edited = True + + +def _gitlink_out_first(root: Path, name: str, path: str, + gitlinks: dict[str, str], removed: frozenset[str], + taken: set[str], + edited: bool) -> tuple[str, str, str] | None: + """What `gitmodules-path-unnamed` adds for the section `name`, whose + `path` is a gitlink outside `members/` (#878), `(what it says, the + line it prints first, how it ends)`, or None where it adds nothing. + + THE GITLINK GOES FIRST, out of the index alone, with `git rm --cached`, + which touches no file and leaves the section, and the commit, as + `member-gitlink-unnamed` takes out a second path to a member; `-f` + where git refuses it without, with what `-f` does (`_forced`, + `FORCED_SAID`, #839). Taken out alone, the section left a gitlink no + section gives (`gitlink-unnamed`), at which a plain clone's `git + submodule update --init` stops (`No url found for submodule path`, + exit 128) before it checks out any member, and `git submodule status` + exits 128 (`no submodule mapping found`). It ends as the section's + finding does, `then commit .gitmodules`, but where HEAD's `.gitmodules` + holds no section of its name, as after a `git submodule add` not + committed: staged, the file is HEAD's again, and nothing of it is + left to commit. + + `git add -- .gitmodules` COMES FIRST where `edited`, where lines + printed before it may leave the file with changes not staged (a + `gitmodules-value-refused` finding's, an earlier unnamed section's), + or where it holds some as it stands (`_stage_first`): git takes no + gitlink out of the index then, and the `git rm` exited 128. + + None for a path in `members/`, whose gitlink has a finding of its own + (`member-gitlink-unnamed`), for one that is no gitlink, and one + whose removal is staged (`removed`), where a `git rm` fails (`pathspec + ... did not match`), and for one an earlier section's finding took out + (`taken`): the `git rm` typed again fails there too. A path it adds to + is added to `taken`.""" + if (path not in gitlinks or path in removed or _in_members(path) + or path in taken): + return None + taken.add(path) + head = _forced(root, path, gitlinks[path]) + in_head = _head_gitmodules(root) + then = (f"then commit {GITMODULES}" + if in_head is not None and name in in_head[1] else + f"then stage {GITMODULES}: HEAD's holds no section of that name, " + "so nothing of it is left to commit") + stage = _stage_first(root, edited) + nth = "the second line" if stage else "the first line" + return (" Otherwise take its gitlink out first, since a gitlink no " + "section gives stops a plain clone's git submodule update --init " + "(No url found for submodule path, exit 128) before it checks out " + "any member: " + (STAGE_SAID if stage else "") + + f"take it out of the index alone, which touches no file, with " + f"{nth}" + _forced_said(head, nth) + ".", + (f"\n {STAGE_LINE}" if stage else "") + + f"\n {_cached_removal(path, head)}", then) def _section_lines(path: str, repository, @@ -2586,7 +2821,8 @@ def _init_stop_cure(stopper: str, sections: dict[str, dict[str, str]], stops at, as `_init_stop_said` prints it: `--no-members` for a path two of `sections` give or a gitlink in `members/` no row names, the section written back for a row's own path (`named`), and `git rm --cached` for - a gitlink outside `members/`, which no mode here reads.""" + a gitlink outside `members/`, as `gitlink-unnamed` and + `gitmodules-path-unnamed` name it in every mode (#878).""" given = Counter(keys["path"] for keys in sections.values() if "path" in keys) if given[stopper] > 1: @@ -2597,7 +2833,7 @@ def _init_stop_cure(stopper: str, sections: dict[str, dict[str, str]], return (f"Write the section of {GITMODULES} for {stopper} back as " "`family.py add` writes one, with its path and url, and " f"commit {GITMODULES}.") - if stopper.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR: + if _in_members(stopper): return (f"`{PYTHON} scripts/validate-family.py --no-members` names " f"each gitlink in {DEFAULT_MEMBERS_DIR}/ that no row names, " "and the commit that cures it.") @@ -2653,7 +2889,8 @@ def _check_members(root: Path, members: list, no_members: bool, gitlinks = _member_gitlinks(root) removed = _staged_removals(root, gitlinks) _check_gitlinks_named(root, gitlinks, named, removed, report) - _check_gitmodules_named(sections, gitlinks, named, removed, report) + _check_gitlinks_elsewhere(root, gitlinks, named, removed, report) + _check_gitmodules_named(root, sections, gitlinks, named, removed, report) unregistered = _check_gitlinks_registered(root, members, gitlinks, report, refused) unregistered |= _check_removals_staged(root, members, gitlinks, removed, diff --git a/tests/test_family_bootstrap_gate_order.py b/tests/test_family_bootstrap_gate_order.py index fc3a7caa..6b687433 100644 --- a/tests/test_family_bootstrap_gate_order.py +++ b/tests/test_family_bootstrap_gate_order.py @@ -659,11 +659,11 @@ def test_a_hand_mount_outside_members_is_taken_out_of_the_index( family, holder, offline, tmp_path): """rv648's H2. SPARE's gitlink recorded by hand at `vendor/`, outside members/, with no row and no section, and committed: in a plain - clone git's init dies there before it checks out any member. No mode - of the validator reads a gitlink outside members/, and `--no-members` - says `family ok`, so the line names the `git rm --cached` that takes - the gitlink out. Typed as printed and committed, the same run checks - both members out, and the gate passes in every mode.""" + clone git's init dies there before it checks out any member. The line + names the `git rm --cached` that takes the gitlink out, and the + validator's `--no-members`, which said `family ok` there, names the same + line now, `gitlink-unnamed` (#878). Typed as printed and committed, the + same run checks both members out, and the gate passes in every mode.""" env = fetching(isolated(offline, tmp_path)) stopper = f"vendor/{SPARE}" stage_gitlink(holder, stopper, spare_commits(family)[1]) @@ -680,8 +680,11 @@ def test_a_hand_mount_outside_members_is_taken_out_of_the_index( failed_on(MEMBER, OTHER)], result.stderr assert f"No url found for submodule path '{stopper}'" in result.stderr fallback = validate(clone, "--no-members") - assert fallback.returncode == 0, "premise: it reads only members/" + assert fallback.returncode == 1, fallback.stdout + fallback.stderr (line,) = re.findall(r"`(git rm [^`]*)`", result.stderr) + (named,) = findings(fallback) + assert named.startswith(f"FINDING gitlink-unnamed: {stopper} "), named + assert commands(named) == [line], "the same line (#878)" type_line(clone, env, line) commit_all(clone, env, "The gitlink taken out as printed") diff --git a/tests/test_family_unnamed_gitlink.py b/tests/test_family_unnamed_gitlink.py new file mode 100644 index 00000000..6f922260 --- /dev/null +++ b/tests/test_family_unnamed_gitlink.py @@ -0,0 +1,545 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The family root's `validate-family.py`: a gitlink outside `members/` +that no row names, and the `gitmodules-path-unnamed` remedy that left one +(#878). + +THE ISSUE'S LINE, reproduced on #876's head (77f8a6c8) before this change: +`git submodule add tools/x`, committed, or IRSS added at +`Members/IRRS` and moved by hand with `git mv` to `renamed-1`, committed. +The gate exited 1 in every mode, `gitmodules-path-unnamed`, and printed +only the `--unset-all` lines that take the section out. Typed as printed, +with `.gitmodules` staged and committed, the gate exited 0 in every mode, +while `git submodule status` exited 128 (`no submodule mapping found`) +and a plain clone's `git submodule update --init --recursive` exited 128 +(`No url found for submodule path`) with no member checked out: the +gate's gitlinks were read in `members/` alone, so no mode asked about the +one the lines left. + +ITS THREE DIRECTIONS, each held here: + +- The section's finding prints the `git rm --cached` that takes its + gitlink out of the index first, and the commit, then its own lines. +- A gitlink outside `members/` that no section gives is a finding of its + own, `gitlink-unnamed`, whose `git rm --cached` is the one bootstrap's + `init_stop` cure names for it. +- Where the index stages that gitlink at a commit HEAD does not record + and a checkout is on disk at it, the `git rm` takes `-f`, with what + `-f` does, as #876's `_forced` gives it: git refused it there (`staged + content different from both the file and the HEAD`, exit 1). + +Each remedy is typed as printed, split as a POSIX shell splits it and run +with no shell, then the gate is asked in every mode; where HEAD changed, a +plain clone's `git submodule update --init --recursive` must check out +both members and `git submodule status` exit 0 there and in the holder. +WHERE LINES PRINTED BEFORE IT WRITE `.gitmodules` (a value finding's, in +the order #879's C3 names for a stray's `git rm --cached`, or an earlier +unnamed section's), or the file holds a hand edit not staged, git refused +#878's `git rm --cached` (`please stage your changes to .gitmodules or +stash them to proceed`, exit 128), so `git add -- .gitmodules` comes +directly before it there, and every line is typed in the order printed. +THE CONTROLS: the holder as `family.py` made it, a member under +`members/` with its section and its row, and a section with no gitlink, +or none in the index, which prints its own lines alone, as it did. + +THE SAME FAMILY, OFFLINE RULES AND HELPERS as +`tests/test_family_holder_bootstrap_texts.py` (#839) and +`tests/test_validate_family_unnamed_mounts.py` (#361). A checkout is git's +own, made by `git submodule add` from the fixture's local remotes; a state +with no checkout is an index record (`stage_gitlink`). The `git mv` form +needs `Members/IRRS` apart from `members/IRRS` on disk, and is skipped +where the disk folds case (`case_sensitive_disk`). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from conftest import git +from test_family import family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bootstrap_gate_order import cloned, fetching, type_line +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( # noqa: F401 + FORCED_SAID, assert_passes_in_every_mode, case_sensitive_disk, + commit_all, staged_and_committed, typed, +) +from test_family_holder_bootstrap_texts import WORK +from test_family_member_path import MEMBER, OTHER, raw +from test_validate_family_remedies_typed import STAGED +from test_validate_family_unnamed_mounts import ( + SECTION_REMEDY, SECTION_SAID, codes, commands, findings, removal, + spare_commits, stage_gitlink, with_section, +) + +#: The gitlink outside `members/` the issue mounts IRSS at, the case +#: variant of MEMBER's path it mounts it at for the `git mv`, and where +#: that moves it. +TOOLS = "tools/x" +VARIANT = f"Members/{MEMBER}" +RENAMED = "renamed-1" + +#: `gitlink-unnamed` up to what it says of its `git rm --cached`. A +#: holder's CI matches on text, so it is pinned here and not rebuilt from +#: the gate's own strings. +OUTSIDE_SAID = ( + "FINDING gitlink-unnamed: {path} is a gitlink at {commit}, outside " + "members/, that no section of .gitmodules gives, so in a plain clone of " + "this holder git submodule update --init stops there (No url found for " + "submodule path, exit 128) before it checks out any member, and a " + "recursive clone checks out nothing at it. No row names {path} and no " + "clone can check it out. ") + +#: What every text says first of the `git rm --cached` it prints. +TAKE_OUT = "take it out of the index alone, which touches no file" + +#: What it says first where lines printed before it may leave `.gitmodules` +#: with changes not staged, or it holds some, and the line it prints for +#: that: git takes no gitlink out of the index then (#879's C3). +STAGE_SAID = ("stage .gitmodules with the first line, since git rm refuses to " + "take any gitlink out of the index, -f or not, while " + ".gitmodules holds changes not staged (please stage your " + "changes to .gitmodules or stash them to proceed, exit 128), " + "as lines printed above may leave it or as it stands, then ") +STAGE_LINE = "\n git add -- .gitmodules" + +#: What it says next, by what `_forced` reads: the commit; or `-f`, where +#: the index stages the gitlink at a commit HEAD does not record there +#: (`FORCED_OVER`), or where HEAD records nothing (`FORCED_NO_HEAD`), over +#: a checkout. `{line}` names the line that takes it. +COMMIT_THAT = ", and commit that" +FORCED_OVER = (", and commit that. The index stages it at a commit HEAD " + "does not record there over a checkout, so {line} takes -f: " + + FORCED_SAID) +FORCED_NO_HEAD = (": HEAD records nothing there, so nothing of it is left to " + "commit. The index stages it over a checkout, so {line} " + "takes -f: " + FORCED_SAID) + +#: What `gitmodules-path-unnamed` says of a section no row names, before +#: what it says to do. +ROW_BACK = (" If its row was deleted or its path: edited by hand, put the " + "row back as git last had it.") + +#: What it says of the gitlink at a section's path outside `members/`, +#: which it takes out first, then of the section. +GITLINK_FIRST = ( + " Otherwise take its gitlink out first, since a gitlink no section " + "gives stops a plain clone's git submodule update --init (No url found " + "for submodule path, exit 128) before it checks out any member: ") +SECTION_THEN = (" Then, with the lines after it, take the section out by " + "hand, in a POSIX shell, every key it holds, so a submodule " + "later given its name inherits none of them, {then}:") + +#: How the section's text ends: the commit, or, where HEAD's `.gitmodules` +#: holds no section of its name, the file staged and nothing to commit. +THEN_COMMIT = "then commit .gitmodules" +THEN_STAGE = ("then stage .gitmodules: HEAD's holds no section of that name, " + "so nothing of it is left to commit") + + +def outside(path: str, commit: str, said: str, flag: str = "", *, + stage: bool = False) -> str: + """`gitlink-unnamed` whole for the gitlink at `path`, at `commit`: + `said` (`COMMIT_THAT`, `FORCED_OVER` or `FORCED_NO_HEAD`) of its `git + rm --cached`, which carries `flag`, after `git add -- .gitmodules` + where `stage`.""" + first = STAGE_SAID + TAKE_OUT if stage else TAKE_OUT + return (OUTSIDE_SAID.format(path=path, commit=commit) + + first[0].upper() + first[1:] + + said.format(line="the second line" if stage else "the line") + + ":" + (STAGE_LINE if stage else "") + + f"\n git rm --cached{flag} -- {path}") + + +def section_first(path: str, name: str, url: str, said: str, then: str, + flag: str = "", *, stage: bool = False) -> str: + """`gitmodules-path-unnamed` whole for the section `name`, giving `path` + and `url`, at a gitlink outside `members/`: its `git rm --cached`, + carrying `flag`, after `git add -- .gitmodules` where `stage`, of which + it says `said`, then its own lines, and `then`.""" + nth = "the second line" if stage else "the first line" + return (SECTION_SAID.format(path=path, name=name, url=f"url {url}") + + ROW_BACK + GITLINK_FIRST + (STAGE_SAID if stage else "") + + f"{TAKE_OUT}, with {nth}" + said.format(line=nth) + "." + + SECTION_THEN.format(then=then) + (STAGE_LINE if stage else "") + + f"\n git rm --cached{flag} -- {path}" + + "".join(f"\n {line}" for line in removal(name, "path", + "url"))) + + +def found_in_every_mode(root: Path, code: str) -> str: + """The one finding the gate prints, `code`'s, the same in every mode.""" + seen: set[str] = set() + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(root, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + assert codes(result) == [code], (extra, result.stderr) + seen.update(findings(result)) + (finding,) = seen + return finding + + +def staged_at(root: Path, env: dict, path: str) -> str: + """The commit the index stages the gitlink at `path` at.""" + return raw(root, env, "rev-parse", f":{path}").stdout.decode( + "ascii").strip() + + +def section_url(root: Path, env: dict, name: str) -> str: + """The url the work tree's `.gitmodules` gives the section `name`.""" + return raw(root, env, "config", "-f", ".gitmodules", "--get", + f"submodule.{name}.url").stdout.decode("utf-8").strip() + + +def assert_a_plain_clone_checks_out_every_member(root: Path, env: dict, + family, tmp_path: Path + ) -> None: + """`git submodule status` exits 0 in `root`; a plain clone of its HEAD, + then `git submodule update --init --recursive`, exits 0 there with both + members checked out, and `git submodule status` exits 0 there too.""" + raw(root, env, "submodule", "status") + clone = cloned(root, env, family, tmp_path, recurse=False) + raw(clone, env, "submodule", "update", "--init", "--recursive") + for project in (MEMBER, OTHER): + assert (clone / "members" / project / ".git").exists(), project + raw(clone, env, "submodule", "status") + + +def submodule_added(root: Path, env: dict, family, path: str) -> None: + """IRSS mounted at `path` by `git submodule add`, not committed.""" + raw(root, env, "submodule", "add", "-q", "--", + str(family["remotes"] / "IRSS.git"), path) + + +# --- the issue's two states -------------------------------------------------- + +def at_tools(root: Path, env: dict, family) -> tuple[str, str]: + """`git submodule add tools/x`, committed. Returns the section's + name and the path it gives.""" + submodule_added(root, env, family, TOOLS) + commit_all(root, env, "IRSS mounted at tools/x by hand") + return TOOLS, TOOLS + + +def moved_to_renamed(root: Path, env: dict, family) -> tuple[str, str]: + """IRSS added at `Members/IRRS`, committed, then moved by hand with + `git mv` to `renamed-1`, committed: git gives its section that path + under the name it had.""" + submodule_added(root, env, family, VARIANT) + commit_all(root, env, "IRSS mounted at Members/IRRS by hand") + raw(root, env, "mv", "--", VARIANT, RENAMED) + commit_all(root, env, "Moved by hand to renamed-1") + return VARIANT, RENAMED + + +#: `(how the state is made, whether it needs a disk that keeps case)`. +FORMS = {"tools-x": (at_tools, False), + "git-mv-to-renamed-1": (moved_to_renamed, True)} + + +@pytest.mark.parametrize("form, keeps_case", FORMS.values(), ids=FORMS) +def test_a_section_s_remedy_takes_its_gitlink_out_first( + family, holder, offline, tmp_path, request, form, keeps_case): + """THE ISSUE'S LINE, both forms. The section's finding, in every mode, + prints the `git rm --cached` of its gitlink first, and the commit, then + the `--unset-all` lines it printed alone. Typed as printed, with + `.gitmodules` staged and committed, the gate passes in every mode, and + a plain clone checks out both members: the lines alone left the + gitlink, and its update exited 128 at it.""" + if keeps_case: + request.getfixturevalue("case_sensitive_disk") + env = fetching(isolated(offline, tmp_path)) + name, path = form(holder, env, family) + + finding = found_in_every_mode(holder, "gitmodules-path-unnamed") + + assert finding == section_first(path, name, + section_url(holder, env, name), + COMMIT_THAT, THEN_COMMIT) + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + + +def section_taken_out_alone(root: Path, env: dict, family) -> None: + """THE STATE THE OLD REMEDY LEFT: tools/x's section taken out by the + lines `gitmodules-path-unnamed` printed alone, staged and committed.""" + at_tools(root, env, family) + for line in removal(TOOLS, "path", "url"): + type_line(root, env, line) + staged_and_committed(root, env, "The section taken out alone") + + +def test_a_gitlink_outside_members_no_section_gives_is_a_finding( + family, holder, offline, tmp_path): + """THE NEW FINDING, in the state the old remedy left, where every mode + said `family ok` while a plain clone's update stopped at the gitlink. + It names it in every mode, with bootstrap's `git rm --cached`; typed as + printed and committed, the gate passes in every mode, and a plain + clone checks out both members.""" + env = fetching(isolated(offline, tmp_path)) + section_taken_out_alone(holder, env, family) + before = cloned(holder, env, family, tmp_path / "before", recurse=False) + stopped = raw(before, env, "submodule", "update", "--init", "--recursive", + check=False) + assert stopped.returncode == 128, "premise: a plain clone stops there" + + finding = found_in_every_mode(holder, "gitlink-unnamed") + + assert finding == outside(TOOLS, staged_at(holder, env, TOOLS), + COMMIT_THAT) + typed(holder, env, finding) + commit_all(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + + +# --- -f where git refuses the plain git rm --cached -------------------------- + +def added_not_committed(root: Path, env: dict, family, work) -> None: + """rv865b's (a): `git submodule add tools/x`, not committed, + with `work` in its checkout. HEAD records nothing there.""" + submodule_added(root, env, family, TOOLS) + work(root / TOOLS, env) + + +def moved_on(root: Path, env: dict, work) -> None: + """A commit in tools/x's checkout, staged in the holder, where HEAD + records the one before it, then `work` in that checkout.""" + raw(root / TOOLS, env, "commit", "-q", "--allow-empty", "-m", + "Moved on") + raw(root, env, "add", "--", TOOLS) + work(root / TOOLS, env) + + +def committed_then_moved_on(root: Path, env: dict, family, work) -> None: + at_tools(root, env, family) + moved_on(root, env, work) + + +def no_section_then_moved_on(root: Path, env: dict, family, work) -> None: + section_taken_out_alone(root, env, family) + moved_on(root, env, work) + + +#: `(how the state is made, the finding's code, what it says of its -f, +#: how the section's text ends, or None for the gitlink's own finding)`. +FORCED = { + "section-added-not-committed": (added_not_committed, + "gitmodules-path-unnamed", + FORCED_NO_HEAD, THEN_STAGE), + "section-committed-moved-on": (committed_then_moved_on, + "gitmodules-path-unnamed", FORCED_OVER, + THEN_COMMIT), + "no-section-moved-on": (no_section_then_moved_on, "gitlink-unnamed", + FORCED_OVER, None), +} + + +@pytest.mark.parametrize("work", WORK.values(), ids=WORK) +@pytest.mark.parametrize("make, code, said, then", FORCED.values(), + ids=FORCED) +def test_a_gitlink_staged_over_a_checkout_is_taken_out_with_f( + family, holder, offline, tmp_path, make, code, said, then, work): + """DIRECTION 3: the index stages tools/x at a commit HEAD does not + record there, or where HEAD records nothing, and a checkout is on disk + at it that has moved on or holds a file git does not track. The plain + `git rm --cached` refuses there, exit 1, and changes nothing; the line + takes `-f`, in every mode, and says what it does. Typed as printed, + then staged and committed as the text says, or staged where it says + nothing is left to commit, the gate passes in every mode, the checkout + is still on disk, and where HEAD changed a plain clone checks out both + members.""" + env = fetching(isolated(offline, tmp_path)) + make(holder, env, family, work) + refused = raw(holder, env, "rm", "-q", "--cached", "--", TOOLS, + check=False) + assert refused.returncode == 1, "premise: the plain git rm refuses" + + finding = found_in_every_mode(holder, code) + + assert finding == ( + outside(TOOLS, staged_at(holder, env, TOOLS), said, " -f") + if then is None else + section_first(TOOLS, TOOLS, section_url(holder, env, TOOLS), said, + then, " -f")) + typed(holder, env, finding) + if then == THEN_STAGE: + raw(holder, env, "add", "--", ".gitmodules") + raw(holder, env, "diff", "--cached", "--quiet") + else: + staged_and_committed(holder, env, "Typed as printed") + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + assert_passes_in_every_mode(holder, 2) + assert (holder / TOOLS / ".git").is_file(), "-f touched no file" + + +# --- after lines that write .gitmodules, git add first (#879's C3) ---------- + +def value_first(root: Path, env: dict, family) -> str: + """#879's C3 order, before this finding: MEMBER's section given + `update = None`, committed, beside the state the old remedy left, so + the value finding's `--unset-all` line prints first. Returns the text + the gitlink's finding prints whole.""" + section_taken_out_alone(root, env, family) + raw(root, env, "config", "-f", ".gitmodules", + f"submodule.members/{MEMBER}.update", "None") + staged_and_committed(root, env, "update = None") + return outside(TOOLS, staged_at(root, env, TOOLS), COMMIT_THAT, + stage=True) + + +def earlier_section_first(root: Path, env: dict, family) -> str: + """A section no row names, for tools/y, with no gitlink, committed + before tools/x is added: its `--unset-all` lines print first.""" + for key, value in (("path", "tools/y"), ("url", "/nowhere/y.git")): + raw(root, env, "config", "-f", ".gitmodules", + f"submodule.tools/y.{key}", value) + staged_and_committed(root, env, "A section for tools/y") + at_tools(root, env, family) + return section_first(TOOLS, TOOLS, section_url(root, env, TOOLS), + COMMIT_THAT, THEN_COMMIT, stage=True) + + +def hand_edit_first(root: Path, env: dict, family) -> str: + """tools/x added and committed, then `.gitmodules` edited by hand, not + staged.""" + at_tools(root, env, family) + gitmodules = root / ".gitmodules" + gitmodules.write_bytes(gitmodules.read_bytes() + b"# a hand edit\n") + return section_first(TOOLS, TOOLS, section_url(root, env, TOOLS), + COMMIT_THAT, THEN_COMMIT, stage=True) + + +#: `(how the state is made, which returns the text #878's finding prints +#: last, the codes printed in order)`: lines printed before #878's `git rm +#: --cached` write `.gitmodules`, or it holds a change not staged. +STAGE_FIRST = { + "value-finding-first": (value_first, ["gitmodules-value-refused", + "gitlink-unnamed"]), + "earlier-section-first": (earlier_section_first, + ["gitmodules-path-unnamed", + "gitmodules-path-unnamed"]), + "hand-edit-not-staged": (hand_edit_first, ["gitmodules-path-unnamed"]), +} + + +@pytest.mark.parametrize("make, printed", STAGE_FIRST.values(), + ids=STAGE_FIRST) +def test_after_lines_that_write_gitmodules_its_git_rm_stages_it_first( + family, holder, offline, tmp_path, make, printed): + """#879's C3, for #878's own line. Git takes no gitlink out of the + index, `-f` or not, while `.gitmodules` holds changes not staged + (`please stage your changes to .gitmodules or stash them to proceed`, + exit 128), as a value finding's `--unset-all` lines leave it, or an + earlier unnamed section's, or a hand edit. So `git add -- .gitmodules` + comes directly before #878's `git rm --cached`, in every mode, and says + why. Every line of every finding typed in the order printed, with + nothing between, then `.gitmodules` staged and committed: the gate + passes in every mode, and a plain clone checks out both members.""" + env = fetching(isolated(offline, tmp_path)) + expected = make(holder, env, family) + + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(holder, *extra) + assert codes(result) == printed, (extra, result.stderr) + assert findings(result)[-1] == expected, extra + for finding in findings(result): + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed, in order") + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + + +# --- where no git rm is printed, or once ------------------------------------- + +def test_a_gitlink_outside_members_whose_removal_is_staged_says_commit_it( + family, holder): + """`gitlink-unnamed` for a gitlink HEAD records and the index no longer + holds: its `git rm` fails there (`pathspec ... did not match`), so it + prints none and says to commit the removal; committed, the gate passes + in every mode.""" + _, tip = spare_commits(family) + stage_gitlink(holder, TOOLS, tip) + git("commit", "-qm", "A gitlink recorded by hand at tools/x", + cwd=holder) + git("rm", "-q", "--cached", "--", TOOLS, cwd=holder) + + finding = found_in_every_mode(holder, "gitlink-unnamed") + + assert finding == (OUTSIDE_SAID.format(path=TOOLS, commit=tip) + STAGED + + " Commit that removal.") + git("commit", "-qm", "The removal committed", cwd=holder) + assert_passes_in_every_mode(holder, 2) + + +def test_two_sections_for_one_gitlink_outside_members_print_one_git_rm( + family, holder, offline, tmp_path): + """Two sections give tools/x (`gitmodules-path-twice`): the first + section's finding prints the gitlink's `git rm --cached`, and the + second its own lines alone, since that `git rm` typed again fails. + Every line typed as printed, `.gitmodules` staged and committed, the + gate passes in every mode.""" + _, tip = spare_commits(family) + stage_gitlink(holder, TOOLS, tip) + with_section(holder, tmp_path, TOOLS, TOOLS, "/nowhere/x.git") + with_section(holder, tmp_path, "second", TOOLS, "/nowhere/second.git") + + result = validate(holder, "--no-members") + + assert codes(result) == ["gitmodules-path-twice", + "gitmodules-path-unnamed", + "gitmodules-path-unnamed"], result.stderr + first, second = findings(result)[1:] + assert commands(first) == [f"git rm --cached -- {TOOLS}", + *removal(TOOLS, "path", "url")] + assert commands(second) == removal("second", "path", "url") + env = isolated(offline, tmp_path) + for finding in (first, second): + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) + + +# --- the controls ------------------------------------------------------------ + +def test_the_holder_as_made_passes_and_a_plain_clone_checks_it_out( + family, holder, offline, tmp_path): + """CONTROL: a member under `members/` with its section and its row, + two of them, as `init` and `add` made the holder.""" + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member( + holder, fetching(isolated(offline, tmp_path)), family, tmp_path) + + +@pytest.mark.parametrize("recorded", [False, True], + ids=["no-gitlink", "gitlink-removal-staged"]) +def test_a_section_with_no_gitlink_in_the_index_prints_its_lines_alone( + family, holder, offline, tmp_path, recorded): + """CONTROL: a section no row names, at tools/x, where the index holds + no gitlink, prints what it printed before, its `--unset-all` lines + alone, in every mode: there is no gitlink, or one HEAD records whose + removal is staged, where a `git rm` fails. Typed as printed, staged + and committed, the gate passes in every mode.""" + env = isolated(offline, tmp_path) + if recorded: + stage_gitlink(holder, TOOLS, spare_commits(family)[1]) + with_section(holder, tmp_path, TOOLS, TOOLS, "/nowhere/x.git") + if recorded: + git("rm", "-q", "--cached", "--", TOOLS, cwd=holder) + + finding = found_in_every_mode(holder, "gitmodules-path-unnamed") + + assert finding == ( + SECTION_SAID.format(path=TOOLS, name=TOOLS, url="url /nowhere/x.git") + + SECTION_REMEDY + "".join(f"\n {line}" for line in removal( + TOOLS, "path", "url"))) + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) diff --git a/tests/test_validate_family_unnamed_mounts.py b/tests/test_validate_family_unnamed_mounts.py index db0d4ed3..23defc41 100644 --- a/tests/test_validate_family_unnamed_mounts.py +++ b/tests/test_validate_family_unnamed_mounts.py @@ -421,16 +421,19 @@ def test_a_gitlink_is_read_from_the_index_first_and_from_head_second( def test_only_members_is_asked_about(family, holder): """`members/` in any case, and nothing beside it: a gitlink elsewhere - in the holder is not a member's mount, and no row could name it.""" + in the holder is not a member's mount, and no row could name it. One + no section gives is `gitlink-unnamed` instead, after this finding + (#878, `tests/test_family_unnamed_gitlink.py` types it).""" _, tip = spare_commits(family) stage_gitlink(holder, f"tools/{SPARE}", tip) stage_gitlink(holder, f"membersX/{SPARE}", tip) stage_gitlink(holder, f"MEMBERS/{SPARE}", tip) result = validate(holder, "--no-members") assert result.returncode == 1, result.stderr + result.stdout - (gitlink,) = findings(result) - assert gitlink.startswith(GITLINK_SAID.format(path=f"MEMBERS/{SPARE}", - commit=tip)) + assert codes(result) == ["member-gitlink-unnamed", "gitlink-unnamed", + "gitlink-unnamed"], result.stderr + assert findings(result)[0].startswith(GITLINK_SAID.format( + path=f"MEMBERS/{SPARE}", commit=tip)) def test_a_removed_row_is_named_by_its_gitlink_and_its_section(holder): From 445b1e552ecda2e94f7bfaa17a0e7203cc861bc4 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 08:52:10 +0000 Subject: [PATCH 15/20] validate-family.py and bootstrap.py: the holder's remedies, typed as printed in order, put a `.gitmodules` gone from the work tree back once, stage it before a gitlink's `git rm`, and name the checkout `git mv` moved before the update, and bootstrap fails where git's init stopped with every member checked out, so no remedy undoes another or exits 128 and no `family bootstrap ok` follows git's exit 128 (#879) - C1: where the work tree's `.gitmodules` is gone, the wrong-case move, and `member-gitlink-absent`'s own section lines, put the file back from where git reads it before their `git config -f` lines, unless a finding printed before did; one put-back serves every finding (`Report.put_back`). - C2: `member-gitlink-removal-staged` reads "section taken" from the file git reads, puts nothing back after an earlier finding did, and writes a section a plain `git rm` took out back key by key into that file. - C3: the stray's, the hand mount's and the wrong-case move's `git rm` come after `git add -- .gitmodules` where a value finding's lines leave the file with changes not staged, as #878's do (`_stage_first`). - C4: `member-gitlink-removal-staged` names the checkout `git mv` moved, with its `git -C status`, before its update. - C5: `unfetchable` asks `init_stop` with no row left, and `fetch_members` fails on the stop, or on an unmerged `.gitmodules`, through `cures`. - bootstrap.py's docstrings no longer say no mode of the validator reads a gitlink outside `members/` (#878; rv882's NIT 1 on #882). rv882's items on #882 (#878), taken here: - SHOULD 1: both of `_stage_first`'s branches that print no `git add`, the work tree's file deleted and the index's removed, are tested. - SHOULD 2: `gitlink-unnamed` reads the paths `.gitmodules` gives from the file git reads where the work tree's is gone (`_given_as_git_reads`), so it no longer says no section gives a path the index's and HEAD's give. - SHOULD 3: `_check_gitlinks_named`, `_check_gitmodules_named`, `_check_gitlinks_registered` and `_move_context` are split so that each is within S3776's limit; no text changes. - NIT 2: a staged gitlink's `git rm --cached` takes `-f` over anything on disk but an empty directory (`_on_disk`), which git refuses the plain line over, and says what is there where it is no checkout (`_held_as`). - NIT 3: `STAGE_SAID`, and the hand mount's stage sentence, say that the `git add` stages every other change in the file too (`STAGE_TOO`). Tests: tests/test_family_holder_remedies_compose.py; the `STAGE_SAID` constant in tests/test_family_unnamed_gitlink.py. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 97 +- .../family-root/scripts/validate-family.py | 789 +++++++++++----- tests/test_family_holder_remedies_compose.py | 875 ++++++++++++++++++ tests/test_family_unnamed_gitlink.py | 6 +- 4 files changed, 1526 insertions(+), 241 deletions(-) create mode 100644 tests/test_family_holder_remedies_compose.py diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index aec82b91..5f1dccb3 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -30,6 +30,14 @@ no url for, nor where it read nothing from `.gitmodules`, both below, since it fetched nothing then (#839). + Where every row's member is checked out and git's init died all the + same, at a gitlink it has no url for or at a `.gitmodules` a merge + left unmerged, git updated none of them, so the run fails, saying + where git stopped and what cures it, as below, with no advice + (#879). It said `family bootstrap ok` there, beside git's exit 128, + as a run with every member checked out passes (#648), and gave the + advice where git stopped at a gitlink. + "every member and leg fetched" is said only where git exited 0 AND every row's member is checked out (#648). A member left NOT CHECKED OUT where no credential would check it out is said here instead, with @@ -64,8 +72,8 @@ checks the member out whatever the clone chose. For the third and the last, `validate-family.py --no-members` names a path two sections give, or a gitlink in members/ that no row names; `git rm - --cached` takes out one outside members/, which the validator does - not read; and where the gitlink is a row's own, with one section, + --cached` takes out one outside members/, as every mode of the + validator says (#878); and where it is a row's own, with one section, the line says to write its section back. (b) RUNS EACH MEMBER'S OWN `make bootstrap`. Each member is a whole project @@ -83,9 +91,12 @@ (c) EXITS 0 unless a member's own command failed, or (a) left a member not checked out that no credential would check out, or found a line or a - value in `.gitmodules` that git cannot read (#768). Any other member that - is not checked out is REPORTED and skipped, not a failure: it is the - state a missing credential leaves, and it is already reported in (a). + value in `.gitmodules` that git cannot read (#768), or found that, + with every member checked out, git's init died at a gitlink it has + no url for or at an unmerged `.gitmodules` (#879). Any + other member that is not checked out is REPORTED and skipped, not a + failure: it is the state a missing credential leaves, and it is + already reported in (a). Where (a) cannot start git, the run STOPS there, exit 2, refused `git-not-runnable` as `validate-family.py` refuses it (#694): each member's own bootstrap runs git too, so (b) would only say it again. @@ -182,9 +193,11 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: """(a) `git submodule update --init --recursive`, authenticated or not. Where every member is checked out and git died on a value in - `.gitmodules`, or on a work tree file it cannot parse, so it updated - none of them, the one failure line it returns is for `.gitmodules` - (#768). + `.gitmodules`, or on a work tree file it cannot parse, or its init + died at a `.gitmodules` a merge left unmerged, so it updated none of + them, the one failure line it returns is for `.gitmodules` (#768, + #879); where its init died at a gitlink it has no url for, it is for + that gitlink (`updated_none`, #879). Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for @@ -215,17 +228,19 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: if proc.returncode != 0: fetch_incomplete(proc, source, token, credential_advice(bool(left), stopper, modules)) + whole = updated_none(left, stopper, modules) if left: print(" NOT CHECKED OUT, and not for want of a credential: " + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" for _, path, why in left) + ". " + cures(left, stopper, rows, modules), file=sys.stderr) - elif modules.refused or modules.unreadable: - why = REFUSED if modules.refused else UNREADABLE + elif whole is not None: print(" NO MEMBER UPDATED, and not for want of a credential: " - + reason(why, stopper, modules) + ". " - + cures(left, stopper, rows, modules), file=sys.stderr) - return [f"{GITMODULES} (git cannot read it)"] + + reason(whole, stopper, modules) + ". " + + cures(left, stopper, rows, modules, (whole,)), + file=sys.stderr) + return [f"{spelled(stopper)} (git stopped at it)" if whole == STOPPED + else f"{GITMODULES} (git cannot read it)"] elif proc.returncode == 0: print(" every member and leg fetched") return [f"{spelled(row.get('project') or '?')} (not checked out)" @@ -296,9 +311,12 @@ def git_not_started(exc: OSError) -> Refusal | None: def cures(left: list[tuple[dict, str, str]], stopper: str | None, - rows: list[dict], modules: Gitmodules) -> str: + rows: list[dict], modules: Gitmodules, + whole: tuple[str, ...] = ()) -> str: """What (a)'s line says cures the rows `left` names, one sentence for - each kind of cause among them (#648): + each kind of cause among them (#648), or, where it names none, what + cures `whole`, why git updated no member (`updated_none`, #879), one + reason or none: - A `.gitmodules` a merge left unmerged in the index, which git reads nothing from (`Gitmodules.unmerged`): the file resolved and added. @@ -323,13 +341,13 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, `member-uninitialized` for the first row where every row's gitlink is there, as with a stray or a hand mount, and `member-gitlink-absent` where a row's is only in another case, which the first sentence - names. A gitlink outside `members/`, which no mode of the validator - asks about, is taken out of the index: no row names it, and with no - url no clone can check it out. + names. A gitlink outside `members/` is taken out of the index, as + every mode of the validator says (#878): no row names it, and with + no url no clone can check it out. - A row git exited 0 over: the update that checks it out whatever the clone chose. `--init` with the path registers a member a subset clone left out, and `--checkout` overrides an `update = none`.""" - whys = {why for _, _, why in left} + whys = {why for _, _, why in left} | set(whole) said = gitmodules_cures(whys, modules) if whys & {NO_GITLINK, NO_SECTION}: said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " @@ -747,7 +765,10 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool checked out is the missing credential's, and stays the skip it always was. A path two rows give is said once. Where every row's member is checked out and git did not exit 0, `.gitmodules` is read all the - same, for `fetch_members` to ask `refused` of.""" + same, and git's init asked where it stopped, for `fetch_members` to + ask why git updated none (`updated_none`): this returned before + `init_stop` there, so a run whose update git's init stopped said + `family bootstrap ok` (#879).""" left: dict[str, dict] = {} for row in rows: if not (member_path(root, row) / ".git").exists(): @@ -755,8 +776,6 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool if not left and exited_0: return [], None, Gitmodules(Counter(), {}, {}) modules = gitmodules_sections(root) - if not left: - return [], None, modules config = submodule_config(root) stopper = None if exited_0 else init_stop(root, modules, config) found: list[tuple[dict, str, str]] = [] @@ -773,12 +792,9 @@ def row_reason(root: Path, path: str, modules: Gitmodules, """Why `unfetchable` says the member at `path`, not checked out, is none a credential would check out (#648, #768), in the order it asks, or None where it is the missing credential's.""" - if modules.unmerged: - return UNMERGED - if modules.refused: - return REFUSED - if modules.unreadable: - return UNREADABLE + unread = file_reason(modules) + if unread is not None: + return unread if recorded_gitlink(root, path) is None: return NO_GITLINK if not modules.given[path]: @@ -790,6 +806,31 @@ def row_reason(root: Path, path: str, modules: Gitmodules, return None if stopper is None else STOPPED +def file_reason(modules: Gitmodules) -> str | None: + """Why git read nothing from `.gitmodules` (#648, #768), as (a)'s line + says it, or None where it read the file: a merge left it unmerged, it + gives a value git dies on, or it is a work tree file git cannot + parse.""" + if modules.unmerged: + return UNMERGED + if modules.refused: + return REFUSED + return UNREADABLE if modules.unreadable else None + + +def updated_none(left: list[tuple[dict, str, str]], stopper: str | None, + modules: Gitmodules) -> str | None: + """Why git updated no member, where every row's member is checked out + (`left` empty) and git did not exit 0, or None: what `file_reason` + says, or `STOPPED` where git's init died at a gitlink it has no url + for (`stopper`, `init_stop`), before it updated any (#879). None + where it asked for a credential, as a leg's fetch does, and where + `left` names a row, which (a)'s line says why of.""" + if left: + return None + return file_reason(modules) or (None if stopper is None else STOPPED) + + def members_of(root: Path) -> tuple[str, list[dict]]: manifest_path = root / MANIFEST if not manifest_path.is_file(): diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 9e77c3ce..3fc8183e 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -125,6 +125,15 @@ `contracts/shape-pin.yaml`. Editing a copy in place is drift and the exit is to carry it upstream, never to update the digest. +EVERY REMEDY IS TYPED AS PRINTED, IN THE ORDER PRINTED (#879). Where the +work tree's `.gitmodules` is gone from disk, the first remedy that writes +it puts it back first, from where git reads it, and no later one puts it +back over what the lines between wrote; a `git rm` of a gitlink comes +after `git add -- .gitmodules` where lines printed before it leave that +file with changes not staged; and a checkout `git mv` moved to a case +variant of a row's path is named, with its `git -C` status, before the +update that points its store at the row's path. + EXIT CODES 0 the manifest is valid, every member is in lockstep, every copy matches 1 a FINDING: drift a reviewer can act on @@ -261,6 +270,12 @@ def __init__(self) -> None: self.notes: list[str] = [] #: Each finding's code, in step with `findings` (#539). self.codes: list[str] = [] + #: Where the work tree's `.gitmodules` is gone from disk, git's name + #: for the file an earlier finding's line put back there, + #: `:.gitmodules` or `HEAD:.gitmodules`, so that no later finding + #: puts it back over what the lines between wrote (#879); empty + #: until one has. + self.put_back = "" def finding(self, code: str, detail: str) -> None: self.findings.append(f"FINDING {code}: {detail}") @@ -548,16 +563,26 @@ def _gitlink_put_back(root: Path, path: str, commit: str, as `member-gitmodules-absent` writes one (`_section_lines`); and a member that is to go has its row taken out by hand. Either is committed, and a member put back is checked out as - `member-uninitialized` says. + `member-uninitialized` says. Where the work tree's `.gitmodules` is + gone from disk, the section's lines come after the one that puts the + file back from where git reads it (`_file_put_back`): `git config -f` + made a file of that one section, which staged and committed dropped + every other (#879). """ head = _head_gitmodules(root) sections = head[1] if head is not None else {} given = {keys["path"] for keys in sections.values() if "path" in keys} lines = ("" if path in given else _section_lines(path, repository, sections)) + put_back, whose = _file_put_back(root, None) if lines else ("", "") written = (", and its .gitmodules section back by hand, in a POSIX " - "shell, as family.py add writes one, with the lines after it, " - "then stage .gitmodules" if lines else "") + "shell, as family.py add writes one, with the lines after it" + + (f", the first of which puts {GITMODULES}, gone from the " + f"work tree, back from {whose}, so that every other " + "section stays" if put_back else "") + + ", then stage .gitmodules" if lines else "") + if put_back: + lines = f"\n {put_back}{lines}" return ( "Remediation: family.py add refuses the row, which is there " "(member-already-present), and family.py remove cannot take the " @@ -629,7 +654,14 @@ def _gitlink_in_another_case(root: Path, path: str, commit: str, "Remediation: the member is recorded in the wrong case. " + _wrong_case_move(path, commit, variants, _move_context(root, path, variants, repository, - gitlinks))) + gitlinks, None))) + + +#: What the `-f` sentence says is on disk at a gitlink's path (`_held_as`): +#: a checkout, as #839 said, or what is there, which is none (rv882's NIT 2 +#: on #882). +CHECKOUT = "a checkout" +NO_CHECKOUT = "what is on disk there, which is no checkout" class MoveContext(NamedTuple): @@ -639,21 +671,29 @@ class MoveContext(NamedTuple): or under the row's path; and `section`, the lines that make `.gitmodules` give the row's path, with what they do, or empty where a section of HEAD's already gives it. `forced`, the variants whose `git - rm --cached` takes `-f` (`_forced`, #839).""" + rm --cached` takes `-f` (`_forced`, #839), and `forced_over`, what is + on disk at them, as the `-f` sentence says it (`_held_as`, rv882's NIT + 2 on #882). `stage`, whether `git add -- .gitmodules` comes first + (`_stage_first`, #879).""" sparse: tuple[str, ...] = () in_the_way: tuple[str, ...] = () section: str = "" section_said: str = "" forced: tuple[str, ...] = () + stage: bool = False + forced_over: str = CHECKOUT def _move_context(root: Path, path: str, variants: list[str], - repository, gitlinks: dict[str, str]) -> MoveContext: + repository, gitlinks: dict[str, str], + report: Report | None) -> MoveContext: """What the index and HEAD's `.gitmodules` hold around the move of a member recorded only at `variants` to its row's `path` (#762), and which variants the move's `git rm --cached` takes out only with `-f` - (`_forced`, by `gitlinks`, `_member_gitlinks`, #839). + (`_forced`, by `gitlinks`, `_member_gitlinks`, #839). `report` holds + the findings printed before the move, or is None for members mode's + refusal, which is printed alone (#879). `--sparse` (the issue's (f)): a variant a sparse checkout keeps off disk (`git ls-files -t`'s `S`, as `family.py`'s `_skip_worktree` reads @@ -684,7 +724,35 @@ def _move_context(root: Path, path: str, variants: list[str], first variant's where two have one; and where none gives a variant's either, the section is written as `family.py add` writes one (`_section_lines`). Then `git add` stages the file. + + WHERE THE WORK TREE'S `.gitmodules` IS GONE FROM DISK, the section's + lines come after the one that puts it back from where git reads it, + unless a finding printed before the move did (`_file_put_back`): `git + config -f` made a file of that one key, which the move's `git add` + staged, so the commit dropped every other section, and the put-back of + a `member-gitmodules-url-absent` after it restored that staged file + (#879's C1, rv876's P5b and P5e). + + `git add -- .gitmodules` COMES FIRST where a value finding's lines, + printed before it, may leave that file with changes not staged, or it + holds some as it stands (`_stage_first`): git takes no gitlink out of + the index then, and the move's `git rm` exited 128 (#879's C3). """ + sparse, in_the_way = _index_around(root, path, variants) + section, said = _move_section(root, path, variants, repository, report) + edited = report is not None and "gitmodules-value-refused" in report.codes + forced = _forced_variants(root, variants, gitlinks) + over = (CHECKOUT if all(_held_as(root, variant) == CHECKOUT + for variant in forced) else NO_CHECKOUT) + return MoveContext(sparse, in_the_way, section, said, forced, + _stage_first(root, edited), over) + + +def _index_around(root: Path, path: str, variants: list[str] + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + """`(the variants a sparse checkout keeps off disk, every entry the + index holds at or under path but those a merge left unmerged)`, from + `git ls-files -t`, for `_move_context` (#762's (f) and (g)).""" with _git_runs(f"the index entries at and around {path} could not be " "listed"): listed = git_out(["ls-files", "-t", "-z"], cwd=root, binary=True) @@ -696,29 +764,43 @@ def _move_context(root: Path, path: str, variants: list[str], sparse.add(name) if tag != b"M" and (name == path or name.startswith(path + "/")): in_the_way[name] = None + return tuple(sorted(sparse)), tuple(in_the_way) + + +def _move_section(root: Path, path: str, variants: list[str], repository, + report: Report | None) -> tuple[str, str]: + """`(the lines that make .gitmodules give path, what they do)` for the + wrong-case move (`_move_context`, #762's (e)), or two empty strings + where a section of HEAD's gives it already: the section of a variant's + path given the row's, or one written as `family.py add` writes it, + after the line that puts the file back where it is gone from the work + tree and no finding before put it back (`_file_put_back`, #879's C1), + then `git add`.""" head = _head_gitmodules(root) sections = head[1] if head is not None else {} given = {keys["path"]: name for name, keys in sections.items() if "path" in keys} - section = said = "" - if path not in given: - named = next((given[variant] for variant in variants - if variant in given), None) - if named is not None: - section = (f"\n git config -f {GITMODULES} " - f"{shlex.quote(f'submodule.{named}.path')} " - f"{shlex.quote(path)}") - said = (f"give {path} back to the section of HEAD's " - f"{GITMODULES} that gives the variant's path, as git mv " - "leaves it") - else: - section = _section_lines(path, repository, sections) - said = (f"write its section as family.py add writes one, since " - f"no section of HEAD's {GITMODULES} gives {path} or the " - "variant's path") - section += f"\n git add -- {GITMODULES}" - return MoveContext(tuple(sorted(sparse)), tuple(in_the_way), section, - said, _forced_variants(root, variants, gitlinks)) + if path in given: + return "", "" + named = next((given[variant] for variant in variants + if variant in given), None) + if named is not None: + section = (f"\n git config -f {GITMODULES} " + f"{shlex.quote(f'submodule.{named}.path')} " + f"{shlex.quote(path)}") + said = (f"give {path} back to the section of HEAD's {GITMODULES} " + "that gives the variant's path, as git mv leaves it") + else: + section = _section_lines(path, repository, sections) + said = (f"write its section as family.py add writes one, since no " + f"section of HEAD's {GITMODULES} gives {path} or the " + "variant's path") + put_back, whose = _file_put_back(root, report) + if put_back: + section = f"\n {put_back}{section}" + said = (f"put {GITMODULES}, gone from the work tree, back from " + f"{whose}, so that every other section stays, then {said}") + return f"{section}\n git add -- {GITMODULES}", said def _forced_variants(root: Path, variants: list[str], @@ -734,15 +816,18 @@ def _forced(root: Path, path: str, commit: str) -> str | None: `_member_gitlinks` reads at `commit`, out of the index as it is, else what HEAD records at `path`, its commit or `""` for nothing, where that takes `-f` (#839): the index stages the gitlink at a commit HEAD does - not record there, and a checkout is on disk at it, a `.git` entry. Git - refuses it then wherever that checkout has moved on or holds a file it - does not track (`the following file has staged content different from - both the file and the HEAD`, exit 1), measured on git 2.43 after `git - mv members/

Members/

`, staged, and a commit or an untracked file - in the checkout it moved (w761's candidate 2 on #828), and `-f`, with - `--cached`, touches no file. A gitlink HEAD alone records, at a staged - removal or a conflict, is read at HEAD's commit and takes none.""" - if not os.path.lexists(root / path / ".git"): + not record there, and something is on disk at it (`_on_disk`). Git + refuses it then wherever a checkout there has moved on or holds a file + it does not track (`the following file has staged content different + from both the file and the HEAD`, exit 1), measured on git 2.43 after + `git mv members/

Members/

`, staged, and a commit or an untracked + file in the checkout it moved (w761's candidate 2 on #828), and `-f`, + with `--cached`, touches no file. It refuses it too over a directory + that holds files and no `.git`, a file or a symbolic link, and takes it + over an empty directory or nothing (git 2.43, rv882's NIT 2 on #882). + A gitlink HEAD alone records, at a staged removal or a conflict, is + read at HEAD's commit and takes none.""" + if not _on_disk(root, path): return None with _git_runs(f"the gitlink HEAD records at {path} could not be read"): try: @@ -753,6 +838,33 @@ def _forced(root: Path, path: str, commit: str) -> str | None: return None if head == commit else head +def _on_disk(root: Path, path: str) -> bool: + """Whether anything but an empty directory is on disk at `path`, where + a staged gitlink's `git rm --cached` takes `-f` (`_forced`): git 2.43 + refused it, with no `-f`, over a checkout, a directory holding a file + and no `.git` (rv882's NIT 2 on #882), a file and a symbolic link, and + took it over an empty directory and over nothing. A directory this user + cannot list is taken to hold something: `-f`, with `--cached`, touches + no file.""" + where = root / path + if not os.path.lexists(where): + return False + if where.is_symlink() or not where.is_dir(): + return True + try: + with os.scandir(where) as entries: + return any(True for _ in entries) + except OSError: + return True + + +def _held_as(root: Path, path: str) -> str: + """What the `-f` sentence says is on disk at `path` (`_forced_said`): + `CHECKOUT` where a `.git` entry is there, else `NO_CHECKOUT`.""" + return (CHECKOUT if os.path.lexists(root / path / ".git") + else NO_CHECKOUT) + + def _wrong_case_move(path: str, commit: str, variants: list[str], context: MoveContext = MoveContext()) -> str: """The index move for a member recorded only at `variants`, case @@ -789,7 +901,8 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], HEAD does not record there and a checkout is on disk at it (`MoveContext.forced`, `_forced`, #839): git refuses it there wherever that checkout has moved on or holds a file it does not track, and the - text says what `-f` does (`FORCED_SAID`). + text says what `-f` does (`FORCED_SAID`). It comes after `git add -- + .gitmodules` where `MoveContext.stage` (`_stage_first`, #879's C3). """ taken_out = " ".join(shlex.quote(variant) for variant in variants) moved = " and ".join(_guarded_move(variant, path) for variant in variants) @@ -800,12 +913,15 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], for variant in variants) flag = (" --sparse" if context.sparse else "") + ( " -f" if context.forced else "") + lead = (STAGE_SAID if context.stage else "") + "move" + lead = (f"Then {lead}" if context.in_the_way + else f"{lead[0].upper()}{lead[1:]}") return ( _entries_aside(path, context.in_the_way) - + ("Then move" if context.in_the_way else "Move") - + " its gitlink to the row's path, at the commit the row pins, in " - "the index alone, which touches no file, with the " - + ("first two lines below" if context.section + + f"{lead} its gitlink to the row's path, at the commit the row " + "pins, in the index alone, which touches no file, with the " + + ("next two lines" if context.stage + else "first two lines below" if context.section else "two lines below") + _first_line_flags(context) + (f"; the lines after them {context.section_said}, and stage " @@ -838,8 +954,9 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], "checkout git mv moved to a variant names the member's own store, " f"which that update points at {path}, so {status} reads {path} " "after it. The " - + ("lines" if context.section else "two lines") + ":\n" - f" git rm -r --cached{flag} --ignore-unmatch -- {taken_out}\n" + + ("lines" if context.section or context.stage else "two lines") + + ":\n" + (f" {STAGE_LINE}\n" if context.stage else "") + + f" git rm -r --cached{flag} --ignore-unmatch -- {taken_out}\n" " git update-index --add --cacheinfo " f"{shlex.quote(f'160000,{commit},{path}')}{context.section}") @@ -892,7 +1009,8 @@ def _first_line_flags(context: MoveContext) -> str: if context.forced: said.append(f"-f, since the index stages " f"{' and '.join(context.forced)} at a commit HEAD does " - f"not record there, over a checkout: {FORCED_SAID}") + f"not record there, over {context.forced_over}: " + f"{FORCED_SAID}") return f" (the first with {', and with '.join(said)})" if said else "" @@ -1538,6 +1656,8 @@ def _check_gitmodules_values(root: Path, members: list, report.finding("gitmodules-value-refused", _value_taken_out( said, f"submodule.{name}.{key}", back, put_back=put_back, committed=(name, key) in in_head)) + if not put_back: + report.put_back = f"HEAD:{GITMODULES}" put_back = True return bool(in_work), bool(in_work or in_head) @@ -1720,6 +1840,23 @@ def _gitmodules_put_back(root: Path) -> tuple[str, str, str]: return "", "", "" +def _file_put_back(root: Path, report: Report | None) -> tuple[str, str]: + """`(the line that puts the work tree's .gitmodules back, whose file it + puts back)` for a remedy whose lines write that file with `git config + -f` where it is gone from disk (`_gitmodules_put_back`), or two empty + strings: where the work tree holds one, and where a finding printed + before it put it back (`Report.put_back`), so ONE PUT-BACK SERVES EVERY + FINDING and none puts it back over what the lines between wrote + (#879). `report` holds the findings printed before it, and records the + put-back; None for a refusal, printed alone, which always says it.""" + if report is not None and report.put_back: + return "", "" + spec, line, whose = _gitmodules_put_back(root) + if report is not None: + report.put_back = spec + return line, whose + + def _check_gitmodules_paths(sections: dict[str, dict[str, str]], report: Report) -> None: """No two sections of `.gitmodules` give ONE path (#300). @@ -2034,63 +2171,98 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], typed again failed: it says to commit the removal instead and prints no `git rm` (#681). A hand mount's keeps the branch and the `add`. + EACH `git rm` COMES AFTER `git add -- .gitmodules` where a value + finding's lines, printed before it, may leave that file with changes + not staged, or it holds some as it stands (`_stage_first`): git takes + no gitlink out of the index then, `--cached` and `-f` or not, and the + stray's, the hand mount's and the move's `git rm` each exited 128 + (#879's C3). The move puts back a `.gitmodules` gone from the work + tree before it writes one, unless a finding before it did + (`_move_context`, #879's C1). + A gitlink outside `members/` is no member's mount, and no row can name it: `_check_gitlinks_elsewhere` and `_check_gitmodules_named` ask about it (#878). + + The gitlinks in `members/` no row names are listed once, before the + loop, and each one's remedy is `_variant_remedy`'s or + `_mounted_by_hand`'s, so this stays within S3776's limit (rv882's + SHOULD 3 on #882). """ - holds = set(gitlinks) - head = _head_gitmodules(root) if set(gitlinks) - set(named) else None + unnamed = [path for path in sorted(gitlinks) + if path not in named and _in_members(path)] + if not unnamed: + return + head = _head_gitmodules(root) fetched = ({keys["path"] for keys in head[1].values() if "path" in keys and "url" in keys} if head is not None else set()) - for path, commit in sorted(gitlinks.items()): - if path in named or not _in_members(path): - continue - said = (f"{path} is a gitlink at {commit} that no row of members: " - + ("names, so a recursive clone of this holder fetches it " - "and nothing here pins, digests or identifies it." - if path in fetched else - f"names and no section of HEAD's {GITMODULES} gives a " - "url, so a recursive clone of this holder checks out " - "nothing at it, in a plain clone git submodule update " - "--init stops there (No url found for submodule path, " - "exit 128) before it checks out any member, and nothing " - "here pins, digests or identifies it.")) + stage = head is not None and _stage_first( + root, "gitmodules-value-refused" in report.codes) + for path in unnamed: + said = _gitlink_unnamed_said(path, gitlinks[path], path in fetched) variant = _case_variant(path, named) if variant is not None: - row_path, row = variant - kind = _variant_case(path, row_path, row, holds) - if kind == WRONG_CASE and row_path == row.canonical and row.pin: - variants = sorted(other for other in holds - if other != row_path - and other.casefold() == row_path.casefold()) - report.finding( - "member-gitlink-unnamed", - f"{said}{_variant_said(row_path, row)} The member is " - "recorded in the wrong case. " - + _wrong_case_move(row_path, row.pin, variants, - _move_context(root, row_path, - variants, - row.repository, - gitlinks))) - continue - if kind != STRAY: - report.finding( - "member-gitlink-unnamed", - said + _variant_advice(kind, path, row_path, row, - "gitlink")) - continue report.finding( "member-gitlink-unnamed", - f"{said}{_variant_said(row_path, row)} " - + _second_path_out(root, path, commit, path in removed)) + said + _variant_remedy(root, path, variant, gitlinks, + path in removed, stage, report)) continue - parts = path.split("/") - project = parts[1] if len(parts) == 2 and parts[1] else None report.finding( "member-gitlink-unnamed", - f"{said} " + _mounted_by_hand(root, path, project, - path in removed)) + f"{said} " + _mounted_by_hand(root, path, _hand_project(path), + path in removed, stage)) + + +def _gitlink_unnamed_said(path: str, commit: str, fetched: bool) -> str: + """What `member-gitlink-unnamed` says first of the gitlink at `path`, + at `commit`, by whether a section of HEAD's `.gitmodules` gives it a + url (`fetched`): a recursive clone fetches it, or no clone checks it + out and a plain clone's update stops there (#762).""" + return (f"{path} is a gitlink at {commit} that no row of members: " + + ("names, so a recursive clone of this holder fetches it and " + "nothing here pins, digests or identifies it." if fetched else + f"names and no section of HEAD's {GITMODULES} gives a url, so " + "a recursive clone of this holder checks out nothing at it, " + "in a plain clone git submodule update --init stops there (No " + "url found for submodule path, exit 128) before it checks out " + "any member, and nothing here pins, digests or identifies " + "it.")) + + +def _hand_project(path: str) -> str | None: + """The project a hand mount at `members/

` would be mounted as, `

`, + or None for a path deeper or shallower than that.""" + parts = path.split("/") + return parts[1] if len(parts) == 2 and parts[1] else None + + +def _variant_remedy(root: Path, path: str, variant: tuple[str, RowPath], + gitlinks: dict[str, str], staged: bool, stage: bool, + report: Report) -> str: + """What `member-gitlink-unnamed` says, after what the gitlink at `path` + is, of one a row names in another case (`variant`, `_case_variant`): + the wrong-case move where it is the only record of the member + (`_wrong_case_move`), the second path's `git rm --cached` where it is + a STRAY (`_second_path_out`, after `git add -- .gitmodules` where + `stage`, and the commit alone where its removal is `staged`), and + `_variant_advice`'s words for the others, with no command.""" + row_path, row = variant + kind = _variant_case(path, row_path, row, set(gitlinks)) + if kind == WRONG_CASE and row_path == row.canonical and row.pin: + variants = sorted(other for other in gitlinks + if other != row_path + and other.casefold() == row_path.casefold()) + return (f"{_variant_said(row_path, row)} The member is recorded in " + "the wrong case. " + + _wrong_case_move(row_path, row.pin, variants, + _move_context(root, row_path, variants, + row.repository, gitlinks, + report))) + if kind != STRAY: + return _variant_advice(kind, path, row_path, row, "gitlink") + return (f"{_variant_said(row_path, row)} " + + _second_path_out(root, path, gitlinks[path], staged, stage)) #: What a gitlink whose removal is staged and not committed says in place of @@ -2115,8 +2287,9 @@ def _second_path_out(root: Path, path: str, commit: str, (w761's candidate 2 on #828, measured on the gate's state). `gitlink-unnamed` says the same of a gitlink outside `members/` that no - section gives (#878), after `git add -- .gitmodules` where - `stage_first` (`_stage_first`).""" + section gives (#878). Both come after `git add -- .gitmodules` where + `stage_first` (`_stage_first`): after a value finding's lines the + stray's plain `git rm --cached` exited 128 (#879's C3).""" if staged: return f"{STAGED_REMOVAL} Commit that removal." head = _forced(root, path, commit) @@ -2124,7 +2297,7 @@ def _second_path_out(root: Path, path: str, commit: str, else "Take") + " it out of the index alone, which touches no file" + _forced_said(head, "the second line" if stage_first - else "the line") + else "the line", _held_as(root, path)) + ":\n " + (f"{STAGE_LINE}\n " if stage_first else "") + _cached_removal(path, head)) @@ -2133,11 +2306,16 @@ def _second_path_out(root: Path, path: str, commit: str, #: after where `.gitmodules` may hold changes not staged (`_stage_first`, #: #878), and what the text says of it, before what it says of the `git rm`. STAGE_LINE = f"git add -- {GITMODULES}" -STAGE_SAID = (f"stage {GITMODULES} with the first line, since git rm refuses " - "to take any gitlink out of the index, -f or not, while " - f"{GITMODULES} holds changes not staged (please stage your " - f"changes to {GITMODULES} or stash them to proceed, exit 128), " - "as lines printed above may leave it or as it stands, then ") +STAGE_WHY = ("since git rm refuses to take any gitlink out of the index, -f " + f"or not, while {GITMODULES} holds changes not staged (please " + f"stage your changes to {GITMODULES} or stash them to proceed, " + "exit 128), as lines printed above may leave it or as it stands") +#: Said of `git add -- .gitmodules` beside why: it stages every other +#: change in the file too, which the commit after it then holds (rv882's +#: NIT 3 on #882). +STAGE_TOO = "which stages every other change in it too" +STAGE_SAID = (f"stage {GITMODULES} with the first line, {STAGE_TOO}, " + f"{STAGE_WHY}, then ") def _stage_first(root: Path, edited: bool) -> bool: @@ -2176,19 +2354,21 @@ def _stage_first(root: Path, edited: bool) -> bool: return git_out(["hash-object", "--", GITMODULES], cwd=root) != staged -def _forced_said(head: str | None, line: str) -> str: +def _forced_said(head: str | None, line: str, + held: str = CHECKOUT) -> str: """What the text says after "take it out of the index alone, which touches no file" of the `git rm --cached` it prints as `line`, where `_forced` answered `head` (#839): that the removal is committed, and - where the line takes `-f`, why, and what `-f` does (`FORCED_SAID`), - and that nothing is left to commit where HEAD records nothing.""" + where the line takes `-f`, why, over what is on disk there (`held`, + `_held_as`), and what `-f` does (`FORCED_SAID`), and that nothing is + left to commit where HEAD records nothing.""" if head is None: return ", and commit that" return ((", and commit that. The index stages it at a commit HEAD does " "not record there" if head else ": HEAD records nothing there, so nothing of it is left to " "commit. The index stages it") - + f" over a checkout, so {line} takes -f: {FORCED_SAID}") + + f" over {held}, so {line} takes -f: {FORCED_SAID}") def _cached_removal(path: str, head: str | None) -> str: @@ -2231,13 +2411,18 @@ def _check_gitlinks_elsewhere(root: Path, gitlinks: dict[str, str], `git add -- .gitmodules` COMES FIRST where a `gitmodules-value-refused` finding, printed before this one, may leave the file with changes not staged, or it holds some as it stands (`_stage_first`): git takes no - gitlink out of the index then, and the `git rm` exited 128.""" + gitlink out of the index then, and the `git rm` exited 128. + + WHERE THE WORK TREE'S `.gitmodules` IS GONE FROM DISK, the paths are + read from the file git reads in its place (`_given_as_git_reads`): + read from the work tree, none was given, and this said "no section of + .gitmodules gives" of a gitlink the index's and HEAD's gave, whose + plain clone checked it out (rv882's SHOULD 2 on #882).""" unnamed = [path for path in sorted(gitlinks) if path not in named and not _in_members(path)] if not unnamed: return - given = {keys["path"] for keys in _gitmodules_sections(root).values() - if "path" in keys} + given = _given_as_git_reads(root) stage_first = _stage_first(root, "gitmodules-value-refused" in report.codes) for path in unnamed: @@ -2256,14 +2441,29 @@ def _check_gitlinks_elsewhere(root: Path, gitlinks: dict[str, str], stage_first)) +def _given_as_git_reads(root: Path) -> set[str]: + """Every path a section of `.gitmodules` gives, as git's submodule + commands read the file: the work tree's where it is on disk, else the + one git reads in its place (`_gitmodules_put_back`, the index's, then + HEAD's), else none (rv882's SHOULD 2 on #882).""" + if os.path.lexists(root / GITMODULES): + return {keys["path"] for keys in _gitmodules_sections(root).values() + if "path" in keys} + spec = _gitmodules_put_back(root)[0] + return _blob_paths(root, spec) if spec else set() + + def _mounted_by_hand(root: Path, path: str, project: str | None, - staged: bool) -> str: + staged: bool, stage_first: bool = False) -> str: """What a gitlink no row names, in no case, says to do after what it is: put its row back, or take a hand mount out, commit that, and mount it with `family.py add`, on a branch (`_check_gitlinks_named`). Where its removal is staged already (#681), the same with no `git rm`, and the checkout `git rm --cached` leaves at the path, which `family.py add` - refuses (`member-path-taken`), deleted by hand first (rv681's S1).""" + refuses (`member-path-taken`), deleted by hand first (rv681's S1). + Its `git rm` comes after `git add -- .gitmodules` where `stage_first` + (`_stage_first`): after a value finding's lines it exited 128 (#879's + C3).""" member = (PROJECT_PLACEHOLDER if project is None else shlex.quote(project)) add = (f" {PYTHON} scripts/family.py add --family-root " @@ -2284,20 +2484,25 @@ def _mounted_by_hand(root: Path, path: str, project: str | None, "one commit, and names the object store git rm leaves under " f".git/modules if one is still there. {BRANCH_LEFT}\n" f" {_switch_line(root, 'add', project)}\n{add}") + staging, rm_at, add_at = ((f"stage {GITMODULES} with the second, " + f"{STAGE_TOO}, {STAGE_WHY}, ", "third", + "fourth") + if stage_first else ("", "second", "third")) return ( f"{RESTORE_ROW} If it was mounted by hand, make a branch " "with the first command below, since family.py add refuses the " - "holder's tracking branch, take it out with the second, which " - f"refuses while its checkout holds changes and takes its " - f"{GITMODULES} section out too, and commit that removal, since " + f"holder's tracking branch, {staging}take it out with the " + f"{rm_at}, which refuses while its checkout holds changes and takes " + f"its {GITMODULES} section out too, and commit that removal, since " "family.py add refuses a path under another spelling of members/ " "that HEAD holds and the index no longer does, then mount it " - "with the third, from a checkout of openRepoShape: family.py add " - "writes its row and its gitlink in one commit, and names the " + f"with the {add_at}, from a checkout of openRepoShape: family.py " + "add writes its row and its gitlink in one commit, and names the " "object store git rm leaves under .git/modules if one is still " f"there. {BRANCH_LEFT}\n" f" {_switch_line(root, 'add', project)}\n" - f" git rm -- {shlex.quote(path)}\n{add}") + + (f" {STAGE_LINE}\n" if stage_first else "") + + f" git rm -- {shlex.quote(path)}\n{add}") def _check_gitmodules_named(root: Path, sections: dict[str, dict[str, str]], @@ -2342,6 +2547,11 @@ def _check_gitmodules_named(root: Path, sections: dict[str, dict[str, str]], After a finding whose lines write `.gitmodules`, a value's or an earlier section's (`edited`), the `git rm` comes after `git add -- .gitmodules`, which it needs (`_stage_first`). + + What a section in another case says is `_variant_advice`'s, and the + rest of each remedy is `_section_out`'s, after `_gitlink_out_first`'s + `git rm` where it prints one, so this stays within S3776's limit + (rv882's SHOULD 3 on #882). """ holds = {keys["path"] for keys in sections.values() if "path" in keys} given = Counter(keys["path"] for keys in sections.values() @@ -2352,61 +2562,71 @@ def _check_gitmodules_named(root: Path, sections: dict[str, dict[str, str]], path = keys.get("path") if path is None or path in named: continue - url = f"url {keys['url']}" if "url" in keys else "no url" - said = (f'{GITMODULES} gives the path {path} to the submodule named ' - f'"{name}" ({url}), and no row of members: names that path, ' - "so nothing pins what it registers, and the next family.py " - "add of its name or its path meets it.") + said = _section_unnamed_said(name, keys) variant = _case_variant(path, named) - lead = "Take" if variant is not None: - row_path, row = variant - kind = _variant_case(path, row_path, row, holds) + kind = _variant_case(path, variant[0], variant[1], holds) if kind != STRAY: - report.finding( - "gitmodules-path-unnamed", - said + _variant_advice(kind, path, row_path, row, - "section")) + report.finding("gitmodules-path-unnamed", + said + _variant_advice(kind, path, *variant, + "section")) continue - said += _variant_said(row_path, row) + said += _variant_said(*variant) + _section_out( + name, keys, "Take", f"then commit {GITMODULES}") + elif (path in gitlinks and path not in removed and given[path] == 1 + and _in_members(path)): + report.finding( + "gitmodules-path-unnamed", + f"{said}{SECTION_ROW_BACK} Otherwise the git rm printed above " + f"for the gitlink at {path} takes this section out too, so no " + "command is printed here.") + continue else: - said += (" If its row was deleted or its path: edited by hand, " - "put the row back as git last had it.") - lead = "Otherwise take" - if (path in gitlinks and path not in removed and given[path] == 1 - and _in_members(path)): - report.finding( - "gitmodules-path-unnamed", - f"{said} Otherwise the git rm printed above for the " - f"gitlink at {path} takes this section out too, so no " - "command is printed here.") - continue - first = (_gitlink_out_first(root, name, path, gitlinks, removed, - taken, edited) - if variant is None else None) - then, line = f"then commit {GITMODULES}", "" - if first is not None: - first_said, line, then = first - said += first_said - lead = "Then, with the lines after it, take" - removal = "".join( - f"\n git config -f {GITMODULES} --unset-all " - f"{shlex.quote(f'submodule.{name}.{key}')}" for key in keys) - report.finding( - "gitmodules-path-unnamed", - f"{said} {lead} the section out by hand, in a POSIX shell, every " - "key it holds, so a submodule later given its name inherits none " - f"of them, {then}:{line}{removal}") + said += SECTION_ROW_BACK + _gitlink_out_first( + root, name, keys, gitlinks, removed, taken, edited) + report.finding("gitmodules-path-unnamed", said) edited = True -def _gitlink_out_first(root: Path, name: str, path: str, +#: What `gitmodules-path-unnamed` says of a section no row names, in no +#: case, before what it says to do. +SECTION_ROW_BACK = (" If its row was deleted or its path: edited by hand, put " + "the row back as git last had it.") + + +def _section_unnamed_said(name: str, keys: dict[str, str]) -> str: + """What `gitmodules-path-unnamed` says first of the section `name`, + which gives `keys["path"]` and no row names (#361).""" + url = f"url {keys['url']}" if "url" in keys else "no url" + return (f'{GITMODULES} gives the path {keys["path"]} to the submodule ' + f'named "{name}" ({url}), and no row of members: names that ' + "path, so nothing pins what it registers, and the next " + "family.py add of its name or its path meets it.") + + +def _section_out(name: str, keys: dict[str, str], lead: str, then: str, + line: str = "") -> str: + """The end of `gitmodules-path-unnamed`'s text, which takes the section + `name` out by key (#300), every key it holds (`keys`), after `lead`, + ending `then`, with `line`, a `git rm` printed first, before those + lines.""" + removal = "".join( + f"\n git config -f {GITMODULES} --unset-all " + f"{shlex.quote(f'submodule.{name}.{key}')}" for key in keys) + return (f" {lead} the section out by hand, in a POSIX shell, every key " + "it holds, so a submodule later given its name inherits none of " + f"them, {then}:{line}{removal}") + + +def _gitlink_out_first(root: Path, name: str, keys: dict[str, str], gitlinks: dict[str, str], removed: frozenset[str], - taken: set[str], - edited: bool) -> tuple[str, str, str] | None: - """What `gitmodules-path-unnamed` adds for the section `name`, whose - `path` is a gitlink outside `members/` (#878), `(what it says, the - line it prints first, how it ends)`, or None where it adds nothing. + taken: set[str], edited: bool) -> str: + """The rest of `gitmodules-path-unnamed`'s text for the section `name`, + no row naming its path in any case, after what it says of the row: + where that path (`keys["path"]`) is a gitlink outside `members/` + (#878), the `git rm --cached` that takes the gitlink out first, then + the section's own lines (`_section_out`), and elsewhere those lines + alone, after "Otherwise". THE GITLINK GOES FIRST, out of the index alone, with `git rm --cached`, which touches no file and leaves the section, and the commit, as @@ -2428,15 +2648,17 @@ def _gitlink_out_first(root: Path, name: str, path: str, or where it holds some as it stands (`_stage_first`): git takes no gitlink out of the index then, and the `git rm` exited 128. - None for a path in `members/`, whose gitlink has a finding of its own - (`member-gitlink-unnamed`), for one that is no gitlink, and one + No `git rm` for a path in `members/`, whose gitlink has a finding of + its own (`member-gitlink-unnamed`), for one that is no gitlink, and one whose removal is staged (`removed`), where a `git rm` fails (`pathspec ... did not match`), and for one an earlier section's finding took out - (`taken`): the `git rm` typed again fails there too. A path it adds to - is added to `taken`.""" + (`taken`): the `git rm` typed again fails there too. A path it prints + one for is added to `taken`.""" + path = keys["path"] if (path not in gitlinks or path in removed or _in_members(path) or path in taken): - return None + return _section_out(name, keys, "Otherwise take", + f"then commit {GITMODULES}") taken.add(path) head = _forced(root, path, gitlinks[path]) in_head = _head_gitmodules(root) @@ -2451,9 +2673,11 @@ def _gitlink_out_first(root: Path, name: str, path: str, "(No url found for submodule path, exit 128) before it checks out " "any member: " + (STAGE_SAID if stage else "") + f"take it out of the index alone, which touches no file, with " - f"{nth}" + _forced_said(head, nth) + ".", - (f"\n {STAGE_LINE}" if stage else "") - + f"\n {_cached_removal(path, head)}", then) + f"{nth}" + + _forced_said(head, nth, _held_as(root, path)) + "." + + _section_out(name, keys, "Then, with the lines after it, take", + then, (f"\n {STAGE_LINE}" if stage else "") + + f"\n {_cached_removal(path, head)}")) def _section_lines(path: str, repository, @@ -2524,7 +2748,10 @@ def _check_gitlinks_registered(root: Path, members: list, `member-gitmodules-absent`'s lines, staged and committed, dropped every other section (rv820's review, comment 6237354577 on #839), and a second `member-gitmodules-url-absent`'s put the file back over the url - the first had written (#839). + the first had written (#839). Nor where the wrong-case move printed + before them put it back (`Report.put_back`, #879's C1), whose write the + url-absent finding's put-back undid; the one these print is recorded + there, for `member-gitlink-removal-staged` after them (#879's C2). """ head = _head_gitmodules(root) if head is None: @@ -2532,26 +2759,25 @@ def _check_gitlinks_registered(root: Path, members: list, held, sections = head given = Counter(keys["path"] for keys in sections.values() if "path" in keys) - back = ("", "") if refused else _gitmodules_put_back(root)[1:] + spec, line, whose = (("", "", "") if refused or report.put_back + else _gitmodules_put_back(root)) + back = (line, whose) unregistered: set[str] = set() - for row in members: - project = row.get("project") if isinstance(row, dict) else None - if not isinstance(project, str) or not project: - continue - path = f"{DEFAULT_MEMBERS_DIR}/{project}" - if (str(row.get("path") or path) != path or path in unregistered - or path not in gitlinks): + for row, _, path in _rows_at_their_paths(members): + if path not in gitlinks: continue if path in given: if given[path] == 1 and _url_absent(root, path, gitlinks[path], row, sections, report, back): unregistered.add(path) + report.put_back = report.put_back or spec back = ("", "") continue unregistered.add(path) report.finding("member-gitmodules-absent", _absent_said( path, gitlinks[path], held, _section_lines(path, row.get("repository"), sections), back)) + report.put_back = report.put_back or spec back = ("", "") return unregistered @@ -2654,63 +2880,161 @@ def _check_removals_staged(root: Path, members: list, if not removed: return set() head = _head_gitmodules(root) - in_head = ({keys["path"] for keys in head[1].values() if "path" in keys} - if head is not None else set()) - in_worktree = {keys["path"] for keys in worktree.values() - if "path" in keys} + sections = head[1] if head is not None else {} staged: set[str] = set() + for row, project, path in _rows_at_their_paths(members): + if path not in removed: + continue + staged.add(path) + lines = _put_back_lines(root, path, _section_back( + root, path, sections, worktree, report)) + report.finding("member-gitlink-removal-staged", _removal_staged_said( + root, row, project, path, gitlinks[path], lines, + _moved_checkouts(root, path, gitlinks))) + return staged + + +def _rows_at_their_paths(members: list) -> Iterator[tuple[dict, str, str]]: + """`(row, project, path)` for each row of `members:` whose project is a + name and whose `path:` is its own, `members/`, or none, once + for each path: the rows `_check_gitlinks_registered` and + `_check_removals_staged` ask, which `_check_member` reads at that path. + A row whose `path:` is another is `member-path`'s (#611, #762).""" + seen: set[str] = set() for row in members: project = row.get("project") if isinstance(row, dict) else None if not isinstance(project, str) or not project: continue path = f"{DEFAULT_MEMBERS_DIR}/{project}" - if (str(row.get("path") or path) != path or path in staged - or path not in removed): - continue - staged.add(path) - lines = _put_back_lines(root, path, path in in_head - and path not in in_worktree) - report.finding("member-gitlink-removal-staged", _removal_staged_said( - root, row, project, path, gitlinks[path], lines)) - return staged + if str(row.get("path") or path) == path and path not in seen: + seen.add(path) + yield row, project, path + + +def _moved_checkouts(root: Path, path: str, + gitlinks: dict[str, str]) -> list[str]: + """Each case variant of `path` the index holds a gitlink at and a `.git` + entry is on disk at, where none is at `path`: the checkout `git mv` + moved, which `member-gitlink-removal-staged` names (`_moved_said`, + #879's C4); none where a checkout is at `path`.""" + if (root / path / ".git").exists(): + return [] + return [variant for variant in sorted(gitlinks) + if variant != path and variant.casefold() == path.casefold() + and os.path.lexists(root / variant / ".git")] + + +#: What `member-gitlink-removal-staged` says its `git checkout HEAD -- +#: .gitmodules` puts back (#762). +SECTION_TAKEN = (f"the {GITMODULES} section git rm took out, putting that " + "file back as HEAD has it and dropping any other change to " + "it") + + +def _section_back(root: Path, path: str, + sections: dict[str, dict[str, str]], + worktree: dict[str, dict[str, str]], + report: Report) -> list[tuple[list[str], str]]: + """The lines `member-gitlink-removal-staged` prints for `.gitmodules`, + with what they put back, where HEAD's (`sections`) gives `path` (#762): + `git checkout HEAD -- .gitmodules` where a plain `git rm` took the + section out of the work tree's (`worktree`), and nothing where it is + there. + + WHERE THE WORK TREE'S FILE IS GONE FROM DISK it is read as git reads it + (#879's C2, rv876's P5d): from the work tree, empty, the section was + always said to be taken, and the `git checkout HEAD` after a finding + that had put the file back and written a section there took that + section out again. So where a finding printed before this one put the + file back (`Report.put_back`), nothing is put back again: where that + file gives the path, nothing at all, and where it does not, the section + HEAD's gives is written into it, key by key, as HEAD has it, since + `git submodule update --init` dies `No url found` without it. Where + none did, this one puts it back, and records that: from HEAD where the + file git reads, the index's, lacks the section (`SECTION_TAKEN`), else + from where git reads it (`_gitmodules_put_back`), so that the line + below keeps the member as HEAD has it and a deletion of `.gitmodules` + staged beside its removal is undone too, with nothing left to + commit.""" + given = {keys["path"] for keys in sections.values() if "path" in keys} + if path not in given: + return [] + if os.path.lexists(root / GITMODULES): + return ([] if any(keys.get("path") == path + for keys in worktree.values()) + else [([f"git checkout HEAD -- {GITMODULES}"], + SECTION_TAKEN)]) + if report.put_back: + if path in _blob_paths(root, report.put_back): + return [] + name = [name for name, keys in sections.items() + if keys.get("path") == path][-1] + return [([f"git config -f {GITMODULES} " + f"{shlex.quote(f'submodule.{name}.{key}')} " + f"{shlex.quote(value)}" + for key, value in sections[name].items()], + f"the {GITMODULES} section git rm took out, key by key as " + "HEAD has it, in the file a line above put back")] + spec, line, whose = _gitmodules_put_back(root) + if path in _blob_paths(root, spec): + report.put_back = spec + return [([line], f"the {GITMODULES} gone from the work tree, from " + f"{whose}")] + report.put_back = f"HEAD:{GITMODULES}" + return [([f"git checkout HEAD -- {GITMODULES}"], SECTION_TAKEN)] + + +def _blob_paths(root: Path, spec: str) -> set[str]: + """Every path a section of the `.gitmodules` git names `spec` gives + (`:.gitmodules`, the index's, or `HEAD:.gitmodules`), read as + `_gitmodules_sections` reads the work tree's; none where git cannot + read it (#879).""" + with _git_runs(f"{spec} could not be read as git reads it"): + try: + listed = git_out(["config", "-z", "--blob", spec, "--list"], + cwd=root, binary=True) + except Refusal: + return set() + return {keys["path"] for keys in _sections_listed(listed).values() + if "path" in keys} def _put_back_lines(root: Path, path: str, - section_taken: bool) -> list[tuple[str, str]]: + section: list[tuple[list[str], str]] + ) -> list[tuple[list[str], str]]: """The lines `member-gitlink-removal-staged` prints that keep the member - at `path`, each with what it puts back (#762): its gitlink; the section - a plain `git rm` took out of the work tree's `.gitmodules`, where - `section_taken` (HEAD's file gives the path and the work tree's does - not); and its checkout, where none is there.""" - lines = [(f"git reset -q -- {shlex.quote(path)}", "its gitlink")] - if section_taken: - lines.append((f"git checkout HEAD -- {GITMODULES}", - f"the {GITMODULES} section git rm took out, " - "putting that file back as HEAD has it and " - "dropping any other change to it")) + at `path`, each group with what it puts back (#762): its gitlink; + `section`, the lines for `.gitmodules` (`_section_back`, #879); and + its checkout, where none is there.""" + lines = [([f"git reset -q -- {shlex.quote(path)}"], "its gitlink"), + *section] if not (root / path / ".git").exists(): - lines.append(("git submodule update --init --recursive -- " - f"{shlex.quote(path)}", "its checkout")) + lines.append((["git submodule update --init --recursive -- " + f"{shlex.quote(path)}"], "its checkout")) return lines def _removal_staged_said(root: Path, row: dict, project: str, path: str, - commit: str, lines: list[tuple[str, str]]) -> str: + commit: str, lines: list[tuple[list[str], str]], + moved: list[str]) -> str: """`member-gitlink-removal-staged`'s text for the row naming `project`, whose gitlink at `path` is at `commit` in HEAD, with `lines` - (`_put_back_lines`) below it (#762).""" + (`_put_back_lines`) below it (#762), and what it says of a checkout of + the member that `git mv` moved to a case variant of `path`, each of + `moved` (`_moved_said`, #879's C4).""" member_id = str(row.get("id") or project) remove = (f"{PYTHON} scripts/family.py remove --family-root " f"{shlex.quote(str(root))} --member {shlex.quote(project)}") - commands = "\n ".join(line for line, _ in lines) + commands = "\n ".join(line for group, _ in lines for line in group) return ( f"{path} is a gitlink at {commit} that a row of members: names and " "HEAD records, and the index holds nothing at that path: its removal " "is staged and not committed, so the next commit un-registers the " "member under its row, and this gate then refuses " "member-gitlink-absent. If the member is to stay, " - + _put_back_said([said for _, said in lines]) - + ", and nothing is left to commit. If it is to go, put it back the " + + _put_back_said(lines) + + ", and nothing is left to commit." + _moved_said(path, moved) + + " If it is to go, put it back the " "same way, then take it out with family.py remove, from a checkout " "of openRepoShape, which takes its row, its gitlink and its section " "out in one commit, on a branch other than the holder's tracking " @@ -2719,15 +3043,58 @@ def _removal_staged_said(root: Path, row: dict, project: str, path: str, f"`{remove}`. {BRANCH_LEFT}\n {commands}") -def _put_back_said(put: list[str]) -> str: - """What the lines `member-gitlink-removal-staged` prints put back, one - to a line, in their order: the gitlink first (#762).""" - if len(put) == 1: - return f"the line below puts {put[0]} back as HEAD records it" - ordinals = ["the first", "the second", "the third"] +def _moved_said(path: str, moved: list[str]) -> str: + """What `member-gitlink-removal-staged` says, before its last line, the + update, of each checkout of the member that `git mv` moved to a case + variant of `path`, or nothing (#879's C4): after the lines were typed + as printed it stayed there, untracked and named nowhere, so `family.py + add` refused it (`family-members-dir-variant`), and after the update + `git -C status` read `path`, so a file only it held went + unseen. Said as `family.py add`'s closing sentences say it (#828, + #865).""" + if not moved: + return "" + there = " or ".join(moved) + status = " or ".join(f"`git -C {shlex.quote(variant)} status`" + for variant in moved) + return (f" The checkout git mv moved to {there} stays there, untracked, " + "and family.py add refuses it as family-members-dir-variant: its " + ".git file names the member's store under this holder's " + ".git/modules/ by a path relative to it, and its commits are kept " + "there whatever becomes of the checkout, so only what is not " + f"committed is in the checkout alone, which {status} shows. " + "Delete it by hand once nothing only it holds is needed, before " + f"the last line: that update points the store at {path}, after " + f"which {status} reads {path} instead.") + + +#: How `_put_back_said` counts the lines below a finding. +ORDINALS = ("first", "second", "third", "fourth", "fifth", "sixth", + "seventh", "eighth", "ninth", "tenth") + + +def _put_back_said(groups: list[tuple[list[str], str]]) -> str: + """What the lines `member-gitlink-removal-staged` prints put back, in + their order, the gitlink first (#762): each group of lines + (`_put_back_lines`) with what it puts back, `the second` for one line + and `the second to the fourth` for more (#879).""" + if len(groups) == 1 and len(groups[0][0]) == 1: + return f"the line below puts {groups[0][1]} back as HEAD records it" + said, at = [], 0 + for group, what in groups: + span = f"the {_ordinal(at)}" + if len(group) > 1: + span += f" to the {_ordinal(at + len(group) - 1)}" + said.append(f"{span} {what}") + at += len(group) return ("the lines below put the member back as HEAD has it: " - + ", ".join(f"{ordinals[at]} {said}" - for at, said in enumerate(put))) + + ", ".join(said)) + + +def _ordinal(at: int) -> str: + """`first` for the line at 0, and so on (`ORDINALS`); `th` past the + tenth (#879).""" + return ORDINALS[at] if at < len(ORDINALS) else f"{at + 1}th" def _init_stop_said(root: Path, sections: dict[str, dict[str, str]], diff --git a/tests/test_family_holder_remedies_compose.py b/tests/test_family_holder_remedies_compose.py new file mode 100644 index 00000000..9597d89b --- /dev/null +++ b/tests/test_family_holder_remedies_compose.py @@ -0,0 +1,875 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The family holder's pinned `scripts/validate-family.py` and +`scripts/bootstrap.py`: remedies that undid each other or refused when +typed as printed, a checkout `git mv` moved that nothing named, and `ok` +beside git's exit 128 (#879). + +THE ISSUE'S FIVE LINES, each reproduced on #876's head (a7f6c569) before +this change, and the states beside them found with it: + +- C1. Where the work tree's `.gitmodules` is gone, the wrong-case move's + `git config -f .gitmodules submodule..path ` made a file + of that one key, which its `git add` staged: committed, HEAD lost every + other section (rv876's P5e), and under `--no-members` a later + `member-gitmodules-url-absent` put that staged file back over its own + write (P5b). The move puts the file back first now, and no finding + after it does. So does `member-gitlink-absent`'s own section lines + (#681), whose one-section file was the same fault. +- C2. `member-gitlink-removal-staged` read "section taken" from the empty + work tree and printed `git checkout HEAD -- .gitmodules` after a finding + had put the file back and written a section there, which it took out + again (P5d). It reads the file git reads now, puts nothing back after an + earlier finding did, and where a plain `git rm` took its section out, + writes that section back key by key into the file put back. +- C3. After a `gitmodules-value-refused` finding's lines, a stray's `git + rm --cached` exited 128 (`please stage your changes to .gitmodules or + stash them to proceed`, P5g), and so did a hand mount's `git rm` and the + wrong-case move's `git rm -r --cached`. Each comes after `git add -- + .gitmodules` there now, as #878's do. +- C4. After the `git mv` state's lines were typed, the checkout `git mv` + moved stayed at `Members/

`, untracked and named nowhere, so `family.py + add` refused it, and after the update `git -C Members/

status` read + the row's path. `member-gitlink-removal-staged` names it now, with that + status, before its update. +- C5. With every member checked out, bootstrap said `family bootstrap ok` + where git's update had exited 128, its init stopped at a gitlink it has + no url for or at an unmerged `.gitmodules`. It fails now, naming the + stop and its cure, with no credential advice. + +Every remedy is typed as printed, in the order printed, split as a POSIX +shell splits it and run with no shell; then the gate is asked in every +mode, and a plain clone's `git submodule update --init --recursive` must +check out both members. A url a remedy writes is the one `family.py add` +writes, on GitHub, and the clone reaches the fixture's local remotes +through it by a `url..insteadOf` for the one command (`cloning`). +THE CONTROLS: the same states with the work tree's `.gitmodules` on disk, +or with no value finding before the `git rm`, print what they printed. + +RV882'S ITEMS ON #882, TAKEN HERE (lane 2's review of #878): both of +`_stage_first`'s branches that print no `git add -- .gitmodules`, where +the work tree holds no file and where the index holds none (SHOULD 1); +`gitlink-unnamed` reading the paths from the file git reads where the +work tree's is gone (SHOULD 2); and `-f` over what is on disk at a staged +gitlink and is no checkout, a directory holding a file or a file, and +none over an empty directory, which git takes the plain line over (NIT +2). NIT 3's clause is in `STAGE_SAID`, which the C3 rows pin. + +THE SAME FAMILY, OFFLINE RULES AND HELPERS as +`tests/test_family_holder_bootstrap_texts.py` (#839) and +`tests/test_family_unnamed_gitlink.py` (#878). A case variant is an index +record (`recorded_at`, `stage_gitlink`), but for the `git mv` states, which +need `Members/IRRS` apart from `members/IRRS` on disk and are skipped where +the disk folds case (`case_sensitive_disk`). +""" + +from __future__ import annotations + +import re +import shlex +from pathlib import Path + +import pytest + +from conftest import rmtree, run_script +from test_family import FAMILY, SPARE, family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bootstrap_gate_order import ( + OUTSIDE_CURE, STOPPED, UNMERGED, UNMERGED_CURE, UNNAMED_CURE, bootstrap, + fetching, hand_mounted, stray, type_line, with_config, without_url, +) +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( # noqa: F401 + absent, assert_passes_in_every_mode, case_sensitive_disk, commit_all, + committed_gitmodules, github_to, pinned, recorded_at, + staged_and_committed, typed, variant_refused, +) +from test_family_holder_bootstrap_texts import ( + WORK, after_fetch_line, put_back_first, section_taken_out, + unmerged_gitmodules, +) +from test_family_member_path import MEMBER, OTHER, raw +from test_family_remove_gitmodules import verb_args +from test_family_unnamed_gitlink import ( + COMMIT_THAT, FORCED_NO_HEAD, STAGE_LINE, STAGE_SAID, TAKE_OUT, TOOLS, + assert_a_plain_clone_checks_out_every_member, at_tools, + found_in_every_mode, outside, section_taken_out_alone, staged_at, +) +from test_validate_family_typed_states import ( + GONE, ROW, SECTION_GIVEN_BACK, URL_ABSENT, VARIANT_PATH, gone, + unnamed_as_moved, +) +from test_validate_family_unnamed_mounts import ( + GITLINK_SAID, GITLINK_SAID_FETCHED, VARIANT, codes, commands, findings, + spare_commits, stage_gitlink, +) + +MODES = ((), ("--pins",), ("--no-members",)) +GITMODULES = ".gitmodules" + +#: What a remedy that writes `.gitmodules` says first where the work +#: tree's is gone, `{whose}` as `GONE` names it (C1). +PUT_BACK_THEN = (".gitmodules, gone from the work tree, back from {whose}, " + "so that every other section stays") + + +def cloning(offline, tmp_path: Path, family) -> dict: + """The offline environment under which a plain clone checks the members + out from the fixture's local remotes, through the GitHub url a remedy + writes as `family.py add` writes one (`github_to`).""" + key, value = github_to(family["remotes"])[1].split("=", 1) + return with_config(fetching(isolated(offline, tmp_path)), key, value) + + +def typed_in_order(root: Path, env: dict, result) -> None: + """Every `git` command every finding of `result` prints, typed as + printed, in the order printed: not `family.py add`, which mounts a hand + mount again only where the person wants it.""" + for finding in findings(result): + for line in commands(finding): + if line.startswith("git "): + type_line(root, env, line) + + +def refusal_lines(result) -> list[str]: + """The command lines a refusal prints, as printed.""" + return [line.strip() for line in result.stderr.splitlines()[2:]] + + +def found_alike(root: Path, *modes: tuple[str, ...]) -> list[str]: + """The gate's findings, exit 1, the same in each of `modes`.""" + seen = [] + for extra in modes: + result = validate(root, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + seen.append(findings(result)) + assert all(found == seen[0] for found in seen), seen + return seen[0] + + +def ends_well(root: Path, env: dict, family, tmp_path: Path) -> None: + """The gate passes in every mode, and a plain clone checks out both + members (#878's helper).""" + assert_passes_in_every_mode(root, 2) + assert_a_plain_clone_checks_out_every_member(root, env, family, tmp_path) + + +# --- C1: the wrong-case move where the work tree's .gitmodules is gone ------- + +def moved_in_head(root: Path, env: dict) -> str: + """MEMBER's gitlink recorded only at `Members/IRRS`, and the section of + HEAD's `.gitmodules` giving that path, as a committed `git mv` leaves + them (rv876's P5e state), by index records. Returns the commit.""" + commit = recorded_at(root, env, MEMBER, VARIANT_PATH) + data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout + line = f"\tpath = {ROW}\n".encode() + assert data.count(line) == 1, data + committed_gitmodules(root, env, data.replace( + line, f"\tpath = {VARIANT_PATH}\n".encode())) + return commit + + +def moved_refused(commit: str, how: str | None) -> list[str]: + """The members-mode refusal for `moved_in_head`'s state, with the line + that puts `.gitmodules` back first where the work tree's is gone as + `how` says (`GONE`), or none.""" + section = (f"git config -f .gitmodules submodule.{ROW}.path {ROW}", + "git add -- .gitmodules") + said = SECTION_GIVEN_BACK + if how is not None: + whose, line = GONE[how] + section = (line, *section) + said = f"put {PUT_BACK_THEN.format(whose=whose)}, then {said}" + return variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}, section=section, + section_said=said) + + +@pytest.mark.parametrize("how", GONE) +def test_c1_the_move_puts_the_file_back_before_its_section_line( + family, holder, offline, tmp_path, how): + """C1, rv876's P5e. Members mode's refusal is printed alone; its `git + config -f` line made a file of one key, which committed left HEAD with + no other section. It puts the file back first now, from where git + reads it. Typed as printed and committed, HEAD keeps OTHER's section, + the gate passes in every mode and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + commit = moved_in_head(holder, env) + gone(holder, env, how) + + refused = validate(holder) + + assert refused.returncode == 2, refused.stdout + refused.stderr + assert refused.stderr.splitlines() == moved_refused(commit, how) + for line in refusal_lines(refused): + type_line(holder, env, line) + commit_all(holder, env, "Move the gitlink as printed") + assert f'[submodule "members/{OTHER}"]'.encode() in raw( + holder, env, "show", f"HEAD:{GITMODULES}").stdout + ends_well(holder, env, family, tmp_path) + + +@pytest.mark.parametrize("how", GONE) +def test_c1_one_put_back_serves_the_move_and_a_url_finding_after_it( + family, holder, offline, tmp_path, how): + """C1, rv876's P5b: the same state with OTHER's url gone too. Under + `--no-members` the move prints first, and `member-gitmodules-url-absent` + after it put back the file the move had just staged, one key in it. + The move puts the file back first now, and the url finding puts nothing + back. Typed in order and committed, the gate passes in every mode and a + plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + commit = moved_in_head(holder, env) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, without_url(data, OTHER)) + gone(holder, env, how) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == [ + unnamed_as_moved(VARIANT_PATH, commit, moved_refused(commit, how), + said=GITLINK_SAID_FETCHED), + URL_ABSENT.format(commit=pinned(holder, OTHER))] + typed_in_order(holder, env, result) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) + + +def test_c1_control_with_the_file_on_disk_the_move_is_as_it_was( + family, holder, offline, tmp_path): + """No misfire: the same state with the work tree's `.gitmodules` on + disk. The move prints no put-back, as before; typed and committed, the + gate passes in every mode.""" + env = cloning(offline, tmp_path, family) + commit = moved_in_head(holder, env) + + refused = validate(holder) + + assert refused.returncode == 2, refused.stdout + refused.stderr + assert refused.stderr.splitlines() == moved_refused(commit, None) + for line in refusal_lines(refused): + type_line(holder, env, line) + commit_all(holder, env, "Move the gitlink as printed") + ends_well(holder, env, family, tmp_path) + + +@pytest.mark.parametrize("how", GONE) +def test_c1_the_absent_gitlink_s_section_lines_put_the_file_back_first( + family, holder, offline, tmp_path, how): + """C1's sibling, found with it: `member-gitlink-absent`'s own remedy + (#681) for a member taken out by a plain `git rm`, committed, with the + work tree's `.gitmodules` then gone. Its section lines made a file of + MEMBER's section alone, so committed, HEAD lost OTHER's. The file is + put back first now. Typed as printed, committed, and the member checked + out as the text says, the gate passes in every mode.""" + env = cloning(offline, tmp_path, family) + commit = pinned(holder, MEMBER) + raw(holder, env, "rm", "-q", "--", ROW) + commit_all(holder, env, "MEMBER taken out by hand") + gone(holder, env, how) + whose, line = GONE[how] + + refused = validate(holder) + + assert refused.returncode == 2, refused.stdout + refused.stderr + assert (f"with the lines after it, the first of which puts " + f"{PUT_BACK_THEN.format(whose=whose)}, then stage .gitmodules. " + in refused.stderr), refused.stderr + assert refusal_lines(refused) == [ + f"git update-index --add --cacheinfo 160000,{commit},{ROW}", line, + f"git config -f .gitmodules submodule.{ROW}.path {ROW}", + f"git config -f .gitmodules submodule.{ROW}.url " + f"https://github.com/InkRouter/{MEMBER}.git"] + for line in refusal_lines(refused): + type_line(holder, env, line) + staged_and_committed(holder, env, "Put the member back as printed") + raw(holder, env, "submodule", "update", "--init", "--recursive") + ends_well(holder, env, family, tmp_path) + + +# --- C2: the removal staged, beside a finding that put the file back -------- + +@pytest.mark.parametrize("how", GONE) +def test_c2_the_removal_s_line_puts_nothing_back_over_an_earlier_write( + family, holder, offline, tmp_path, how): + """C2, rv876's P5d: `git rm --cached members/IRRS`, OTHER's section out + of HEAD's `.gitmodules`, and the work tree's file gone. The absent + finding puts the file back and writes OTHER's section; the removal's + `git checkout HEAD -- .gitmodules`, read from the empty work tree, then + took it out, so the commit had nothing to commit. It reads the file git + reads now, which gives MEMBER's section, and prints no put-back. Typed + in order and committed, the gate passes and a plain clone checks both + out.""" + env = cloning(offline, tmp_path, family) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, section_taken_out(data, OTHER)) + raw(holder, env, "rm", "-q", "--cached", "--", ROW) + gone(holder, env, how) + + found = found_alike(holder, *MODES) + + assert [finding.split(":", 1)[0] for finding in found] == [ + "FINDING member-gitmodules-absent", + "FINDING member-gitlink-removal-staged"] + assert found[0] == put_back_first( + absent(OTHER, pinned(holder, OTHER), held=True), how) + assert commands(found[1]) == [f"git reset -q -- {ROW}"] + typed_in_order(holder, env, validate(holder, "--no-members")) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) + + +def url_of(root: Path, env: dict, path: str) -> str: + """The url the section of HEAD's `.gitmodules` named `path` gives.""" + return raw(root, env, "config", "--blob", f"HEAD:{GITMODULES}", + f"submodule.{path}.url").stdout.decode("utf-8").strip() + + +def test_c2_a_section_git_rm_took_is_written_back_into_the_file_put_back( + family, holder, offline, tmp_path): + """C2's other half, found with it: a plain `git rm members/IRRS`, which + takes its section out of the work tree's `.gitmodules` and stages that, + beside OTHER's section out of HEAD's, and the work tree's file then + deleted. The absent finding puts back the index's, which lacks MEMBER's + section, and the removal's `git checkout HEAD -- .gitmodules` took + OTHER's out again. Now the removal writes MEMBER's section back, key by + key as HEAD has it, into the file put back, before the update, which + dies `No url found` without it. Typed in order and committed, the gate + passes and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, section_taken_out(data, OTHER)) + member_url = url_of(holder, env, ROW) + raw(holder, env, "rm", "-q", "--", ROW) + gone(holder, env, "deleted") + + result = validate(holder, "--no-members") + + assert codes(result) == ["member-gitmodules-absent", + "member-gitlink-removal-staged"] + removal = findings(result)[1] + assert commands(removal) == [ + f"git reset -q -- {ROW}", + f"git config -f .gitmodules submodule.{ROW}.path {ROW}", + f"git config -f .gitmodules submodule.{ROW}.url " + f"{shlex.quote(member_url)}", + f"git submodule update --init --recursive -- {ROW}"] + assert ("the first its gitlink, the second to the third the .gitmodules " + "section git rm took out, key by key as HEAD has it, in the file " + "a line above put back, the fourth its checkout, and nothing is " + "left to commit.") in removal, removal + typed_in_order(holder, env, result) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) + + +#: `(how the removal and the file's loss are made, the line the removal +#: prints for .gitmodules)`: the removal's own finding alone, where it is +#: the one that puts the file back. +ALONE_GONE = { + "plain-rm-file-deleted": ( + lambda root, env: (raw(root, env, "rm", "-q", "--", ROW), + (root / GITMODULES).unlink()), + "git checkout HEAD -- .gitmodules"), + "cached-rm-deletion-staged": ( + lambda root, env: (raw(root, env, "rm", "-q", "--cached", "--", ROW), + raw(root, env, "rm", "-q", "--", GITMODULES)), + "git checkout HEAD -- .gitmodules"), + "cached-rm-file-deleted": ( + lambda root, env: (raw(root, env, "rm", "-q", "--cached", "--", ROW), + (root / GITMODULES).unlink()), + "git checkout -- .gitmodules"), +} + + +@pytest.mark.parametrize("make, line", ALONE_GONE.values(), ids=ALONE_GONE) +def test_c2_a_removal_alone_puts_the_file_back_as_git_reads_it( + family, holder, offline, tmp_path, make, line): + """The removal's own finding, with no other before it, where the work + tree's `.gitmodules` is gone: it puts the file back, from HEAD where a + plain `git rm` took MEMBER's section out of the index's, else from + where git reads it, so a deletion of `.gitmodules` staged beside the + removal is undone too. Typed as printed, nothing is left to commit, and + the gate passes in every mode.""" + env = cloning(offline, tmp_path, family) + make(holder, env) + + (removal,) = found_alike(holder, *MODES) + + assert line in commands(removal) + typed_in_order(holder, env, validate(holder, "--no-members")) + assert raw(holder, env, "status", "--porcelain").stdout == b"" + ends_well(holder, env, family, tmp_path) + + +def test_c2_control_with_the_file_on_disk_the_removal_is_as_it_was( + family, holder, offline, tmp_path): + """No misfire: a plain `git rm members/IRRS` with the work tree's + `.gitmodules` on disk prints what it printed: the reset, `git checkout + HEAD -- .gitmodules` for the section `git rm` took out, and the + update.""" + env = cloning(offline, tmp_path, family) + raw(holder, env, "rm", "-q", "--", ROW) + + (removal,) = found_alike(holder, *MODES) + + assert commands(removal) == [ + f"git reset -q -- {ROW}", "git checkout HEAD -- .gitmodules", + f"git submodule update --init --recursive -- {ROW}"] + assert ("the first its gitlink, the second the .gitmodules section git " + "rm took out, putting that file back as HEAD has it and dropping " + "any other change to it, the third its checkout") in removal + + +# --- C3: a git rm of a gitlink after a value finding's lines ---------------- + +def value_committed(root: Path, env: dict) -> None: + """MEMBER's section given `update = None`, committed (#768).""" + data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout + line = f"\tpath = {ROW}\n".encode() + assert data.count(line) == 1, data + committed_gitmodules(root, env, + data.replace(line, line + b"\tupdate = None\n")) + + +def staged_first(text: str) -> str: + """`text` after #878's sentence that stages `.gitmodules` first.""" + said = STAGE_SAID + text + return said[0].upper() + said[1:] + + +@pytest.mark.parametrize("value", [True, False], ids=["value", "control"]) +def test_c3_the_stray_s_git_rm_stages_the_file_first_after_a_value( + family, holder, offline, tmp_path, value): + """C3, rv876's P5g: a committed `update = None`, and a stray gitlink at + `Members/IRRS` staged with nothing on disk. The value finding's + `--unset-all` leaves `.gitmodules` with changes not staged, and the + stray's `git rm --cached` then exited 128. It comes after `git add -- + .gitmodules` now, in every mode, as #878's do. With no value finding + before it, the control, it prints none. Typed in order, and the value's + fix committed, nothing is left, the gate passes and a plain clone + checks both out.""" + env = cloning(offline, tmp_path, family) + if value: + value_committed(holder, env) + commit = pinned(holder, MEMBER) + stage_gitlink(holder, VARIANT_PATH, commit) + take = (staged_first(TAKE_OUT) + COMMIT_THAT + ":" + STAGE_LINE if value + else f"Take{TAKE_OUT[4:]}{COMMIT_THAT}:") + + found = found_alike(holder, *MODES) + + assert found[-1] == ( + GITLINK_SAID.format(path=VARIANT_PATH, commit=commit) + + VARIANT.format(row_path=ROW, project=MEMBER) + f" {take}" + + f"\n git rm --cached -- {VARIANT_PATH}") + typed_in_order(holder, env, validate(holder, "--no-members")) + if value: + staged_and_committed(holder, env, "Typed in order") + assert raw(holder, env, "status", "--porcelain").stdout == b"" + ends_well(holder, env, family, tmp_path) + + +def test_c3_a_hand_mount_s_git_rm_stages_the_file_first_after_a_value( + family, holder, offline, tmp_path): + """C3's sibling, found with it: a hand mount's plain `git rm`, after + the value finding's lines, exited 128 too. `git add -- .gitmodules` + comes before it now, the second of the four, and the text counts the + others on. Typed in order to the removal, and committed, the gate + passes and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + value_committed(holder, env) + path = hand_mounted(holder, env, family) + + result = validate(holder, "--no-members") + + assert codes(result) == ["gitmodules-value-refused", + "member-gitlink-unnamed"] + mount = findings(result)[1] + switch, stage, removal, add = commands(mount) + assert (stage, removal) == ("git add -- .gitmodules", f"git rm -- {path}") + assert (", stage .gitmodules with the second, which stages every other " + "change in it too, since git rm refuses to take any gitlink out " + "of the index") in mount + assert ("take it out with the third, which refuses while its checkout " + "holds changes") in mount + assert "then mount it with the fourth, " in mount + assert add.startswith("python3 scripts/family.py add "), add + for line in commands(findings(result)[0]) + [switch, stage, removal]: + type_line(holder, env, line) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) + + +def test_c3_the_wrong_case_move_s_git_rm_stages_the_file_first_after_a_value( + family, holder, offline, tmp_path): + """C3's sibling, found with it: the wrong-case move under + `--no-members`, after the value finding's lines: its `git rm -r + --cached` exited 128, and its `update-index` then left the gitlink at + both paths. `git add -- .gitmodules` comes first now. Typed in order + and committed, the gate passes and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + value_committed(holder, env) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + + result = validate(holder, "--no-members") + + assert codes(result) == ["gitmodules-value-refused", + "member-gitlink-unnamed"] + move = findings(result)[1] + assert commands(move) == [ + "git add -- .gitmodules", + f"git rm -r --cached --ignore-unmatch -- {VARIANT_PATH}", + f"git update-index --add --cacheinfo 160000,{commit},{ROW}"] + assert (" The member is recorded in the wrong case. " + + staged_first("move its gitlink to the row's path, at the " + "commit the row pins, in the index alone, which " + "touches no file, with the next two lines, and " + "commit that.")) in move + typed_in_order(holder, env, result) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) + + +# --- C4: the checkout git mv moved ------------------------------------------ + +#: What `member-gitlink-removal-staged` says of the checkout `git mv` moved +#: to `Members/IRRS`, before its update (C4). +MOVED = ( + " The checkout git mv moved to {variant} stays there, untracked, and " + "family.py add refuses it as family-members-dir-variant: its .git file " + "names the member's store under this holder's .git/modules/ by a path " + "relative to it, and its commits are kept there whatever becomes of the " + "checkout, so only what is not committed is in the checkout alone, which " + "`git -C {variant} status` shows. Delete it by hand once nothing only it " + "holds is needed, before the last line: that update points the store at " + "{path}, after which `git -C {variant} status` reads {path} instead.") + +#: The `git mv` states: nothing more, and #876's two. +MOVES = {"clean": lambda checkout, env: None, **WORK} + + +@pytest.mark.parametrize("work", MOVES.values(), ids=MOVES) +def test_c4_the_checkout_git_mv_moved_is_named_before_the_update( + family, holder, offline, tmp_path, case_sensitive_disk, work): + """C4, in the three `git mv` states rv876 measured: `git mv + members/IRRS Members/IRRS`, staged, clean, with a commit, or with an + untracked file in the checkout it moved. After the lines were typed, + `Members/IRRS` stayed, untracked and named nowhere, `family.py add` + refused it, and the update made its status read `members/IRRS`. The + removal's finding names it now, in every mode, with `git -C + Members/IRRS status` before the update. Typed in order, with that + status read and the checkout deleted by hand before the last line as + the text says, nothing is left untracked, the gate passes, `family.py + add` takes the spare member, the commit is in the member's store, and a + plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + (holder / "Members").mkdir() + raw(holder, env, "mv", ROW, VARIANT_PATH) + work(holder / VARIANT_PATH, env) + made = raw(holder / VARIANT_PATH, env, "rev-parse", "HEAD").stdout + + found = found_alike(holder, *MODES) + + assert ("and nothing is left to commit." + + MOVED.format(variant=VARIANT_PATH, path=ROW) + + " If it is to go,") in found[-1], found[-1] + lines = [line for finding in found for line in commands(finding)] + assert lines[-1] == f"git submodule update --init --recursive -- {ROW}" + for line in lines[:-1]: + type_line(holder, env, line) + status = raw(holder, env, "-C", VARIANT_PATH, "status", "--porcelain") + assert (b"?? scratch.txt\n" in status.stdout) is (work is WORK.get( + "untracked")), status.stdout + rmtree(holder / VARIANT_PATH) + type_line(holder, env, lines[-1]) + assert raw(holder, env, "status", "--porcelain").stdout == b"" + store = holder / ".git" / "modules" / ROW + assert raw(holder, env, f"--git-dir={store}", "cat-file", "-e", + made.decode("ascii").strip(), check=False).returncode == 0 + added = run_script(FAMILY, *verb_args("add", holder, family), + "--dry-run", env=env) + assert added.returncode == 0, added.stdout + added.stderr + ends_well(holder, env, family, tmp_path) + + +# --- C5: bootstrap with every member checked out ---------------------------- + +#: What (a) says where every row's member is checked out and git updated +#: none of them, and how the run then ends (C5). +NONE_UPDATED = (" NO MEMBER UPDATED, and not for want of a credential: " + "{why}. {cure}") +STOP_FAILED = "family bootstrap FAILED: {path} (git stopped at it)" + + +def outside_gitlink(root: Path, env: dict, family) -> str: + """SPARE's gitlink recorded by hand at `vendor/IRQS`, outside + `members/`, with no section, and committed. Returns its path.""" + stage_gitlink(root, f"vendor/{SPARE}", spare_commits(family)[1]) + commit_all(root, env, "A gitlink mounted by hand outside members/") + return f"vendor/{SPARE}" + + +#: `(how the stop is made in the holder, every member checked out, what +#: the line says cures it)`: each a gitlink git's init has no url for. +STOPS = { + "outside": (outside_gitlink, OUTSIDE_CURE), + "stray": (stray, UNNAMED_CURE), + "hand-mount": (hand_mounted, UNNAMED_CURE), +} + + +@pytest.mark.parametrize("make, cure", STOPS.values(), ids=STOPS) +def test_c5_a_stop_with_every_member_checked_out_fails_naming_it( + family, holder, offline, tmp_path, make, cure): + """C5: every member checked out, as in the holder itself, and a gitlink + git's init has no url for, at which `git submodule update --init` + exited 128 before it updated any member. Bootstrap gave the credential + advice and said ok. It names the stop now, with its cure, gives no + advice, and fails on the gitlink. The gate's remedy for that gitlink + typed as printed and committed, the run ends ok and a plain clone + checks both out.""" + env = cloning(offline, tmp_path, family) + path = make(holder, env, family) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert after_fetch_line(result) == [ + NONE_UPDATED.format(why=STOPPED.format(path=path), + cure=cure.format(path=path)), + STOP_FAILED.format(path=path)] + typed_in_order(holder, env, validate(holder, "--no-members")) + commit_all(holder, env, "The gate's remedy, typed as printed") + booted = bootstrap(holder, env, "--skip-members") + assert booted.stdout.endswith("\nfamily bootstrap ok\n"), ( + booted.stdout + booted.stderr) + ends_well(holder, env, family, tmp_path) + + +def test_c5_a_bad_line_after_the_first_section_fails_naming_a_stop( + family, holder, offline, tmp_path): + """C5, the issue's own state (RV877's O1): every member checked out, a + line git cannot parse committed in HEAD's `.gitmodules` after the first + section, and the work tree's file deleted. Git reads the index's up to + that line and dies `No url found` at the second member. Bootstrap said + ok with the advice. It fails now on the gitlink it stopped at, with no + advice: which one is the blob's reading, #877's (#856), so either + member's path is taken here.""" + env = cloning(offline, tmp_path, family) + data = (holder / GITMODULES).read_bytes() + first = data.index(b"[submodule", 1) + bad = (chr(9) + 'branch = "unterminated' + chr(10)).encode("ascii") + committed_gitmodules(holder, env, data[:first] + bad + data[first:]) + (holder / GITMODULES).unlink() + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + said = after_fetch_line(result) + assert said[0].startswith( + " NO MEMBER UPDATED, and not for want of a credential: git stopped " + "at members/"), said + assert re.fullmatch(rf"family bootstrap FAILED: members/({MEMBER}|" + rf"{OTHER}) \(git stopped at it\)", said[1]), said + + +def test_c5_an_unmerged_file_with_every_member_checked_out_fails( + family, holder, offline, tmp_path): + """C5's unmerged case: every member checked out, and `.gitmodules` held + only at a merge's conflict stages, which git reads nothing from, so its + init died at the first gitlink. Bootstrap said ok. It says why and the + merge's cure now, and fails on `.gitmodules`; resolved and added as the + cure says, the run ends ok.""" + env = cloning(offline, tmp_path, family) + unmerged_gitmodules(holder, env) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert after_fetch_line(result) == [ + NONE_UPDATED.format(why=UNMERGED, cure=UNMERGED_CURE), + "family bootstrap FAILED: .gitmodules (git cannot read it)"] + raw(holder, env, "add", GITMODULES) + booted = bootstrap(holder, env, "--skip-members") + assert booted.stdout.endswith("\nfamily bootstrap ok\n"), ( + booted.stdout + booted.stderr) + + +def test_c5_control_the_holder_as_made_bootstraps_ok(family, holder, offline, + tmp_path): + """No misfire: the holder as `family.py` made it, every member checked + out and nothing that stops git's init. Asking `init_stop` with no row + left finds nothing, and the run ends ok, saying every member and leg + was fetched, with nothing on stderr.""" + env = cloning(offline, tmp_path, family) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert " every member and leg fetched" in result.stdout.splitlines() + assert result.stderr == "" + + +# --- rv882's items on #882, taken here --------------------------------------- + +#: What the `-f` sentence says is on disk at a gitlink with no `.git` entry +#: there (rv882's NIT 2), in place of "a checkout". +OVER_FILES = "what is on disk there, which is no checkout" + + +def head_paths(root: Path, env: dict) -> set[str]: + """Every path a section of HEAD's `.gitmodules` gives.""" + listed = raw(root, env, "config", "--blob", f"HEAD:{GITMODULES}", + "--get-regexp", r"\.path$").stdout.decode("utf-8") + return {line.split(" ", 1)[1] for line in listed.splitlines()} + + +def test_rv882_should_1_no_git_add_where_the_work_tree_holds_no_file( + family, holder, offline, tmp_path): + """rv882's SHOULD 1 on #882: (D), the gitlink at tools/x that no + section gives, with the work tree's `.gitmodules` deleted and not + staged. No `git add -- .gitmodules` is printed, in any mode: it would + stage that deletion, the commit after it deleted the file from HEAD, + and a plain clone stopped at members/IRRS. Typed as printed and + committed, HEAD's `.gitmodules` still gives both members, the gate + passes in every mode and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + section_taken_out_alone(holder, env, family) + gone(holder, env, "deleted") + + finding = found_in_every_mode(holder, "gitlink-unnamed") + + assert finding == outside(TOOLS, staged_at(holder, env, TOOLS), + COMMIT_THAT) + typed(holder, env, finding) + commit_all(holder, env, "Typed as printed") + assert head_paths(holder, env) == {ROW, f"members/{OTHER}"} + ends_well(holder, env, family, tmp_path) + + +def test_rv882_should_1_no_git_add_where_the_index_holds_no_file( + family, holder, offline, tmp_path): + """rv882's SHOULD 1, the other branch: (D) with `.gitmodules` on disk + and its removal staged (`git rm --cached`), so the index holds none. + No `git add -- .gitmodules` is printed: it would stage the file again + over the removal the person staged. The `git rm --cached` alone, typed + as printed, exits 0, and the staged removal stays as it was.""" + env = cloning(offline, tmp_path, family) + section_taken_out_alone(holder, env, family) + raw(holder, env, "rm", "-q", "--cached", "--", GITMODULES) + + finding = found_in_every_mode(holder, "gitlink-unnamed") + + assert finding == outside(TOOLS, staged_at(holder, env, TOOLS), + COMMIT_THAT) + typed(holder, env, finding) + staged = raw(holder, env, "diff", "--cached", "--name-status").stdout + assert staged == f"D\t{GITMODULES}\nD\t{TOOLS}\n".encode(), staged + + +@pytest.mark.parametrize("how", GONE) +def test_rv882_should_2_a_gitlink_the_file_git_reads_gives_is_no_finding( + family, holder, offline, tmp_path, how): + """rv882's SHOULD 2 on #882: (A), IRSS mounted at tools/x with `git + submodule add` and committed, with the work tree's `.gitmodules` gone. + The index's or HEAD's file, which git reads in its place, gives tools/x + a section, and a plain clone checks it out, while `gitlink-unnamed` + said that no section gave it. The paths are read from the file git + reads now: no mode finds `gitlink-unnamed`, and the gate passes.""" + env = cloning(offline, tmp_path, family) + at_tools(holder, env, family) + gone(holder, env, how) + + assert_passes_in_every_mode(holder, 2) + + +def on_disk_under(root: Path, path: str) -> list[str]: + """Every entry on disk under `path`, and its bytes where it is a file.""" + return sorted((str(entry.relative_to(root)), + entry.read_bytes() if entry.is_file() else b"") + for entry in [root / path, *(root / path).rglob("*")]) + + +def files_at(root: Path, path: str) -> None: + """A directory at `path` that holds a file and no `.git`.""" + (root / path).mkdir(parents=True) + (root / path / "kept.txt").write_bytes(b"kept\n") + + +def a_file_at(root: Path, path: str) -> None: + """A file at `path`.""" + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(b"kept\n") + + +def an_empty_directory_at(root: Path, path: str) -> None: + """An empty directory at `path`.""" + (root / path).mkdir(parents=True) + + +#: `(what is on disk at tools/x, the flag its git rm --cached carries, what +#: the text says of it)`: git 2.43 refused the plain line over the first +#: two and took it over the third (rv882's NIT 2). +ON_DISK = { + "a-directory-holding-a-file": (files_at, " -f", FORCED_NO_HEAD.replace( + "over a checkout", f"over {OVER_FILES}")), + "a-file": (a_file_at, " -f", FORCED_NO_HEAD.replace( + "over a checkout", f"over {OVER_FILES}")), + "an-empty-directory": (an_empty_directory_at, "", COMMIT_THAT), +} + + +@pytest.mark.parametrize("make, flag, said", ON_DISK.values(), ids=ON_DISK) +def test_rv882_nit_2_a_gitlink_staged_over_what_is_no_checkout( + family, holder, offline, tmp_path, make, flag, said): + """rv882's NIT 2 on #882: IRQS's gitlink staged by hand at tools/x, + HEAD recording nothing there, over a directory holding a file and no + `.git`, or a file. Git refused the plain `git rm --cached` there + (`staged content different from both the file and the HEAD`, exit 1), + and `_forced` asked for a `.git` entry alone, so no `-f` was printed. + It is now, with what is on disk there; over an empty directory, which + git takes the plain line over, it is not. Typed as printed, the line + exits 0 and leaves what was on disk; the gate passes in every mode and + a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + make(holder, TOOLS) + commit = spare_commits(family)[1] + stage_gitlink(holder, TOOLS, commit) + before = on_disk_under(holder, "tools") + + finding = found_in_every_mode(holder, "gitlink-unnamed") + + assert finding == outside(TOOLS, commit, said, flag) + typed(holder, env, finding) + assert on_disk_under(holder, "tools") == before + ends_well(holder, env, family, tmp_path) + + +def test_rv882_nit_2_the_move_takes_f_over_what_is_no_checkout( + family, holder, offline, tmp_path, case_sensitive_disk): + """rv882's NIT 2, in the wrong-case move, which asks `_forced` too: + MEMBER recorded only at `Members/IRRS` and committed, then staged + there at another commit, over a directory holding a file and no + `.git`. The move's `git rm -r --cached` carries `-f` now, and says + what is on disk there; typed as printed and committed, it exits 0, + leaves the file, and the gate passes in members mode.""" + env = cloning(offline, tmp_path, family) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + other = spare_commits(family)[1] + stage_gitlink(holder, VARIANT_PATH, other) + files_at(holder, VARIANT_PATH) + + refused = validate(holder) + + assert refused.returncode == 2, refused.stdout + refused.stderr + assert refused.stderr.splitlines() == [ + line.replace("over a checkout", f"over {OVER_FILES}") + for line in variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: other}, + forced=(VARIANT_PATH,))] + for line in refusal_lines(refused): + type_line(holder, env, line) + commit_all(holder, env, "Moved as printed") + assert (holder / VARIANT_PATH / "kept.txt").read_bytes() == b"kept\n" + assert validate(holder).returncode == 0 diff --git a/tests/test_family_unnamed_gitlink.py b/tests/test_family_unnamed_gitlink.py index 6f922260..4450822d 100644 --- a/tests/test_family_unnamed_gitlink.py +++ b/tests/test_family_unnamed_gitlink.py @@ -96,8 +96,10 @@ #: What it says first where lines printed before it may leave `.gitmodules` #: with changes not staged, or it holds some, and the line it prints for -#: that: git takes no gitlink out of the index then (#879's C3). -STAGE_SAID = ("stage .gitmodules with the first line, since git rm refuses to " +#: that: git takes no gitlink out of the index then (#879's C3), and the +#: line stages every other change in the file too (rv882's NIT 3). +STAGE_SAID = ("stage .gitmodules with the first line, which stages every " + "other change in it too, since git rm refuses to " "take any gitlink out of the index, -f or not, while " ".gitmodules holds changes not staged (please stage your " "changes to .gitmodules or stash them to proceed, exit 128), " From 3b15de030c398edd5dfd86622c358b2decba96cd Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 12:46:28 +0000 Subject: [PATCH 16/20] validate-family.py and bootstrap.py: a staged removal's section written back key by key is staged, no removal puts HEAD's `.gitmodules` back over an earlier finding's write, a clone of its own at a case variant is said to be one, a checkout `git mv` moved out of `members/` is named, and a run with no row says nothing was updated (#879) Fix round 1 for #889: lane 1's review (issuecomment-6249673233), every item taken. - MUST 1: `_keys_back` ends the key-by-key lines with `git add -- .gitmodules` and the text says to commit the file (P5); the first removal with the file gone puts back HEAD's where the index's lacks a section another staged removal needs (P4, as #876's head printed). - SHOULD 1: `Report.wrote`, set by every finding that writes the file or says to commit it as it stands; a removal after one writes its section key by key and stages it, on disk too, leaving out a key HEAD's gives a value git dies on. - SHOULDs 2 and 3: `_moved_checkouts` names each gitlink whose `.git` file names the member's store (`_store`, `_names_store`), at any path, and says of a case variant whose `.git` is a directory what the wrong-case move says of a clone of its own. - SHOULD 4: bootstrap.py's docstrings say "where git's init in this holder stopped", and name the exit 128 its records do not show. - NITs: the value finding's put-back in the module docstring (1); a symbolic link row (2); `_wrong_case_move`'s docstring and `_forced_over` per variant (3); `updated_none` asks nothing of the rows (4); `NOTHING UPDATED` under no row (5); a row for MX1 (6). Tests: 16 new rows in tests/test_family_holder_remedies_compose.py, and the C2 and C4 rows' expected text. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 68 ++- .../family-root/scripts/validate-family.py | 422 +++++++++++---- tests/test_family_holder_remedies_compose.py | 489 +++++++++++++++++- 3 files changed, 816 insertions(+), 163 deletions(-) diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 009a99fb..a318ead3 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -30,13 +30,19 @@ no url for, nor where it read nothing from `.gitmodules`, both below, since it fetched nothing then (#839). - Where every row's member is checked out and git's init died all the - same, at a gitlink it has no url for or at a `.gitmodules` a merge - left unmerged, git updated none of them, so the run fails, saying - where git stopped and what cures it, as below, with no advice - (#879). It said `family bootstrap ok` there, beside git's exit 128, - as a run with every member checked out passes (#648), and gave the - advice where git stopped at a gitlink. + Where every row's member is checked out and git's init in this + holder died all the same, at a gitlink this holder records with no + url or at a `.gitmodules` a merge left unmerged, git updated none of + them, so the run fails, saying where git stopped and what cures it, + as below, with no advice (#879). It said `family bootstrap ok` + there, beside git's exit 128, as a run with every member checked + out passes (#648), and gave the advice where git stopped at a + gitlink. This holder's records are the test, so an exit 128 they do + not show is read as before, the advice and `ok` where every member + is checked out: git stopped inside a member, at a gitlink in that + member's own index with no url, or at a gitlink of this holder's at + a commit no remote holds (`not our ref`) (lane 1's SHOULD 4 on + #889). "every member and leg fetched" is said only where git exited 0 AND every row's member is checked out (#648). A member left NOT CHECKED @@ -100,8 +106,10 @@ (c) EXITS 0 unless a member's own command failed, or (a) left a member not checked out that no credential would check out, or found a line or a value in `.gitmodules` that git cannot read (#768), or found that, - with every member checked out, git's init died at a gitlink it has - no url for or at an unmerged `.gitmodules` (#879). Any + with every member checked out, git's init in this holder died at a + gitlink this holder records with no url or at an unmerged + `.gitmodules` (#879); an exit 128 this holder's records do not show + is no failure here, as (a) says. Any other member that is not checked out is REPORTED and skipped, not a failure: it is the state a missing credential leaves, and it is already reported in (a). @@ -151,6 +159,10 @@ NO_URL = "git has no url for that path" EXITED_0 = "git exited 0 and checked nothing out there" STOPPED = "git stopped at {}, a gitlink it has no url for" +#: What (a)'s line says where git updated no member, by whether `members:` +#: names any row: under "0 member(s)" it said no member was updated (lane +#: 1's NIT 5 on #889). +NONE_UPDATED = {True: "NO MEMBER UPDATED", False: "NOTHING UPDATED"} #: The whys a `.gitmodules` git read only up to a line it cannot parse, or #: up to a key with no value it stops at, may be the cause of, since what #: it did not read is past that line (`Gitmodules.cut`, #856). @@ -213,8 +225,10 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: `.gitmodules`, or on a work tree file it cannot parse, or its init died at a `.gitmodules` a merge left unmerged, so it updated none of them, the one failure line it returns is for `.gitmodules` (#768, - #879); where its init died at a gitlink it has no url for, it is for - that gitlink (`updated_none`, #879). + #879); where its init in this holder died at a gitlink this holder + records with no url, it is for that gitlink (`updated_none`, #879). + Its line says no member was updated, or, where `rows` names none, + nothing was (lane 1's NIT 5 on #889). Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for @@ -245,23 +259,24 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: if proc.returncode != 0: fetch_incomplete(proc, source, token, credential_advice(bool(left), stopper, modules)) - whole = updated_none(left, stopper, modules) if left: print(" NOT CHECKED OUT, and not for want of a credential: " + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" for _, path, why in left) + ". " + cures(left, stopper, rows, modules), file=sys.stderr) - elif whole is not None: - print(" NO MEMBER UPDATED, and not for want of a credential: " - + reason(whole, stopper, modules) + ". " + return [f"{spelled(row.get('project') or '?')} (not checked out)" + for row, _, _ in left] + whole = updated_none(stopper, modules) + if whole is not None: + print(f" {NONE_UPDATED[bool(rows)]}, and not for want of a " + "credential: " + reason(whole, stopper, modules) + ". " + cures(left, stopper, rows, modules, (whole,)), file=sys.stderr) return [f"{spelled(stopper)} (git stopped at it)" if whole == STOPPED else f"{GITMODULES} (git cannot read it)"] - elif proc.returncode == 0: + if proc.returncode == 0: print(" every member and leg fetched") - return [f"{spelled(row.get('project') or '?')} (not checked out)" - for row, _, _ in left] + return [] def fetch_incomplete(proc: subprocess.CompletedProcess, source: str, @@ -946,16 +961,15 @@ def file_reason(modules: Gitmodules) -> str | None: return UNREADABLE if modules.unreadable else None -def updated_none(left: list[tuple[dict, str, str]], stopper: str | None, - modules: Gitmodules) -> str | None: +def updated_none(stopper: str | None, modules: Gitmodules) -> str | None: """Why git updated no member, where every row's member is checked out - (`left` empty) and git did not exit 0, or None: what `file_reason` - says, or `STOPPED` where git's init died at a gitlink it has no url - for (`stopper`, `init_stop`), before it updated any (#879). None - where it asked for a credential, as a leg's fetch does, and where - `left` names a row, which (a)'s line says why of.""" - if left: - return None + and git did not exit 0, or None: what `file_reason` says, or `STOPPED` + where git's init in this holder died at a gitlink this holder records + with no url (`stopper`, `init_stop`), before it updated any (#879). + None where it asked for a credential, as a leg's fetch does. + `fetch_members` asks it only where `unfetchable` left no row, whose + line says why of each, so it asks nothing of the rows (lane 1's NIT 4 + on #889: a check of them here decided nothing).""" return file_reason(modules) or (None if stopper is None else STOPPED) diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index c90e3f94..3e815af9 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -127,12 +127,16 @@ EVERY REMEDY IS TYPED AS PRINTED, IN THE ORDER PRINTED (#879). Where the work tree's `.gitmodules` is gone from disk, the first remedy that writes -it puts it back first, from where git reads it, and no later one puts it -back over what the lines between wrote; a `git rm` of a gitlink comes -after `git add -- .gitmodules` where lines printed before it leave that -file with changes not staged; and a checkout `git mv` moved to a case -variant of a row's path is named, with its `git -C` status, before the -update that points its store at the row's path. +it puts it back first, from where git reads it, but for a +`gitmodules-value-refused` finding, which puts back HEAD's, whose value +its lines take out (#768), and no later one puts it back over what the +lines between wrote: a staged removal's finding after one that writes +the file, or says to commit it, writes its section back key by key and +stages the file, on disk or not. A `git rm` of a gitlink comes after +`git add -- .gitmodules` where lines printed before it leave that file +with changes not staged; and a checkout `git mv` moved off a row's path, +whose `.git` file names that member's store, is named, with its `git -C` +status, before the update that points the store at the row's path. EXIT CODES 0 the manifest is valid, every member is in lockstep, every copy matches @@ -276,6 +280,11 @@ def __init__(self) -> None: #: puts it back over what the lines between wrote (#879); empty #: until one has. self.put_back = "" + #: Whether a finding printed before says to write the work tree's + #: `.gitmodules`, or to commit it as it stands, so that no later + #: finding puts HEAD's file back over that (`_section_back`, lane + #: 1's MUST 1 and SHOULD 1 on #889). + self.wrote = False def finding(self, code: str, detail: str) -> None: self.findings.append(f"FINDING {code}: {detail}") @@ -672,9 +681,9 @@ class MoveContext(NamedTuple): `.gitmodules` give the row's path, with what they do, or empty where a section of HEAD's already gives it. `forced`, the variants whose `git rm --cached` takes `-f` (`_forced`, #839), and `forced_over`, what is - on disk at them, as the `-f` sentence says it (`_held_as`, rv882's NIT - 2 on #882). `stage`, whether `git add -- .gitmodules` comes first - (`_stage_first`, #879).""" + on disk at them, as the `-f` sentence says it (`_forced_over`, rv882's + NIT 2 on #882, and lane 1's NIT 3 on #889). `stage`, whether `git add + -- .gitmodules` comes first (`_stage_first`, #879).""" sparse: tuple[str, ...] = () in_the_way: tuple[str, ...] = () @@ -742,10 +751,22 @@ def _move_context(root: Path, path: str, variants: list[str], section, said = _move_section(root, path, variants, repository, report) edited = report is not None and "gitmodules-value-refused" in report.codes forced = _forced_variants(root, variants, gitlinks) - over = (CHECKOUT if all(_held_as(root, variant) == CHECKOUT - for variant in forced) else NO_CHECKOUT) return MoveContext(sparse, in_the_way, section, said, forced, - _stage_first(root, edited), over) + _stage_first(root, edited), _forced_over(root, forced)) + + +def _forced_over(root: Path, forced: tuple[str, ...]) -> str: + """What the `-f` sentence says is on disk at the variants `forced` + (`_held_as`): `CHECKOUT` or `NO_CHECKOUT` where every one holds the + same, and each by its path where they differ, since one said for all + called a checkout no checkout (lane 1's NIT 3 on #889).""" + held = {variant: _held_as(root, variant) for variant in forced} + if len(set(held.values())) < 2: + return next(iter(held.values()), CHECKOUT) + return ", and over ".join( + f"a checkout at {variant}" if what == CHECKOUT + else f"what is on disk at {variant}, which is no checkout" + for variant, what in held.items()) def _index_around(root: Path, path: str, variants: list[str] @@ -775,7 +796,9 @@ def _move_section(root: Path, path: str, variants: list[str], repository, path given the row's, or one written as `family.py add` writes it, after the line that puts the file back where it is gone from the work tree and no finding before put it back (`_file_put_back`, #879's C1), - then `git add`.""" + then `git add`. Where `report` holds the findings printed before it, + the write is recorded there (`Report.wrote`), so no finding after it + puts HEAD's file back over it (lane 1's MUST 1 on #889, its P5).""" head = _head_gitmodules(root) sections = head[1] if head is not None else {} given = {keys["path"]: name for name, keys in sections.items() @@ -800,6 +823,8 @@ def _move_section(root: Path, path: str, variants: list[str], repository, section = f"\n {put_back}{section}" said = (f"put {GITMODULES}, gone from the work tree, back from " f"{whose}, so that every other section stays, then {said}") + if report is not None: + report.wrote = True return f"{section}\n git add -- {GITMODULES}", said @@ -898,19 +923,20 @@ def _wrong_case_move(path: str, commit: str, variants: list[str], is kept). THE `git rm` TAKES `-f` where the index stages a variant at a commit - HEAD does not record there and a checkout is on disk at it - (`MoveContext.forced`, `_forced`, #839): git refuses it there wherever - that checkout has moved on or holds a file it does not track, and the - text says what `-f` does (`FORCED_SAID`). It comes after `git add -- - .gitmodules` where `MoveContext.stage` (`_stage_first`, #879's C3). + HEAD does not record there and anything but an empty directory is on + disk at it (`MoveContext.forced`, `_forced`, `_on_disk`, #839, rv882's + NIT 2 on #882): git refuses it there wherever a checkout there has + moved on or holds a file it does not track, and over a directory that + holds files and no `.git`, a file or a symbolic link. The text says + what `-f` does (`FORCED_SAID`) and what is on disk there, each + variant's where they differ (`MoveContext.forced_over`, lane 1's NIT 3 + on #889). It comes after `git add -- .gitmodules` where + `MoveContext.stage` (`_stage_first`, #879's C3). """ taken_out = " ".join(shlex.quote(variant) for variant in variants) moved = " and ".join(_guarded_move(variant, path) for variant in variants) - status = " or ".join(f"`git -C {shlex.quote(variant)} status`" - for variant in variants) - unpushed = " or ".join(f"`git -C {shlex.quote(variant)} log --oneline " - "HEAD --branches --not --remotes`" - for variant in variants) + status = _statuses(variants) + unpushed = _unpushed(variants) flag = (" --sparse" if context.sparse else "") + ( " -f" if context.forced else "") lead = (STAGE_SAID if context.stage else "") + "move" @@ -1638,7 +1664,11 @@ def _check_gitmodules_values(root: Path, members: list, `https://github.com/` and the row's `repository:` (`_written_back`). Where HEAD's file alone gives the value, the work tree's is what to commit, and where the work tree has no `.gitmodules`, the first such - finding puts HEAD's back first.""" + finding puts HEAD's back first, whose value its lines take out, and + not the index's, which git reads there: a fix staged in the index's + file is lost (#768, lane 1's NIT 1 on #889). Every one of these is + recorded as a write (`Report.wrote`): a staged removal's finding after + it then puts no file back over the fix (lane 1's SHOULD 1).""" present = os.path.lexists(root / GITMODULES) here = _submodule_entries(_gitmodules_listed(root)) head = _head_listed(root) @@ -1650,6 +1680,7 @@ def _check_gitmodules_values(root: Path, members: list, and isinstance(row.get("project"), str) and row["project"]} put_back = present for name, key in dict.fromkeys([*in_work, *in_head]): + report.wrote = True said = _value_said(name, key, in_work, in_head) if present and (name, key) not in in_work: report.finding("gitmodules-value-refused", @@ -1873,7 +1904,9 @@ def _check_gitmodules_paths(sections: dict[str, dict[str, str]], fetched. `add` refuses to write one now; this finds a holder that already has one. `sections` is `_gitmodules_sections`'s reading. A section whose path no row names is `_check_gitmodules_named`'s question - (#361). + (#361). Its by-hand edit is recorded as a write (`Report.wrote`), so a + staged removal's finding after it puts no file back over the edit + (lane 1's SHOULD 1 on #889). """ paths: dict[str, list[str]] = {} for name, keys in sections.items(): @@ -1897,6 +1930,7 @@ def _check_gitmodules_paths(sections: dict[str, dict[str, str]], f"-f {GITMODULES} --unset-all submodule..` for every " "key that section holds, so a submodule later given its name " f"inherits none of them, then commit {GITMODULES}.") + report.wrote = True def _in_members(path: str) -> bool: @@ -2192,6 +2226,11 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], loop, and each one's remedy is `_variant_remedy`'s or `_mounted_by_hand`'s, so this stays within S3776's limit (rv882's SHOULD 3 on #882). + + A hand mount's plain `git rm` takes its section out of the work tree's + `.gitmodules`, as the move's lines write one (`_move_section`): each is + recorded as a write (`Report.wrote`), so a staged removal's finding + after it puts no file back over it (lane 1's SHOULD 1 on #889). """ unnamed = [path for path in sorted(gitlinks) if path not in named and _in_members(path)] @@ -2216,6 +2255,7 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], "member-gitlink-unnamed", f"{said} " + _mounted_by_hand(root, path, _hand_project(path), path in removed, stage)) + report.wrote = report.wrote or path not in removed def _gitlink_unnamed_said(path: str, commit: str, fetched: bool) -> str: @@ -2550,7 +2590,10 @@ def _check_gitmodules_named(root: Path, sections: dict[str, dict[str, str]], again fails. A gitlink whose removal is staged prints none, as above. After a finding whose lines write `.gitmodules`, a value's or an earlier section's (`edited`), the `git rm` comes after `git add -- - .gitmodules`, which it needs (`_stage_first`). + .gitmodules`, which it needs (`_stage_first`). Each finding that + prints lines is recorded as a write (`Report.wrote`), so a staged + removal's finding after it puts no file back over them (lane 1's + SHOULD 1 on #889). What a section in another case says is `_variant_advice`'s, and the rest of each remedy is `_section_out`'s, after `_gitlink_out_first`'s @@ -2589,7 +2632,7 @@ def _check_gitmodules_named(root: Path, sections: dict[str, dict[str, str]], said += SECTION_ROW_BACK + _gitlink_out_first( root, name, keys, gitlinks, removed, taken, edited) report.finding("gitmodules-path-unnamed", said) - edited = True + edited = report.wrote = True #: What `gitmodules-path-unnamed` says of a section no row names, in no @@ -2755,7 +2798,10 @@ def _check_gitlinks_registered(root: Path, members: list, the first had written (#839). Nor where the wrong-case move printed before them put it back (`Report.put_back`, #879's C1), whose write the url-absent finding's put-back undid; the one these print is recorded - there, for `member-gitlink-removal-staged` after them (#879's C2). + there, for `member-gitlink-removal-staged` after them (#879's C2), and + so is each write, on disk or not (`Report.wrote`): that finding's + `git checkout HEAD -- .gitmodules` took a section these wrote out + again (lane 1's SHOULD 1 on #889). """ head = _head_gitmodules(root) if head is None: @@ -2775,6 +2821,7 @@ def _check_gitlinks_registered(root: Path, members: list, row, sections, report, back): unregistered.add(path) report.put_back = report.put_back or spec + report.wrote = True back = ("", "") continue unregistered.add(path) @@ -2782,6 +2829,7 @@ def _check_gitlinks_registered(root: Path, members: list, path, gitlinks[path], held, _section_lines(path, row.get("repository"), sections), back)) report.put_back = report.put_back or spec + report.wrote = True back = ("", "") return unregistered @@ -2872,30 +2920,38 @@ def _check_removals_staged(root: Path, members: list, BOTH WAYS OUT ARE TYPED (rv762's probes, git 2.43): to stay, `git reset -q -- ` puts the gitlink back as HEAD records it; where a plain `git rm` also took its section out of the work tree's `.gitmodules` - (`worktree`), `git checkout HEAD -- .gitmodules` puts that back, and + (`worktree`), the lines `_section_back` gives put that back, and where it took the checkout, `git submodule update --init --recursive -- ` checks it out again, since that update dies `No url found` while the section is gone. To go, the same, then `family.py remove`, which takes the row, the gitlink and the section out in one commit and refuses a work tree's `.gitmodules` that is not the index's (`gitmodules-unstaged`) and a staged removal alike, on a branch, as the - duplicate-id remedy names it. + duplicate-id remedy names it. HEAD's `.gitmodules` is read once: the + rows whose removal is staged and whose path it gives, `wanted`, for + the first put-back of a file gone from disk (lane 1's MUST 1 on #889), + and every key it gives a value git dies on, `refused` + (`_values_refused`), which a section written back key by key leaves + out. The checkouts it names are `_moved_checkouts`'s. """ if not removed: return set() - head = _head_gitmodules(root) - sections = head[1] if head is not None else {} - staged: set[str] = set() - for row, project, path in _rows_at_their_paths(members): - if path not in removed: - continue - staged.add(path) + head = _head_listed(root) + listed = head[1] if head is not None else b"" + sections = _sections_listed(listed) + given = {keys["path"] for keys in sections.values() if "path" in keys} + rows = [(row, project, path) + for row, project, path in _rows_at_their_paths(members) + if path in removed] + wanted = {path for _, _, path in rows if path in given} + refused = set(_values_refused(root, _submodule_entries(listed))) + for row, project, path in rows: lines = _put_back_lines(root, path, _section_back( - root, path, sections, worktree, report)) + root, path, sections, worktree, report, (wanted, refused))) report.finding("member-gitlink-removal-staged", _removal_staged_said( root, row, project, path, gitlinks[path], lines, - _moved_checkouts(root, path, gitlinks))) - return staged + _moved_checkouts(root, path, gitlinks, sections))) + return {path for _, _, path in rows} def _rows_at_their_paths(members: list) -> Iterator[tuple[dict, str, str]]: @@ -2915,17 +2971,75 @@ def _rows_at_their_paths(members: list) -> Iterator[tuple[dict, str, str]]: yield row, project, path -def _moved_checkouts(root: Path, path: str, - gitlinks: dict[str, str]) -> list[str]: - """Each case variant of `path` the index holds a gitlink at and a `.git` - entry is on disk at, where none is at `path`: the checkout `git mv` - moved, which `member-gitlink-removal-staged` names (`_moved_said`, - #879's C4); none where a checkout is at `path`.""" +def _moved_checkouts(root: Path, path: str, gitlinks: dict[str, str], + sections: dict[str, dict[str, str]] + ) -> tuple[list[str], list[str]]: + """`(moved, clones)`, what `member-gitlink-removal-staged` names + (`_moved_said`) of the row's gitlink at `path`, where no checkout is + there, else two empty lists. + + `moved`: each other gitlink the index holds whose `.git` FILE names + the member's own store (`_store`, `_names_store`), the checkout `git + mv` moved, whose store the update the finding prints last points back + at `path` (#879's C4). + Any path, not a case variant alone: after `git mv members/

+ renamed-1`, typed as printed, `renamed-1` was left untracked, named + by no finding, with a file only it held unseen (lane 1's SHOULD 3 on + #889). + + `clones`: each case variant of `path` the index holds a gitlink at + whose `.git` is a DIRECTORY, a clone of its own, which holds its + commits there and nowhere else. It was said to be the checkout `git + mv` moved, to delete by hand, which lost a commit only it held (lane + 1's SHOULD 2 on #889).""" if (root / path / ".git").exists(): - return [] - return [variant for variant in sorted(gitlinks) - if variant != path and variant.casefold() == path.casefold() - and os.path.lexists(root / variant / ".git")] + return [], [] + store = _store(root, path, sections) + others = [other for other in sorted(gitlinks) if other != path] + moved = [other for other in others + if _names_store(root / other / ".git", store)] + clones = [other for other in others + if other.casefold() == path.casefold() + and (root / other / ".git").is_dir()] + return moved, clones + + +def _store(root: Path, path: str, + sections: dict[str, dict[str, str]]) -> str: + """Where this holder keeps the store of the submodule at `path`, as git + does, `modules/` in its git directory, resolved and with its case + normalised for `_names_store`, or nothing where git cannot name that + directory: `` the section of HEAD's `.gitmodules` (`sections`) + that gives the path, the last, as git reads it, else the path, as `git + submodule add` names one (lane 1's SHOULD 3 on #889).""" + names = [name for name, keys in sections.items() + if keys.get("path") == path] + with _git_runs("this holder's git directory could not be read"): + try: + common = git_out(["rev-parse", "--git-common-dir"], cwd=root, + binary=True) + except Refusal: + return "" + common_dir = root / common.decode("utf-8", "surrogateescape").strip() + return os.path.normcase(os.path.realpath( + common_dir / "modules" / (names[-1] if names else path))) + + +def _names_store(gitfile: Path, store: str) -> bool: + """Whether `gitfile` is a `.git` FILE that names `store` (`_store`), read + as git reads one: `gitdir: `, trailing blanks stripped, the path + relative to the file's directory unless it is absolute.""" + if not store or not gitfile.is_file(): + return False + try: + text = gitfile.read_bytes().decode("utf-8", "surrogateescape") + except OSError: + return False + text = text.rstrip() + if not text.startswith("gitdir: "): + return False + named = os.path.join(gitfile.parent, text[len("gitdir: "):]) + return os.path.normcase(os.path.realpath(named)) == store #: What `member-gitlink-removal-staged` says its `git checkout HEAD -- @@ -2938,49 +3052,54 @@ def _moved_checkouts(root: Path, path: str, def _section_back(root: Path, path: str, sections: dict[str, dict[str, str]], worktree: dict[str, dict[str, str]], - report: Report) -> list[tuple[list[str], str]]: + report: Report, + asked: tuple[set[str], set[tuple[str, str]]] + ) -> list[tuple[list[str], str]]: """The lines `member-gitlink-removal-staged` prints for `.gitmodules`, with what they put back, where HEAD's (`sections`) gives `path` (#762): - `git checkout HEAD -- .gitmodules` where a plain `git rm` took the - section out of the work tree's (`worktree`), and nothing where it is - there. - - WHERE THE WORK TREE'S FILE IS GONE FROM DISK it is read as git reads it - (#879's C2, rv876's P5d): from the work tree, empty, the section was - always said to be taken, and the `git checkout HEAD` after a finding - that had put the file back and written a section there took that - section out again. So where a finding printed before this one put the - file back (`Report.put_back`), nothing is put back again: where that - file gives the path, nothing at all, and where it does not, the section - HEAD's gives is written into it, key by key, as HEAD has it, since - `git submodule update --init` dies `No url found` without it. Where - none did, this one puts it back, and records that: from HEAD where the - file git reads, the index's, lacks the section (`SECTION_TAKEN`), else - from where git reads it (`_gitmodules_put_back`), so that the line - below keeps the member as HEAD has it and a deletion of `.gitmodules` - staged beside its removal is undone too, with nothing left to - commit.""" + none where the work tree's (`worktree`) gives it, nor where the file a + finding before put back in place of one gone from disk does + (`Report.put_back`), and else `git checkout HEAD -- .gitmodules`, which + puts back the section a plain `git rm` took out. + + NOTHING IS PUT BACK OVER WHAT A FINDING PRINTED BEFORE WROTE, or where + it put the file back (`Report.wrote`, `Report.put_back`), on disk or + not: the section HEAD's gives is written in key by key, as HEAD has + it, but for a key it gives a value git dies on (`asked`'s second, + which `gitmodules-value-refused` takes out), and the file staged + (`_keys_back`). The `git checkout HEAD` took + out again a section a finding before had written, where the work + tree's file was gone (#879's C2, rv876's P5d) and where it was on disk + (lane 1's SHOULD 1 on #889); and written in alone, the section stayed + out of the index's file, which the commit records, so the commit + dropped it (lane 1's MUST 1, its P5). + + WHERE THE WORK TREE'S FILE IS GONE FROM DISK and no finding before put + it back, this one does, and records that: from where git reads it + (`_gitmodules_put_back`) where that file gives every path `asked`'s + first holds, each row whose removal is staged and whose path HEAD's + gives, so that + a deletion of `.gitmodules` staged beside the removal is undone too, + with nothing left to commit, and else from HEAD (`SECTION_TAKEN`). Put + back from the index's, which a plain `git rm` of a second row had + taken that row's section out of, the second row's finding wrote it in + key by key and the commit dropped it (lane 1's MUST 1, its P4).""" given = {keys["path"] for keys in sections.values() if "path" in keys} if path not in given: return [] - if os.path.lexists(root / GITMODULES): - return ([] if any(keys.get("path") == path - for keys in worktree.values()) - else [([f"git checkout HEAD -- {GITMODULES}"], - SECTION_TAKEN)]) - if report.put_back: - if path in _blob_paths(root, report.put_back): - return [] - name = [name for name, keys in sections.items() - if keys.get("path") == path][-1] - return [([f"git config -f {GITMODULES} " - f"{shlex.quote(f'submodule.{name}.{key}')} " - f"{shlex.quote(value)}" - for key, value in sections[name].items()], - f"the {GITMODULES} section git rm took out, key by key as " - "HEAD has it, in the file a line above put back")] + on_disk = os.path.lexists(root / GITMODULES) + if on_disk and any(keys.get("path") == path for keys in worktree.values()): + return [] + if report.put_back and path in _blob_paths(root, report.put_back): + return [] + wanted, refused = asked + if report.wrote or report.put_back: + report.wrote = True + return _keys_back(path, sections, on_disk, refused) + if on_disk: + return [([f"git checkout HEAD -- {GITMODULES}"], SECTION_TAKEN)] spec, line, whose = _gitmodules_put_back(root) - if path in _blob_paths(root, spec): + if wanted <= _blob_paths(root, spec): report.put_back = spec return [([line], f"the {GITMODULES} gone from the work tree, from " f"{whose}")] @@ -2988,6 +3107,34 @@ def _section_back(root: Path, path: str, return [([f"git checkout HEAD -- {GITMODULES}"], SECTION_TAKEN)] +def _keys_back(path: str, sections: dict[str, dict[str, str]], + on_disk: bool, refused: set[tuple[str, str]] + ) -> list[tuple[list[str], str]]: + """`_section_back`'s lines after a finding that wrote the work tree's + `.gitmodules` or put it back: the section of HEAD's (`sections`) that + gives `path`, the last, as git reads it, written in key by key as HEAD + has it, into the work tree's file as the lines above leave it + (`on_disk`) or the file a line above put back, but for a key HEAD's + gives a value git dies on (`refused`), which written back would be + `gitmodules-value-refused` again; then `git add -- .gitmodules` + (`STAGE_LINE`): the commit records the index's file, and a section + written into the work tree's alone stayed out of it (lane 1's MUST 1 + on #889).""" + name = [name for name, keys in sections.items() + if keys.get("path") == path][-1] + into = ("in the work tree's file, as the lines above leave it" if on_disk + else "in the file a line above put back") + return [([f"git config -f {GITMODULES} " + f"{shlex.quote(f'submodule.{name}.{key}')} " + f"{shlex.quote(value)}" + for key, value in sections[name].items() + if (name, key) not in refused], + f"the {GITMODULES} section git rm took out, key by key as HEAD " + f"has it, {into}"), + ([STAGE_LINE], f"that section in the index, staging {GITMODULES} " + f"whole, {STAGE_TOO}")] + + def _blob_paths(root: Path, spec: str) -> set[str]: """Every path a section of the `.gitmodules` git names `spec` gives (`:.gitmodules`, the index's, or `HEAD:.gitmodules`), read as @@ -3018,18 +3165,27 @@ def _put_back_lines(root: Path, path: str, return lines +#: What `member-gitlink-removal-staged` says after its lines where they +#: stage `.gitmodules` (`_keys_back`), in place of "and nothing is left to +#: commit": the file holds what the findings above write (lane 1's MUST 1 +#: on #889). +STAGED_BACK = (f": commit {GITMODULES} then, with what the lines above " + "write in it, and nothing else is left to commit.") + + def _removal_staged_said(root: Path, row: dict, project: str, path: str, commit: str, lines: list[tuple[list[str], str]], - moved: list[str]) -> str: + found: tuple[list[str], list[str]]) -> str: """`member-gitlink-removal-staged`'s text for the row naming `project`, whose gitlink at `path` is at `commit` in HEAD, with `lines` - (`_put_back_lines`) below it (#762), and what it says of a checkout of - the member that `git mv` moved to a case variant of `path`, each of - `moved` (`_moved_said`, #879's C4).""" + (`_put_back_lines`) below it (#762), and what it says of the checkouts + `_moved_checkouts` `found` (`_moved_said`, #879's C4). Where the lines + stage `.gitmodules`, it says to commit that (`STAGED_BACK`).""" member_id = str(row.get("id") or project) remove = (f"{PYTHON} scripts/family.py remove --family-root " f"{shlex.quote(str(root))} --member {shlex.quote(project)}") commands = "\n ".join(line for group, _ in lines for line in group) + staged = any(group == [STAGE_LINE] for group, _ in lines) return ( f"{path} is a gitlink at {commit} that a row of members: names and " "HEAD records, and the index holds nothing at that path: its removal " @@ -3037,7 +3193,8 @@ def _removal_staged_said(root: Path, row: dict, project: str, path: str, "member under its row, and this gate then refuses " "member-gitlink-absent. If the member is to stay, " + _put_back_said(lines) - + ", and nothing is left to commit." + _moved_said(path, moved) + + (STAGED_BACK if staged else ", and nothing is left to commit.") + + _moved_said(path, *found) + " If it is to go, put it back the " "same way, then take it out with family.py remove, from a checkout " "of openRepoShape, which takes its row, its gitlink and its section " @@ -3047,29 +3204,66 @@ def _removal_staged_said(root: Path, row: dict, project: str, path: str, f"`{remove}`. {BRANCH_LEFT}\n {commands}") -def _moved_said(path: str, moved: list[str]) -> str: +def _moved_said(path: str, moved: list[str], clones: list[str]) -> str: """What `member-gitlink-removal-staged` says, before its last line, the - update, of each checkout of the member that `git mv` moved to a case - variant of `path`, or nothing (#879's C4): after the lines were typed - as printed it stayed there, untracked and named nowhere, so `family.py - add` refused it (`family-members-dir-variant`), and after the update - `git -C status` read `path`, so a file only it held went - unseen. Said as `family.py add`'s closing sentences say it (#828, - #865).""" - if not moved: - return "" - there = " or ".join(moved) - status = " or ".join(f"`git -C {shlex.quote(variant)} status`" - for variant in moved) - return (f" The checkout git mv moved to {there} stays there, untracked, " - "and family.py add refuses it as family-members-dir-variant: its " - ".git file names the member's store under this holder's " - ".git/modules/ by a path relative to it, and its commits are kept " - "there whatever becomes of the checkout, so only what is not " - f"committed is in the checkout alone, which {status} shows. " - "Delete it by hand once nothing only it holds is needed, before " - f"the last line: that update points the store at {path}, after " - f"which {status} reads {path} instead.") + update, of each checkout of the member `git mv` moved off `path` + (`moved`) and each clone of its own at a case variant of it + (`clones`), or nothing (`_moved_checkouts`). + + THE CHECKOUT `git mv` MOVED (#879's C4): after the lines were typed as + printed it stayed there, untracked and named nowhere, so `family.py + add` refused one at a case variant (`family-members-dir-variant`), and + after the update `git -C status` read `path`, so a file only + it held went unseen. Said as `family.py add`'s closing sentences say + it (#828, #865), and of one outside `members/` too (lane 1's SHOULD 3 + on #889). + + A CLONE OF ITS OWN, whose `.git` is a directory, holds its commits + there and nowhere else: said as the wrong-case move says it + (`_wrong_case_move`), to move it out whole, not to delete it (lane 1's + SHOULD 2 on #889).""" + said = "" + if moved: + refused = (", and family.py add refuses it as " + "family-members-dir-variant" + if any(other.casefold() == path.casefold() + for other in moved) else "") + status = _statuses(moved) + said += ( + f" The checkout git mv moved to {' or '.join(moved)} stays " + f"there, untracked{refused}: its .git file names the member's " + "store under this holder's .git/modules/ by a path relative to " + "it, and its commits are kept there whatever becomes of the " + "checkout, so only what is not committed is in the checkout " + f"alone, which {status} shows. Delete it by hand once nothing " + "only it holds is needed, before the last line: that update " + f"points the store at {path}, after which {status} reads {path} " + "instead.") + if clones: + said += ( + f" The clone at {' or '.join(clones)}, whose .git is a " + "directory, stays there, untracked, and family.py add refuses it " + "as family-members-dir-variant: it holds its commits there and " + f"nowhere else, and {_unpushed(clones)} lists the ones not " + "pushed, so move it out of the holder whole, by hand, rather " + "than delete it.") + return said + + +def _statuses(paths: list[str]) -> str: + """`git -C status`, backticked, for each of `paths`, joined by + `or`: what shows what only a checkout holds (#681, #879).""" + return " or ".join(f"`git -C {shlex.quote(path)} status`" + for path in paths) + + +def _unpushed(paths: list[str]) -> str: + """`git -C log --oneline HEAD --branches --not --remotes`, + backticked, for each of `paths`, joined by `or`: what lists the + commits a clone of its own holds that no remote does (rv681's S2, + #879).""" + return " or ".join(f"`git -C {shlex.quote(path)} log --oneline HEAD " + "--branches --not --remotes`" for path in paths) #: How `_put_back_said` counts the lines below a finding. diff --git a/tests/test_family_holder_remedies_compose.py b/tests/test_family_holder_remedies_compose.py index 9597d89b..53eaca2f 100644 --- a/tests/test_family_holder_remedies_compose.py +++ b/tests/test_family_holder_remedies_compose.py @@ -54,6 +54,17 @@ none over an empty directory, which git takes the plain line over (NIT 2). NIT 3's clause is in `STAGE_SAID`, which the C3 rows pin. +LANE 1'S REVIEW OF #889, ROUND 1, TAKEN HERE: a staged removal's section +written back key by key is staged too, and the first removal puts back +HEAD's file where the index's lacks a section a second removal needs +(MUST 1, its P4 and P5); no removal puts HEAD's file back over an earlier +finding's write with the file on disk either (SHOULD 1); a clone of its +own at a case variant is said to be one (SHOULD 2); a checkout `git mv` +moved out of `members/` is named (SHOULD 3); a symbolic link to an empty +directory takes `-f` (NIT 2); the move says what is on disk at each +variant (NIT 3); a run with no row says nothing was updated (NIT 5); and +the value finding's put-back serves the removal after it (NIT 6). + THE SAME FAMILY, OFFLINE RULES AND HELPERS as `tests/test_family_holder_bootstrap_texts.py` (#839) and `tests/test_family_unnamed_gitlink.py` (#878). A case variant is an index @@ -66,6 +77,7 @@ import re import shlex +import shutil from pathlib import Path import pytest @@ -90,9 +102,10 @@ from test_family_member_path import MEMBER, OTHER, raw from test_family_remove_gitmodules import verb_args from test_family_unnamed_gitlink import ( - COMMIT_THAT, FORCED_NO_HEAD, STAGE_LINE, STAGE_SAID, TAKE_OUT, TOOLS, - assert_a_plain_clone_checks_out_every_member, at_tools, + COMMIT_THAT, FORCED_NO_HEAD, RENAMED, STAGE_LINE, STAGE_SAID, TAKE_OUT, + TOOLS, assert_a_plain_clone_checks_out_every_member, at_tools, found_in_every_mode, outside, section_taken_out_alone, staged_at, + submodule_added, ) from test_validate_family_typed_states import ( GONE, ROW, SECTION_GIVEN_BACK, URL_ABSENT, VARIANT_PATH, gone, @@ -325,6 +338,25 @@ def url_of(root: Path, env: dict, path: str) -> str: f"submodule.{path}.url").stdout.decode("utf-8").strip() +#: What `member-gitlink-removal-staged` says its stage line does, and what +#: it says after its lines where they stage `.gitmodules` (lane 1's MUST 1 +#: on #889). +STAGED_WHOLE = ("that section in the index, staging .gitmodules whole, which " + "stages every other change in it too") +STAGED_BACK = (": commit .gitmodules then, with what the lines above write " + "in it, and nothing else is left to commit.") + + +def written_back(root: Path, env: dict, path: str) -> list[str]: + """The removal's lines that write the section of HEAD's `.gitmodules` + named `path` back, key by key as HEAD has it, then stage the file (lane + 1's MUST 1 on #889). Read before the state is made.""" + return [f"git config -f .gitmodules submodule.{path}.path {path}", + f"git config -f .gitmodules submodule.{path}.url " + f"{shlex.quote(url_of(root, env, path))}", + "git add -- .gitmodules"] + + def test_c2_a_section_git_rm_took_is_written_back_into_the_file_put_back( family, holder, offline, tmp_path): """C2's other half, found with it: a plain `git rm members/IRRS`, which @@ -333,13 +365,14 @@ def test_c2_a_section_git_rm_took_is_written_back_into_the_file_put_back( deleted. The absent finding puts back the index's, which lacks MEMBER's section, and the removal's `git checkout HEAD -- .gitmodules` took OTHER's out again. Now the removal writes MEMBER's section back, key by - key as HEAD has it, into the file put back, before the update, which - dies `No url found` without it. Typed in order and committed, the gate - passes and a plain clone checks both out.""" + key as HEAD has it, into the file put back, and stages the file (lane + 1's MUST 1 on #889), before the update, which dies `No url found` + without it. Typed in order and committed, the gate passes and a plain + clone checks both out.""" env = cloning(offline, tmp_path, family) data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout committed_gitmodules(holder, env, section_taken_out(data, OTHER)) - member_url = url_of(holder, env, ROW) + lines = written_back(holder, env, ROW) raw(holder, env, "rm", "-q", "--", ROW) gone(holder, env, "deleted") @@ -349,15 +382,12 @@ def test_c2_a_section_git_rm_took_is_written_back_into_the_file_put_back( "member-gitlink-removal-staged"] removal = findings(result)[1] assert commands(removal) == [ - f"git reset -q -- {ROW}", - f"git config -f .gitmodules submodule.{ROW}.path {ROW}", - f"git config -f .gitmodules submodule.{ROW}.url " - f"{shlex.quote(member_url)}", + f"git reset -q -- {ROW}", *lines, f"git submodule update --init --recursive -- {ROW}"] assert ("the first its gitlink, the second to the third the .gitmodules " "section git rm took out, key by key as HEAD has it, in the file " - "a line above put back, the fourth its checkout, and nothing is " - "left to commit.") in removal, removal + f"a line above put back, the fourth {STAGED_WHOLE}, the fifth its " + "checkout" + STAGED_BACK) in removal, removal typed_in_order(holder, env, result) staged_and_committed(holder, env, "Typed in order") ends_well(holder, env, family, tmp_path) @@ -423,13 +453,19 @@ def test_c2_control_with_the_file_on_disk_the_removal_is_as_it_was( # --- C3: a git rm of a gitlink after a value finding's lines ---------------- -def value_committed(root: Path, env: dict) -> None: - """MEMBER's section given `update = None`, committed (#768).""" - data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout - line = f"\tpath = {ROW}\n".encode() +def valued(data: bytes, path: str) -> bytes: + """`.gitmodules` bytes with the section giving `path` given `update = + None` too, a value git dies on (#768).""" + line = f"\tpath = {path}\n".encode() assert data.count(line) == 1, data - committed_gitmodules(root, env, - data.replace(line, line + b"\tupdate = None\n")) + return data.replace(line, line + b"\tupdate = None\n") + + +def value_committed(root: Path, env: dict, path: str = ROW) -> None: + """The section giving `path`, MEMBER's unless said, given `update = + None`, committed (#768).""" + committed_gitmodules(root, env, valued( + raw(root, env, "show", f"HEAD:{GITMODULES}").stdout, path)) def staged_first(text: str) -> str: @@ -536,8 +572,8 @@ def test_c3_the_wrong_case_move_s_git_rm_stages_the_file_first_after_a_value( #: What `member-gitlink-removal-staged` says of the checkout `git mv` moved #: to `Members/IRRS`, before its update (C4). MOVED = ( - " The checkout git mv moved to {variant} stays there, untracked, and " - "family.py add refuses it as family-members-dir-variant: its .git file " + " The checkout git mv moved to {variant} stays there, untracked{refused}: " + "its .git file " "names the member's store under this holder's .git/modules/ by a path " "relative to it, and its commits are kept there whatever becomes of the " "checkout, so only what is not committed is in the checkout alone, which " @@ -545,6 +581,10 @@ def test_c3_the_wrong_case_move_s_git_rm_stages_the_file_first_after_a_value( "holds is needed, before the last line: that update points the store at " "{path}, after which `git -C {variant} status` reads {path} instead.") +#: What it says of one at a case variant: `family.py add` refuses it. +REFUSED_AS_VARIANT = (", and family.py add refuses it as " + "family-members-dir-variant") + #: The `git mv` states: nothing more, and #876's two. MOVES = {"clean": lambda checkout, env: None, **WORK} @@ -572,7 +612,8 @@ def test_c4_the_checkout_git_mv_moved_is_named_before_the_update( found = found_alike(holder, *MODES) assert ("and nothing is left to commit." - + MOVED.format(variant=VARIANT_PATH, path=ROW) + + MOVED.format(variant=VARIANT_PATH, path=ROW, + refused=REFUSED_AS_VARIANT) + " If it is to go,") in found[-1], found[-1] lines = [line for finding in found for line in commands(finding)] assert lines[-1] == f"git submodule update --init --recursive -- {ROW}" @@ -808,15 +849,31 @@ def an_empty_directory_at(root: Path, path: str) -> None: (root / path).mkdir(parents=True) +def a_link_to_an_empty_directory_at(root: Path, path: str) -> None: + """A symbolic link at `path` to an empty directory outside the holder + (lane 1's NIT 2 on #889), or a skip where none can be made.""" + target = root.parent / "an-empty-directory" + target.mkdir() + (root / path).parent.mkdir(parents=True, exist_ok=True) + try: + (root / path).symlink_to(target, target_is_directory=True) + except OSError as exc: + pytest.skip(f"no symbolic link can be made here: {exc}") + + #: `(what is on disk at tools/x, the flag its git rm --cached carries, what #: the text says of it)`: git 2.43 refused the plain line over the first -#: two and took it over the third (rv882's NIT 2). +#: two (rv882's NIT 2) and the fourth (`use -f to force removal`, exit 1, +#: lane 1's NIT 2 on #889), and took it over the third. ON_DISK = { "a-directory-holding-a-file": (files_at, " -f", FORCED_NO_HEAD.replace( "over a checkout", f"over {OVER_FILES}")), "a-file": (a_file_at, " -f", FORCED_NO_HEAD.replace( "over a checkout", f"over {OVER_FILES}")), "an-empty-directory": (an_empty_directory_at, "", COMMIT_THAT), + "a-link-to-an-empty-directory": ( + a_link_to_an_empty_directory_at, " -f", FORCED_NO_HEAD.replace( + "over a checkout", f"over {OVER_FILES}")), } @@ -873,3 +930,391 @@ def test_rv882_nit_2_the_move_takes_f_over_what_is_no_checkout( commit_all(holder, env, "Moved as printed") assert (holder / VARIANT_PATH / "kept.txt").read_bytes() == b"kept\n" assert validate(holder).returncode == 0 + + +# --- lane 1's review of #889, round 1 --------------------------------------- + +OTHER_ROW = f"members/{OTHER}" + + +def test_round1_must_1_p4_the_first_removal_puts_back_head_s_file( + family, holder, offline, tmp_path): + """Lane 1's MUST 1 on #889, its P4: `git rm --cached members/IRRS`, + `git rm members/IRSS`, and the work tree's `.gitmodules` deleted. The + first removal put back the index's file, which the second `git rm` had + taken IRSS's section out of, and the second wrote that section into + the work tree's file alone: `MM .gitmodules`, and a plain commit + dropped it from HEAD. The first puts back HEAD's now, since the + index's lacks a section another staged removal needs, and the second + prints no line for the file, as #876's head printed. Typed in order, + nothing is left to commit, and the gate passes in every mode.""" + env = cloning(offline, tmp_path, family) + raw(holder, env, "rm", "-q", "--cached", "--", ROW) + raw(holder, env, "rm", "-q", "--", OTHER_ROW) + gone(holder, env, "deleted") + + found = found_alike(holder, *MODES) + + assert [commands(finding) for finding in found] == [ + [f"git reset -q -- {ROW}", "git checkout HEAD -- .gitmodules"], + [f"git reset -q -- {OTHER_ROW}", + f"git submodule update --init --recursive -- {OTHER_ROW}"]] + typed_in_order(holder, env, validate(holder, "--no-members")) + assert raw(holder, env, "status", "--porcelain").stdout == b"" + ends_well(holder, env, family, tmp_path) + + +@pytest.mark.parametrize("how", ["deleted", "on-disk"]) +def test_round1_must_1_p5_the_section_written_back_is_staged( + family, holder, offline, tmp_path, how): + """Lane 1's MUST 1, its P5: MEMBER recorded only at `Members/IRRS` + (`moved_in_head`), then `git rm members/IRSS`, under `--no-members`. + Where the work tree's `.gitmodules` was gone, the move put back the + index's, which lacks IRSS's section, and the removal wrote that + section into the work tree's file alone, so the commit the move names + dropped it; on disk, the removal's `git checkout HEAD -- .gitmodules` + took the move's write out. Either way it writes IRSS's section back + key by key and stages the file now. Typed in order and committed with + a plain `git commit`, as the move says, nothing is left, HEAD's + `.gitmodules` gives both members, and the gate passes in every mode.""" + env = cloning(offline, tmp_path, family) + moved_in_head(holder, env) + lines = written_back(holder, env, OTHER_ROW) + raw(holder, env, "rm", "-q", "--", OTHER_ROW) + if how == "deleted": + gone(holder, env, how) + + result = validate(holder, "--no-members") + + assert codes(result)[0] == "member-gitlink-unnamed" + assert codes(result)[-1] == "member-gitlink-removal-staged" + assert commands(findings(result)[-1]) == [ + f"git reset -q -- {OTHER_ROW}", *lines, + f"git submodule update --init --recursive -- {OTHER_ROW}"] + typed_in_order(holder, env, result) + commit_all(holder, env, "Committed as the move says") + assert raw(holder, env, "status", "--porcelain").stdout == b"" + assert head_paths(holder, env) == {ROW, OTHER_ROW} + ends_well(holder, env, family, tmp_path) + + +def stale_section(root: Path, env: dict, family) -> None: + """A section for `members/IRQS`, which no row names and no gitlink is + at, committed in HEAD's `.gitmodules`: one `family.py remove` left.""" + data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(root, env, data + ( + f'[submodule "members/{SPARE}"]\n\tpath = members/{SPARE}\n' + f"\turl = https://github.com/InkRouter/{SPARE}.git\n").encode()) + + +def mounted_with_its_section(root: Path, env: dict, family) -> None: + """A member mounted by hand at `members/IRQS` with `git submodule + add`, its section with it, committed, and no row.""" + submodule_added(root, env, family, f"members/{SPARE}") + commit_all(root, env, "A member mounted by hand with its section") + + +def head_edited(edit): + """A state made by `edit` on HEAD's `.gitmodules` bytes, committed.""" + def made(root: Path, env: dict, family) -> None: + data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(root, env, edit(data)) + return made + + +def value_staged_out(path: str): + """A state in which `update = None` is committed in the section giving + `path`, and the file without it is staged: git rm dies reading a work + tree's file that gives such a value (`invalid value for + 'submodule..update'`, exit 128), so the person mended it first.""" + def made(root: Path, env: dict, family) -> None: + data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(root, env, valued(data, path)) + (root / GITMODULES).write_bytes(data) + raw(root, env, "add", "--", GITMODULES) + return made + + +#: `(the state before the removal, the code of the finding it makes)`: each +#: finding's lines write the work tree's `.gitmodules`, or it says to +#: commit the file as it stands, and the removal's `git checkout HEAD -- +#: .gitmodules` put HEAD's back over that (lane 1's SHOULD 1 on #889). The +#: value in MEMBER's own section is one a section written back key by key +#: must leave out. +EARLIER_WRITES = { + "section-absent": (head_edited( + lambda data: section_taken_out(data, OTHER)), + "member-gitmodules-absent"), + "url-absent": (head_edited(lambda data: without_url(data, OTHER)), + "member-gitmodules-url-absent"), + "value-in-another-section": (value_staged_out(OTHER_ROW), + "gitmodules-value-refused"), + "value-in-its-own-section": (value_staged_out(ROW), + "gitmodules-value-refused"), + "a-section-no-row-names": (stale_section, "gitmodules-path-unnamed"), + "a-hand-mount-with-its-section": (mounted_with_its_section, + "member-gitlink-unnamed"), +} + + +@pytest.mark.parametrize("make, code", EARLIER_WRITES.values(), + ids=EARLIER_WRITES) +def test_round1_should_1_no_put_back_over_an_earlier_write_on_disk( + family, holder, offline, tmp_path, make, code): + """Lane 1's SHOULD 1 on #889: a finding whose lines write the work + tree's `.gitmodules`, or that says to commit it as it stands, then a + plain `git rm members/IRRS`, with the file on disk. The removal's `git + checkout HEAD -- .gitmodules` put HEAD's file back over that, so the + commit after it had nothing to commit and the earlier finding came + back. It writes MEMBER's section + back key by key now, leaving out a value git dies on, and stages the + file, in every mode. Typed in order and committed, the gate passes in + every mode and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + lines = written_back(holder, env, ROW) + make(holder, env, family) + raw(holder, env, "rm", "-q", "--", ROW) + + found = found_alike(holder, *MODES) + + assert f"FINDING {code}: " in found[0], found[0] + assert found[-1].startswith("FINDING member-gitlink-removal-staged: ") + assert commands(found[-1]) == [ + f"git reset -q -- {ROW}", *lines, + f"git submodule update --init --recursive -- {ROW}"] + assert ("the second to the third the .gitmodules section git rm took " + "out, key by key as HEAD has it, in the work tree's file, as the " + f"lines above leave it, the fourth {STAGED_WHOLE}, the fifth its " + "checkout" + STAGED_BACK) in found[-1], found[-1] + typed_in_order(holder, env, validate(holder, "--no-members")) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) + + +def test_round1_should_1_a_second_section_taken_out_by_hand_stays_out( + family, holder, offline, tmp_path): + """Lane 1's SHOULD 1, beside `gitmodules-path-twice`, whose remedy is a + by-hand `--unset-all` of each key of the section that is not the + member's: a second section giving IRSS's path committed, then a plain + `git rm members/IRRS`. The removal's `git checkout HEAD -- + .gitmodules` put the second section back. It writes MEMBER's section + back key by key now and stages the file. The second section taken out + by hand as that finding says, then the removal's lines, as printed + before, typed and committed, the gate passes in every mode.""" + env = cloning(offline, tmp_path, family) + lines = written_back(holder, env, ROW) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, data + ( + f'[submodule "second"]\n\tpath = {OTHER_ROW}\n' + "\turl = https://github.com/InkRouter/Second.git\n").encode()) + raw(holder, env, "rm", "-q", "--", ROW) + + found = found_alike(holder, *MODES) + + assert [finding.split(":", 1)[0] for finding in found] == [ + "FINDING gitmodules-path-twice", + "FINDING member-gitlink-removal-staged"] + assert commands(found[1]) == [ + f"git reset -q -- {ROW}", *lines, + f"git submodule update --init --recursive -- {ROW}"] + for key in ("path", "url"): + raw(holder, env, "config", "-f", GITMODULES, "--unset-all", + f"submodule.second.{key}") + typed(holder, env, found[1]) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) + + +#: What `member-gitlink-removal-staged` says of a clone of its own at a case +#: variant of the row's path (lane 1's SHOULD 2 on #889), as the wrong-case +#: move says it. +CLONE_KEPT = ( + " The clone at {variant}, whose .git is a directory, stays there, " + "untracked, and family.py add refuses it as family-members-dir-variant: " + "it holds its commits there and nowhere else, and `git -C {variant} log " + "--oneline HEAD --branches --not --remotes` lists the ones not pushed, " + "so move it out of the holder whole, by hand, rather than delete it.") + + +def test_round1_should_2_a_clone_at_a_case_variant_is_moved_out_whole( + family, holder, offline, tmp_path, case_sensitive_disk): + """Lane 1's SHOULD 2 on #889: `git rm --cached members/IRRS`, its + checkout deleted, IRRS cloned at `Members/IRRS` with a commit only that + clone holds, and `git add Members/IRRS`. The removal's finding said + the checkout `git mv` moved was there, its commits kept in the + member's store, to delete by hand: no `git mv` happened, its `.git` is + a directory, and deleted, the commit was lost. It says what a clone + of its own is now, in every mode, with the `git log` that lists that + commit, and to move it out whole. Typed in order, the clone moved out + of the holder as it says, nothing is left, the moved clone keeps the + commit, the gate passes in every mode and a plain clone checks both + out.""" + env = cloning(offline, tmp_path, family) + raw(holder, env, "rm", "-q", "--cached", "--", ROW) + rmtree(holder / ROW) + (holder / "Members").mkdir() + raw(holder, env, "clone", "-q", "--", + str(family["remotes"] / f"{MEMBER}.git"), VARIANT_PATH) + clone = holder / VARIANT_PATH + raw(clone, env, "-c", "user.name=t", "-c", "user.email=t@t", "commit", + "-q", "--allow-empty", "-m", "Only this clone holds it") + made = raw(clone, env, "rev-parse", "HEAD").stdout.decode("ascii").strip() + raw(holder, env, "add", "--", VARIANT_PATH) + + found = found_alike(holder, *MODES) + + assert "The checkout git mv moved" not in found[-1] + assert CLONE_KEPT.format(variant=VARIANT_PATH) in found[-1], found[-1] + for finding in found: + typed(holder, env, finding) + listed = raw(holder, env, "-C", VARIANT_PATH, "log", "--oneline", "HEAD", + "--branches", "--not", "--remotes").stdout.decode().split() + assert made.startswith(listed[0]), listed + kept = tmp_path / "kept-out" + shutil.move(str(clone), str(kept)) + assert raw(holder, env, "status", "--porcelain").stdout == b"" + assert raw(kept, env, "cat-file", "-e", made, + check=False).returncode == 0 + ends_well(holder, env, family, tmp_path) + + +def test_round1_should_3_a_checkout_git_mv_moved_out_of_members_is_named( + family, holder, offline, tmp_path): + """Lane 1's SHOULD 3 on #889: `git mv members/IRRS renamed-1`, staged, + with an untracked file in the checkout it moved. Typed as printed, + `renamed-1/` was left untracked and named by no finding, after the + update its status read members/IRRS, so the file went unseen, and the + gate passed. The removal's finding names it now, in every mode, as C4 + names one at a case variant, with no `family.py add` refusal, which + only a variant meets; its section is written back key by key, after + the unnamed section's lines took it out. Typed in order but the last + line, its status shows the file; deleted by hand as the text says, + then the update typed and the file committed, nothing is left, the + gate passes in every mode and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + lines = written_back(holder, env, ROW) + raw(holder, env, "mv", "--", ROW, RENAMED) + (holder / RENAMED / "scratch.txt").write_bytes(b"mine\n") + + found = found_alike(holder, *MODES) + + assert codes_of(found) == ["gitmodules-path-unnamed", + "member-gitlink-removal-staged"] + assert (MOVED.format(variant=RENAMED, path=ROW, refused="") + + " If it is to go,") in found[-1], found[-1] + assert commands(found[-1]) == [ + f"git reset -q -- {ROW}", *lines, + f"git submodule update --init --recursive -- {ROW}"] + typed_lines = [line for finding in found for line in commands(finding)] + for line in typed_lines[:-1]: + type_line(holder, env, line) + status = raw(holder, env, "-C", RENAMED, "status", "--porcelain").stdout + assert b"?? scratch.txt\n" in status, status + rmtree(holder / RENAMED) + type_line(holder, env, typed_lines[-1]) + staged_and_committed(holder, env, "Typed in order") + assert raw(holder, env, "status", "--porcelain").stdout == b"" + ends_well(holder, env, family, tmp_path) + + +def codes_of(found: list[str]) -> list[str]: + """The code of each of `found`, the findings as printed.""" + return [finding.split(":", 1)[0].removeprefix("FINDING ") + for finding in found] + + +def test_round1_nit_3_the_move_says_what_is_on_disk_at_each_variant( + family, holder, offline, tmp_path, case_sensitive_disk): + """Lane 1's NIT 3 on #889: MEMBER recorded only at `MEMBERS/IRRS` and + `Members/IRRS`, both then staged at another commit, a directory + holding a file at the one and a checkout at the other. The `-f` + sentence said of both "what is on disk there, which is no checkout" + once one was no checkout. It says each by its path now. Typed as + printed and committed, the lines exit 0, and the gate passes in + members mode.""" + env = cloning(offline, tmp_path, family) + variants = ["MEMBERS/IRRS", VARIANT_PATH] + commit = recorded_at(holder, env, MEMBER, *variants) + other = spare_commits(family)[1] + for variant in variants: + stage_gitlink(holder, variant, other) + files_at(holder, variants[0]) + raw(holder, env, "init", "-q", "--", variants[1]) + + refused = validate(holder) + + assert refused.returncode == 2, refused.stdout + refused.stderr + assert refused.stderr.splitlines() == [ + line.replace("over a checkout", f"over what is on disk at " + f"{variants[0]}, which is no checkout, and over a " + f"checkout at {variants[1]}") + for line in variant_refused(ROW, commit, variants, + dict.fromkeys(variants, other), + forced=tuple(variants))] + for line in refusal_lines(refused): + type_line(holder, env, line) + commit_all(holder, env, "Moved as printed") + assert validate(holder).returncode == 0 + + +#: What (a) says where no row is declared and git updated nothing (lane 1's +#: NIT 5 on #889). +NOTHING_UPDATED = NONE_UPDATED.replace("NO MEMBER UPDATED", "NOTHING UPDATED") + + +def no_rows(root: Path, env: dict) -> None: + """`members: []` in `family.yaml`, staged: the rows are its last key, + rewritten wholesale by `family.py`.""" + manifest = root / "family.yaml" + text = manifest.read_text(encoding="utf-8") + manifest.write_text(text[:text.index("\nmembers:")] + "\nmembers: []\n", + encoding="utf-8") + raw(root, env, "add", "--", "family.yaml") + + +def test_round1_nit_5_with_no_rows_nothing_updated_is_said( + family, holder, offline, tmp_path): + """Lane 1's NIT 5 on #889: `members: []`, and a gitlink at vendor/IRQS + that no section gives, committed. Git's init stops there, so the run + fails, as it should, where #876's head said ok; but its line said "NO + MEMBER UPDATED" under "0 member(s)". It says nothing was updated + now.""" + env = cloning(offline, tmp_path, family) + no_rows(holder, env) + path = outside_gitlink(holder, env, family) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stdout.splitlines()[0].endswith(" 0 member(s)") + assert after_fetch_line(result) == [STOP_FAILED.format(path=path)] + assert NOTHING_UPDATED.format( + why=STOPPED.format(path=path), + cure=OUTSIDE_CURE.format(path=path)) in result.stderr.splitlines() + + +def test_round1_nit_6_the_value_s_put_back_serves_the_removal_after_it( + family, holder, offline, tmp_path): + """Lane 1's NIT 6 on #889 (its mutant MX1): `update = None` committed + in IRSS's section, `git rm --cached members/IRRS`, and the work tree's + `.gitmodules` deleted. The value finding puts back HEAD's file and + records that, so the removal after it prints no line for the file, + which HEAD's gives MEMBER's section in. Put back again from the + index's, which holds the value, it undid the fix, and the commit died + (`invalid value for 'submodule.members/IRSS.update'`, exit 128). + Typed in order and committed, the gate passes in every mode and a + plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + value_committed(holder, env, OTHER_ROW) + raw(holder, env, "rm", "-q", "--cached", "--", ROW) + gone(holder, env, "deleted") + + found = found_alike(holder, *MODES) + + assert codes_of(found) == ["gitmodules-value-refused", + "member-gitlink-removal-staged"] + assert commands(found[0])[0] == "git checkout HEAD -- .gitmodules" + assert commands(found[1]) == [f"git reset -q -- {ROW}"] + typed_in_order(holder, env, validate(holder, "--no-members")) + staged_and_committed(holder, env, "Typed in order") + ends_well(holder, env, family, tmp_path) From 29083a5f2d9f812b8640790d2dba6cb9c9301193 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 14:11:33 +0000 Subject: [PATCH 17/20] validate-family.py: a staged removal writes no section into a `.gitmodules` nothing put back after a hand mount's `git rm`, and a hand mount's text says its `git rm` refuses a gitlink staged at a commit HEAD does not record there (#879) Fix round 2 for #889: lane 1's re-review of 3b15de03 (issuecomment-6250664740), every item taken. - MUST 2: `_section_back` asks `Report.put_back`, and `Report.wrote` only with the work tree's `.gitmodules` on disk. With the file gone, only a put-back makes a file to write into (C1), and a hand mount's `git rm` writes into none, so after one the removal puts the file back itself, as round 0 did, and no longer commits a file of one section. - SHOULD, the sentence only: where the index stages a hand mount's gitlink at a commit HEAD does not record there and anything is on disk at it (`_staged_over`; `_head_records`, shared with `_forced`), its text says the `git rm` refuses that too (`STAGED_AT`). No line takes such a gitlink out: `-f` would take the checkout's changes with it. - NIT: a row pins `_moved_checkouts`' case-variant test for clones (lane 1's mutant MY3). Tests: 7 new rows in tests/test_family_holder_remedies_compose.py. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- .../family-root/scripts/validate-family.py | 73 ++++++-- tests/test_family_holder_remedies_compose.py | 162 ++++++++++++++++++ 2 files changed, 220 insertions(+), 15 deletions(-) diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 3e815af9..85a1630e 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -854,13 +854,33 @@ def _forced(root: Path, path: str, commit: str) -> str | None: read at HEAD's commit and takes none.""" if not _on_disk(root, path): return None + head = _head_records(root, path) + return None if head == commit else head + + +def _head_records(root: Path, path: str) -> str: + """The commit HEAD records at `path`, lower case, or `""` for nothing + (`_forced`, `_staged_over`).""" with _git_runs(f"the gitlink HEAD records at {path} could not be read"): try: - head = git_out(["rev-parse", "--verify", "--quiet", + return git_out(["rev-parse", "--verify", "--quiet", f"HEAD:{path}"], cwd=root).lower() except Refusal: - head = "" - return None if head == commit else head + return "" + + +def _staged_over(root: Path, path: str, commit: str) -> bool: + """Whether a plain `git rm -- ` refuses the gitlink at `path`, + which `_member_gitlinks` reads at `commit`, as staged (`STAGED_AT`): + the index stages it at a commit HEAD does not record there, and + anything is on disk at it, an empty directory too, which `git rm + --cached` takes with no `-f` (`_forced`). Measured on git 2.43, with + the gitlink staged and not committed: nothing on disk, it took it; an + empty directory, and the checkout `git mv members/

members/

2` + moved, staged, it refused (`the following file has changes staged in + the index`, exit 1; lane 1's SHOULD on #889, round 2).""" + return (os.path.lexists(root / path) + and _head_records(root, path) != commit) def _on_disk(root: Path, path: str) -> bool: @@ -2253,8 +2273,9 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], continue report.finding( "member-gitlink-unnamed", - f"{said} " + _mounted_by_hand(root, path, _hand_project(path), - path in removed, stage)) + f"{said} " + _mounted_by_hand( + root, path, _hand_project(path), path in removed, stage, + _staged_over(root, path, gitlinks[path]))) report.wrote = report.wrote or path not in removed @@ -2497,8 +2518,17 @@ def _given_as_git_reads(root: Path) -> set[str]: return _blob_paths(root, spec) if spec else set() +#: What a hand mount's text says its `git rm` meets where the index stages +#: the gitlink at a commit HEAD does not record there (`_mounted_by_hand`, +#: lane 1's SHOULD on #889, round 2), measured on git 2.43. +STAGED_AT = (", and while the index stages it at a commit HEAD does not " + "record there, as it does here (the following file has changes " + "staged in the index, exit 1),") + + def _mounted_by_hand(root: Path, path: str, project: str | None, - staged: bool, stage_first: bool = False) -> str: + staged: bool, stage_first: bool = False, + staged_at: bool = False) -> str: """What a gitlink no row names, in no case, says to do after what it is: put its row back, or take a hand mount out, commit that, and mount it with `family.py add`, on a branch (`_check_gitlinks_named`). Where its @@ -2507,7 +2537,12 @@ def _mounted_by_hand(root: Path, path: str, project: str | None, refuses (`member-path-taken`), deleted by hand first (rv681's S1). Its `git rm` comes after `git add -- .gitmodules` where `stage_first` (`_stage_first`): after a value finding's lines it exited 128 (#879's - C3).""" + C3). Where that `git rm` refuses the gitlink as staged (`staged_at`, + as `_staged_over` reads it), as after `git mv members/

+ members/

2`, staged, the text says so (`STAGED_AT`), where it said + only that the line refuses while the checkout holds changes (lane 1's + SHOULD on #889, round 2). No line here takes such a gitlink out: + `-f` would take the checkout's changes with it.""" member = (PROJECT_PLACEHOLDER if project is None else shlex.quote(project)) add = (f" {PYTHON} scripts/family.py add --family-root " @@ -2536,8 +2571,10 @@ def _mounted_by_hand(root: Path, path: str, project: str | None, f"{RESTORE_ROW} If it was mounted by hand, make a branch " "with the first command below, since family.py add refuses the " f"holder's tracking branch, {staging}take it out with the " - f"{rm_at}, which refuses while its checkout holds changes and takes " - f"its {GITMODULES} section out too, and commit that removal, since " + f"{rm_at}, which refuses while its checkout holds changes" + + (STAGED_AT if staged_at else "") + + f" and takes its {GITMODULES} section out too, and commit that " + "removal, since " "family.py add refuses a path under another spelling of members/ " "that HEAD holds and the index no longer does, then mount it " f"with the {add_at}, from a checkout of openRepoShape: family.py " @@ -3063,11 +3100,17 @@ def _section_back(root: Path, path: str, puts back the section a plain `git rm` took out. NOTHING IS PUT BACK OVER WHAT A FINDING PRINTED BEFORE WROTE, or where - it put the file back (`Report.wrote`, `Report.put_back`), on disk or - not: the section HEAD's gives is written in key by key, as HEAD has - it, but for a key it gives a value git dies on (`asked`'s second, - which `gitmodules-value-refused` takes out), and the file staged - (`_keys_back`). The `git checkout HEAD` took + it put the file back (`Report.wrote`, `Report.put_back`): the section + HEAD's gives is written in key by key, as HEAD has it, but for a key + it gives a value git dies on (`asked`'s second, which + `gitmodules-value-refused` takes out), and the file staged + (`_keys_back`). Where the work tree's file is gone, only a put-back + makes a file to write into: the first finding that writes one puts it + back and records that (#879's C1), so `Report.put_back` alone is asked + there. A hand mount's `git rm`, counted as a write, writes nothing + into a file that is gone, and the key-by-key lines then made a file of + one section, which the commit recorded (lane 1's MUST 2 on #889). The + `git checkout HEAD` took out again a section a finding before had written, where the work tree's file was gone (#879's C2, rv876's P5d) and where it was on disk (lane 1's SHOULD 1 on #889); and written in alone, the section stayed @@ -3093,7 +3136,7 @@ def _section_back(root: Path, path: str, if report.put_back and path in _blob_paths(root, report.put_back): return [] wanted, refused = asked - if report.wrote or report.put_back: + if report.put_back or (on_disk and report.wrote): report.wrote = True return _keys_back(path, sections, on_disk, refused) if on_disk: diff --git a/tests/test_family_holder_remedies_compose.py b/tests/test_family_holder_remedies_compose.py index 53eaca2f..af813a84 100644 --- a/tests/test_family_holder_remedies_compose.py +++ b/tests/test_family_holder_remedies_compose.py @@ -65,6 +65,14 @@ variant (NIT 3); a run with no row says nothing was updated (NIT 5); and the value finding's put-back serves the removal after it (NIT 6). +LANE 1'S REVIEW OF #889, ROUND 2, TAKEN HERE: a hand mount's `git rm` +writes nothing into a `.gitmodules` that is gone, so a removal after it +puts the file back itself rather than write a section into no file (MUST +2); a clone at no case variant of the row's path is not said to be one +`family.py add` refuses (the NIT, lane 1's mutant MY3); and a hand mount's +`git rm` says it refuses a gitlink staged at a commit HEAD does not record +there (the SHOULD). + THE SAME FAMILY, OFFLINE RULES AND HELPERS as `tests/test_family_holder_bootstrap_texts.py` (#839) and `tests/test_family_unnamed_gitlink.py` (#878). A case variant is an index @@ -1318,3 +1326,157 @@ def test_round1_nit_6_the_value_s_put_back_serves_the_removal_after_it( typed_in_order(holder, env, validate(holder, "--no-members")) staged_and_committed(holder, env, "Typed in order") ends_well(holder, env, family, tmp_path) + + +# --- lane 1's review of #889, round 2 --------------------------------------- + +#: `(the flags of the removal, the lines its finding prints)` where a hand +#: mount's `git rm` came before it and the work tree's `.gitmodules` is +#: gone: the index's file put back, or HEAD's after a plain `git rm`, as +#: round 0 printed (lane 1's MUST 2 on #889, round 2). +HAND_MOUNT_THEN = { + "cached": (["--cached"], + [f"git reset -q -- {ROW}", "git checkout -- .gitmodules"]), + "plain": ([], [f"git reset -q -- {ROW}", + "git checkout HEAD -- .gitmodules", + f"git submodule update --init --recursive -- {ROW}"]), +} + + +@pytest.mark.parametrize("kind", HAND_MOUNT_THEN) +def test_round2_must_2_a_hand_mount_s_git_rm_writes_nothing_in_a_gone_file( + family, holder, offline, tmp_path, kind): + """Lane 1's MUST 2 on #889, round 2: SPARE mounted by hand at + `members/IRQS` (`hand_mounted`), then `git rm --cached members/IRRS` + or a plain `git rm members/IRRS`, and the work tree's `.gitmodules` + deleted. Round 1 counted the hand mount's `git rm` as a write, though + it writes nothing into a file that is gone, so the removal wrote + MEMBER's section key by key "in the file a line above put back", which + no line had: committed, HEAD's `.gitmodules` held that one section, + and the gate found `member-gitmodules-absent` in every mode. Only a + put-back makes a file to write into there, so the removal puts the + file back itself, as round 0 did. Typed in order and committed, + nothing is left, HEAD's `.gitmodules` gives both members, the gate + passes in every mode and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + flags, lines = HAND_MOUNT_THEN[kind] + hand_mounted(holder, env, family) + raw(holder, env, "rm", "-q", *flags, "--", ROW) + gone(holder, env, "deleted") + + found = found_alike(holder, *MODES) + + assert codes_of(found) == ["member-gitlink-unnamed", + "member-gitlink-removal-staged"] + assert commands(found[1]) == lines + typed_in_order(holder, env, validate(holder, "--no-members")) + staged_and_committed(holder, env, "Typed in order") + assert raw(holder, env, "status", "--porcelain").stdout == b"" + assert head_paths(holder, env) == {ROW, OTHER_ROW} + ends_well(holder, env, family, tmp_path) + + +def test_round2_nit_a_clone_at_no_case_variant_is_not_said_refused( + family, holder, offline, tmp_path): + """Lane 1's NIT on #889, round 2 (its mutant MY3): `git rm --cached + members/IRRS`, its checkout deleted, then IRRS cloned at `tools/x`, + which is no case variant of the row's path, and `git add tools/x`. + `member-gitlink-removal-staged` names a clone of its own only at a + case variant of the row's path, which `family.py add` refuses as + `family-members-dir-variant` (round 1's SHOULD 2); of one at + `tools/x`, which `gitlink-unnamed` takes out of the index, it says in + every mode what it says with no clone there. Typed in order, the clone + stays where it is, untracked, nothing is left to commit, the gate + passes in every mode and a plain clone checks both out.""" + env = cloning(offline, tmp_path, family) + raw(holder, env, "rm", "-q", "--cached", "--", ROW) + rmtree(holder / ROW) + alone = found_alike(holder, *MODES) + raw(holder, env, "clone", "-q", "--", + str(family["remotes"] / f"{MEMBER}.git"), TOOLS) + raw(holder, env, "add", "--", TOOLS) + + found = found_alike(holder, *MODES) + + assert codes_of(found) == ["gitlink-unnamed", + "member-gitlink-removal-staged"] + assert found[1] == alone[-1] + typed_in_order(holder, env, validate(holder, "--no-members")) + assert raw(holder, env, "status", "--porcelain").stdout == b"?? tools/\n" + ends_well(holder, env, family, tmp_path) + + +#: What a hand mount's text says of its `git rm` where git refuses the +#: gitlink as staged (lane 1's SHOULD on #889, round 2), and elsewhere. +STAGED_REFUSED = ( + "take it out with the second, which refuses while its checkout holds " + "changes, and while the index stages it at a commit HEAD does not record " + "there, as it does here (the following file has changes staged in the " + "index, exit 1), and takes its .gitmodules section out too") +PLAIN_REFUSED = ("take it out with the second, which refuses while its " + "checkout holds changes and takes its .gitmodules section " + "out too") + + +def moved_by_git_mv(root: Path, env: dict, family) -> str: + """`git mv members/IRRS members/IRRS2`, staged. Returns the new path.""" + raw(root, env, "mv", "--", ROW, f"{ROW}2") + return f"{ROW}2" + + +def staged_by_hand(make): + """A state in which SPARE's gitlink is staged at `members/IRQS`, not + committed, with what `make` puts on disk there.""" + def staged(root: Path, env: dict, family) -> str: + path = f"members/{SPARE}" + stage_gitlink(root, path, spare_commits(family)[1]) + make(root / path) + return path + return staged + + +def committed_with_its_checkout(root: Path, env: dict, family) -> str: + """`mounted_with_its_section`'s state. Returns the gitlink's path.""" + mounted_with_its_section(root, env, family) + return f"members/{SPARE}" + + +#: `(the state, whether a plain git rm refuses its gitlink as staged)`: +#: measured on git 2.43, it took one staged over nothing on disk, and one +#: committed with its checkout. +STAGED_STATES = { + "git-mv": (moved_by_git_mv, True), + "an-empty-directory": (staged_by_hand(Path.mkdir), True), + "nothing-on-disk": (staged_by_hand(lambda where: None), False), + "committed-with-its-checkout": (committed_with_its_checkout, False), +} + + +@pytest.mark.parametrize("make, refused", STAGED_STATES.values(), + ids=STAGED_STATES) +def test_round2_should_a_hand_mount_s_git_rm_says_it_refuses_a_staged_gitlink( + family, holder, offline, tmp_path, make, refused): + """Lane 1's SHOULD on #889, round 2: a gitlink no row names, staged at + a commit HEAD does not record there, as `git mv members/IRRS + members/IRRS2` leaves one, or as one staged by hand over an empty + directory. The hand mount's `git rm -- ` exits 1 there (`the + following file has changes staged in the index`), as before this + change, where its text said only that it refuses while its checkout holds + changes. It says that refusal too now, in every mode, and not where + nothing is on disk or HEAD records the gitlink as staged, which git + takes. No line takes such a gitlink out: `-f` would take the + checkout's changes with it.""" + env = cloning(offline, tmp_path, family) + path = make(holder, env, family) + + found = found_alike(holder, *MODES) + + mount = found[0] + assert mount.startswith(f"FINDING member-gitlink-unnamed: {path} "), mount + assert (STAGED_REFUSED if refused else PLAIN_REFUSED) in mount, mount + switch, removal, _ = commands(mount) + assert removal == f"git rm -- {path}" + type_line(holder, env, switch) + typed = raw(holder, env, *shlex.split(removal)[1:], check=False) + assert typed.returncode == int(refused), typed.stderr + assert (b"has changes staged in the index" in typed.stderr) is refused From 528adbf75e28600cc6fa5ea6863843a900622a62 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 15:47:44 +0000 Subject: [PATCH 18/20] the family root's siblings.py and bootstrap.py: a member's path this user cannot read is refused by name, `member-unreadable`, exit 2, so Python 3.14 no longer clones a member on its remote's default branch, fetches into a clone it never identified, or ends `siblings ok` and `family bootstrap ok` over members it skipped as absent (#430) Slice F of lane 1's plan for #430 (issuecomment-6251069236). Each guard asked `Path.is_file()` or `Path.exists()` of a member's path: Python 3.12 and 3.13 raise a refused `stat` (`PermissionError`), so a `members/

` this user cannot search ended `make siblings` and `make bootstrap` in that traceback, and 3.14 answers False to any `OSError`, so the path read as absent. Measured on 3b15de03 (#889's round-1 head) with a family made by the repository's own tools, under a real mode on 3.12.3 and under a 3.14-like shim: - siblings.py `tracking_branch`, `members/IRSS` at 000 or 444: 3.12 a traceback after IRRS was cloned; 3.14 IRSS cloned on `elsewhere`, its remote's default branch, "members/IRSS/project.yaml is not checked out", exit 0 `siblings ok`. - `_print_pinned_row`, the same mode, the clones placed: 3.12 a traceback at the layout table; 3.14 `pinned members/IRSS not checked out make bootstrap`, exit 0. - `project_id`, a clone's `project.yaml` it cannot reach: 3.12 a traceback; 3.14 the clone fetched into (and `make park` dispatched to it) with its id never read, exit 0. - `run_member_bootstrap`: 3.12 a traceback after the clone; 3.14 "no scripts/bootstrap.py; cloned and left as it is", exit 0. - `dispatch`'s `Makefile`: 3.12 a traceback; 3.14 "has no Makefile". - bootstrap.py `unfetchable`, `members/IRSS` at 000 or `members/` at 444: 3.12 a traceback; 3.14 NOT CHECKED OUT, skipped, the advice to set SHAPE_LEGS_TOKEN, exit 0 `family bootstrap ok`. - `run_member_target`, `--no-fetch --members-make validate` (make validate's member half): 3.12 a traceback; 3.14 IRSS skipped, exit 0 `family bootstrap ok`; and a member `Makefile` it cannot reach, "no Makefile; skipped", exit 0. - `members_of`: a `family.yaml` it cannot reach, 3.14 `family-manifest-missing`, "does not exist". Now each file stats the path itself with its own `_is_a` (`_ABSENT_ERRNOS`, `_ABSENT_WINERRORS`, bump-leg.py's shape; both files are pinned, so neither imports it): nothing at the name is the absent case it always was, and any other `OSError` is `REFUSED member-unreadable: : cannot be read ()`, with validate-pins.py's `chmod` advice for a permission and its `OTHER_REASON` for any other cause, exit 2, the same on 3.12 and 3.14. `family.yaml` is `family-manifest-unreadable`, the code a `family.yaml` that is no mapping already has. - siblings.py refuses the one member and still places, or dispatches to, the others (a refusal never stops the ones after it, and `make park` exists to save work); the run exits 2 and says how many paths it could not read. A refused `scripts/bootstrap.py` says the clone was made and to run its bootstrap there, since a second `make siblings` only fetches. - bootstrap.py stops where it is refused, exit 2, as it does for `git-not-runnable`: in (a) before any member's make runs, in (b) at the member whose mount or `Makefile` it cannot reach. tests/test_family_siblings_bootstrap_unreadable.py: for each site, the absent control (unchanged; the ten controls pass on 3b15de03 too), a refused `stat` in process on every platform (`refused_stat`), and a real mode where one can be made (skipped on Windows and as root), with an I/O error told no `chmod`. 35 tests; 25 fail on 3b15de03. Ten mutants, one per guard group and one per `_is_a` reading every `OSError` as absent, are each killed. Part of #430 (slice F; lane 1's plan, issuecomment-6251069236). Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 150 ++++- templates/family-root/scripts/siblings.py | 280 ++++++++-- ...st_family_siblings_bootstrap_unreadable.py | 528 ++++++++++++++++++ 3 files changed, 917 insertions(+), 41 deletions(-) create mode 100644 tests/test_family_siblings_bootstrap_unreadable.py diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index a318ead3..ad945c11 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -116,6 +116,16 @@ Where (a) cannot start git, the run STOPS there, exit 2, refused `git-not-runnable` as `validate-family.py` refuses it (#694): each member's own bootstrap runs git too, so (b) would only say it again. + And where (a) or (b) cannot ask whether a row's member is checked + out, or whether it has a `Makefile`, because the operating system + refused the `stat` of its `.git` or its `Makefile` (a `members/` this + user cannot search), the run STOPS there, exit 2, refused + `member-unreadable`, naming the path and the cure (#430). On Python + 3.14 that member read as NOT CHECKED OUT, or as having no Makefile, + and was skipped, and the run said `family bootstrap ok`; before it, + the run ended in a `PermissionError` traceback. A `family.yaml` whose + `stat` is refused is `family-manifest-unreadable`, never + `family-manifest-missing`. THE FAMILY CONFERS NOTHING and neither does this command. It fetches, it delegates, and it says what it could not reach. @@ -124,13 +134,16 @@ from __future__ import annotations import argparse +import errno import os import re import shlex import shutil +import stat import subprocess import sys from collections import Counter +from collections.abc import Callable from pathlib import Path from typing import NamedTuple @@ -208,6 +221,100 @@ #: HTTPS one for the duration of a single command. REWRITE_TARGETS = ("https://github.com/", "git@github.com:") +#: The `OSError`s that `Path.is_file()` and `Path.exists()` answered False to +#: on Python 3.9 to 3.13 (pathlib's `_ignore_error`), because each means there +#: is nothing at the name to open: no such file, a part of the path that is +#: not a directory, a bad descriptor, a symlink loop, and Windows' "not +#: ready", "invalid name" and "cannot resolve". Every other one was raised. +_ABSENT_ERRNOS = (errno.ENOENT, errno.ENOTDIR, errno.EBADF, errno.ELOOP) +_ABSENT_WINERRORS = (21, 123, 1921) + +#: What a path the operating system refused for ANOTHER reason than +#: permission is told: an I/O error, a name longer than the filesystem +#: takes, a file replaced while this ran. No `chmod` puts one right. +#: `validate-pins.py`'s words for the same refusal (#712, #430). +OTHER_REASON = ( + "Where the operating system gave another reason, put a regular file this " + "user can open back in its place, or take away one nobody wants, and make " + "sure nothing is changing the directory while this runs. ") + + +def _is_a(path: Path, kind: Callable[[int], bool]) -> bool: + """Whether `path` is there and of `kind` (`stat.S_ISREG`, `_anything`), + raising the `OSError` of a `stat` the operating system REFUSED (#430). + + NOT `Path.is_file()` OR `Path.exists()`, whose answer depends on the + Python. Up to 3.13 they raised a refused `stat` (`PermissionError`), and + `make bootstrap` ended in that traceback; from 3.14 they answer False to + ANY `OSError`. On 3.14 a member mounted under a `members/` this user + cannot search was NOT CHECKED OUT, skipped, beside the advice to set a + credential, and one whose `Makefile` could not be reached had none, and + the run said `family bootstrap ok` over members nothing ran in. Asked + here, every Python answers as 3.9 to 3.13 did, and a refused `stat` is a + refusal that names the path (`unreadable`). A name the system cannot + take at all (a NUL in a row's `path:`) is no file, as `exists()` + answered it. This file's own copy, as each pinned copy keeps one (#370, + #610): `bump-leg.py` has the same. + """ + try: + mode = path.stat().st_mode + except OSError as exc: + if (exc.errno in _ABSENT_ERRNOS + or getattr(exc, "winerror", None) in _ABSENT_WINERRORS): + return False + raise + except ValueError: + return False + return kind(mode) + + +def _anything(mode: int) -> bool: + """Any kind of file at all: what `Path.exists()` asks of a `.git`, + which is a file in a mounted member and a folder in a clone (#430).""" + return stat.S_IFMT(mode) != 0 + + +def unreadable(code: str, named: str, exc: OSError, what: str) -> Refusal: + """`code`'s refusal of a path this command must ask about and the + operating system will not let it: its `stat` refused (`_is_a`), as in a + folder this user cannot search (#430). The line is `validate-pins.py`'s + `_cannot_read`, `: cannot be read ()`, with + `named` relative to the holder and spelled; `what` says what was not + done, and why the question matters. THE ADVICE FOLLOWS THE CAUSE, in + `validate-pins.py`'s words: a permission (`PermissionError`, which is + `EACCES` and `EPERM`) is put right by the mode or the owner, and any + other cause is `OTHER_REASON`, which is not told to `chmod`.""" + advice = ( + "Make what this user may not open readable by this user: `chmod u+r` " + "on a file, `chmod u+rx` on a directory (and on every directory above " + "it), or fix who owns it. " + if isinstance(exc, PermissionError) else OTHER_REASON) + return Refusal( + code, + f"{named}: cannot be read ({exc.strerror or type(exc).__name__})", + f"Remediation: {what} A path this command cannot read is a question " + f"it cannot answer, never a pass. {advice}Then run the same command " + "again.") + + +def checked_out(root: Path, row: dict) -> bool: + """Whether the row's member is checked out: its mount holds a `.git`, + asked as `_is_a` asks it, or the `member-unreadable` refusal naming that + `.git` where its `stat` was refused (#430). (a) asks it of every row + (`unfetchable`) and (b) before it runs anything in the member + (`run_member_target`); on Python 3.14 the answer was "not checked out", + and the member was skipped.""" + git_dir = member_path(root, row) / ".git" + try: + return _is_a(git_dir, _anything) + except OSError as exc: + raise unreadable( + "member-unreadable", spelled(f"{row_path(row)}/.git"), exc, + "Each row's mount is asked whether git checked its member out " + "before anything is run in it, and this one could not be asked, " + "so the run stopped there: no member's command was run after " + "it.") from exc + def credential() -> tuple[str, str]: """`(source, token)`; `("none", "")` when the environment carries none.""" @@ -911,10 +1018,11 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool same, and git's init asked where it stopped, for `fetch_members` to ask why git updated none (`updated_none`): this returned before `init_stop` there, so a run whose update git's init stopped said - `family bootstrap ok` (#879).""" + `family bootstrap ok` (#879). A row whose mount cannot be asked is + refused, `member-unreadable` (`checked_out`, #430).""" left: dict[str, dict] = {} for row in rows: - if not (member_path(root, row) / ".git").exists(): + if not checked_out(root, row): left.setdefault(row_path(row), row) if not left and exited_0: return [], None, Gitmodules(Counter(), {}, {}) @@ -974,8 +1082,21 @@ def updated_none(stopper: str | None, modules: Gitmodules) -> str | None: def members_of(root: Path) -> tuple[str, list[dict]]: + """`(name:, the member rows)` out of `family.yaml`, or a refusal. + + A `family.yaml` whose `stat` the operating system refused is + `family-manifest-unreadable`, naming it and the cure (#430), never + `family-manifest-missing`, which Python 3.14 said of it, whose remedy + sends a person to the wrong kind of root.""" manifest_path = root / MANIFEST - if not manifest_path.is_file(): + try: + present = _is_a(manifest_path, stat.S_ISREG) + except OSError as exc: + raise unreadable( + "family-manifest-unreadable", MANIFEST, exc, + f"{MANIFEST} declares this holder's members, and which they are " + "cannot be told without it.") from exc + if not present: raise Refusal( "family-manifest-missing", f"{manifest_path} does not exist, so there is no declaration of " @@ -1027,11 +1148,20 @@ def run_member_target(root: Path, row: dict, target: str) -> str | None: project = spelled(row.get("project") or "?") path = member_path(root, row) named = spelled(target) - if not (path / ".git").exists(): + if not checked_out(root, row): print(f" [{project}] {spelled(path.name)}: NOT CHECKED OUT; skipped. " "See the fetch above.") return None - if not (path / "Makefile").is_file(): + try: + has_makefile = _is_a(path / "Makefile", stat.S_ISREG) + except OSError as exc: + # One it cannot reach is no "no Makefile" (#430), which Python 3.14 + # said of it, and `family bootstrap ok` after. + raise unreadable( + "member-unreadable", spelled(f"{row_path(row)}/Makefile"), exc, + f"`make {named}` was not run in {project}, nor in any member " + "after it.") from exc + if not has_makefile: print(f" [{project}] no Makefile; skipped. A member without one is " "not misconfigured — it simply has nothing for this command to " "call.") @@ -1153,7 +1283,15 @@ def main(argv: list[str] | None = None) -> int: "`family.py add` is what puts one here.") else: for row in rows: - failure = run_member_target(root, row, args.members_make) + try: + failure = run_member_target(root, row, args.members_make) + except Refusal as exc: + # Only `member-unreadable` reaches here (#430). Stdout + # first, so a log of both streams reads in the order the run + # went. + sys.stdout.flush() + print(str(exc), file=sys.stderr) + return 2 if failure: failed.append(failure) diff --git a/templates/family-root/scripts/siblings.py b/templates/family-root/scripts/siblings.py index f7115eca..d440ce4a 100755 --- a/templates/family-root/scripts/siblings.py +++ b/templates/family-root/scripts/siblings.py @@ -153,19 +153,27 @@ a `make resume` that skipped every member because nobody had run `make siblings` must never read as success 2 a REFUSAL: the question could not be asked — no `family.yaml`, no - `git`, no `make` for a `--make` run. An unanswerable question is never - an implicit pass. + `git`, no `make` for a `--make` run, or a path of a member's that this + user cannot read (`member-unreadable`, #430): the pinned copy's + `project.yaml` or `.git`, or the working clone's `project.yaml`, + `scripts/bootstrap.py` or `Makefile`, whose `stat` the operating + system refused, as it does in a folder this user cannot search. That + member is refused by name, with the cure, and the others are still + tried. An unanswerable question is never an implicit pass. """ from __future__ import annotations import argparse +import errno import os import re import shlex import shutil +import stat import subprocess import sys +from collections.abc import Callable from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -236,6 +244,89 @@ GIT_INT = re.compile(r"([+-]?)(0[xX][0-9a-fA-F]+|0[0-7]*|[1-9][0-9]*)" r"([kKmMgG]?)") +#: The `OSError`s that `Path.is_file()` and `Path.exists()` answered False to +#: on Python 3.9 to 3.13 (pathlib's `_ignore_error`), because each means there +#: is nothing at the name to open: no such file, a part of the path that is +#: not a directory, a bad descriptor, a symlink loop, and Windows' "not +#: ready", "invalid name" and "cannot resolve". Every other one was raised. +_ABSENT_ERRNOS = (errno.ENOENT, errno.ENOTDIR, errno.EBADF, errno.ELOOP) +_ABSENT_WINERRORS = (21, 123, 1921) + +#: What a path the operating system refused for ANOTHER reason than +#: permission is told: an I/O error, a name longer than the filesystem +#: takes, a file replaced while this ran. No `chmod` puts one right. +#: `validate-pins.py`'s words for the same refusal (#712, #430). +OTHER_REASON = ( + "Where the operating system gave another reason, put a regular file this " + "user can open back in its place, or take away one nobody wants, and make " + "sure nothing is changing the directory while this runs. ") + + +def _is_a(path: Path, kind: Callable[[int], bool]) -> bool: + """Whether `path` is there and of `kind` (`stat.S_ISREG`, `_anything`), + raising the `OSError` of a `stat` the operating system REFUSED (#430). + + NOT `Path.is_file()` OR `Path.exists()`, whose answer depends on the + Python. Up to 3.13 they raised a refused `stat` (`PermissionError`), and + `make siblings` ended in that traceback; from 3.14 they answer False to + ANY `OSError`. On 3.14 a pinned copy under a `members/` this + user cannot search was "not checked out", and the member's working + clone was made on its remote's default branch, not its tracking branch; + a clone's `project.yaml` that could not be reached was no id, so the + clone was fetched into, or a verb run in it, unidentified; and a + `scripts/bootstrap.py` or `Makefile` that could not be reached was none, + so nothing ran there. Asked here, every Python answers as 3.9 to 3.13 + did, and a refused `stat` is `member_unreadable`'s refusal, naming the + path. A name the system cannot take at all (a NUL in a row's `path:`) + is no file, as `exists()` answered it. This file's own copy, as each + pinned copy keeps one (#370, #610): `bump-leg.py` has the same. + """ + try: + mode = path.stat().st_mode + except OSError as exc: + if (exc.errno in _ABSENT_ERRNOS + or getattr(exc, "winerror", None) in _ABSENT_WINERRORS): + return False + raise + except ValueError: + return False + return kind(mode) + + +def _anything(mode: int) -> bool: + """Any kind of file at all: what `Path.exists()` asks of a `.git`, + which is a folder in a clone and a file in a mounted member (#430).""" + return stat.S_IFMT(mode) != 0 + + +def member_unreadable(named: str, exc: OSError, what: str, + then: str = "Then run the same command again." + ) -> Refusal: + """`member-unreadable`: `: cannot be read ()`, `validate-pins.py`'s `_cannot_read` line (#430), for a path + of a member's that this command must ask about and the operating system + will not let it: its `stat` refused (`_is_a`), as in a folder this user + cannot search. + + `named` is the path as this run names it, spelled: relative to the + holder where it is in it (`members//project.yaml`), and whole + where it is the working clone beside it. `what` says what was not done + for the member, and why the question matters; `then` is the step after + the cure. THE ADVICE FOLLOWS THE CAUSE, in `validate-pins.py`'s words: a + permission (`PermissionError`, which is `EACCES` and `EPERM`) is put + right by the mode or the owner, and any other cause is `OTHER_REASON`, + which is not told to `chmod`.""" + advice = ( + "Make what this user may not open readable by this user: `chmod u+r` " + "on a file, `chmod u+rx` on a directory (and on every directory above " + "it), or fix who owns it. " + if isinstance(exc, PermissionError) else OTHER_REASON) + return Refusal( + "member-unreadable", + f"{named}: cannot be read ({exc.strerror or type(exc).__name__})", + f"Remediation: {what} A path this command cannot read is a question " + f"it cannot answer, never a pass. {advice}{then}") + # --------------------------------------------------------------------------- # where a member comes from @@ -440,11 +531,27 @@ def tracking_branch(root: Path, row: dict) -> tuple[str | None, str]: print. An ESC or a newline would reach this command's output raw, where the member's own bootstrap escapes the same field (#391); `repr` escapes exactly what `isprintable` refuses, so the two agree with no import. + + A COPY THAT CANNOT BE ASKED ABOUT IS REFUSED, NOT "NOT CHECKED OUT" + (#430): where the `stat` of its `project.yaml` is refused, as under a + `members/` this user cannot search, this raises + `member-unreadable`, and the member is not cloned. On Python 3.14 the + copy read as not checked out and the clone was made on the remote's + default branch, which is the wrong one wherever the two differ. """ label = (str(row.get("path") or f"members/{row.get('project')}") + "/project.yaml") manifest = member_path(root, row) / "project.yaml" - if not manifest.is_file(): + try: + checked_out = _is_a(manifest, stat.S_ISREG) + except OSError as exc: + raise member_unreadable( + spelled(label), exc, + "`make siblings` reads the member's pinned copy for the " + "tracking branch its working clone is made on, so nothing was " + "cloned for it: a clone on the remote's default branch is on " + "the wrong one wherever the two differ.") from exc + if not checked_out: return None, (f"{label} is not checked out, so its tracking branch " "is not known here (`make bootstrap` checks it out)") try: @@ -553,6 +660,9 @@ def __init__(self, project: str, path: Path): self.state = "?" self.branch = "-" self.finding: str | None = None + #: A path of this member's the system would not let this command + #: read (`member-unreadable`, #430): the run then exits 2. + self.refusal: Refusal | None = None def row(self, kind: str = "sibling") -> str: # `project` is a manifest's text and `branch` a clone's (#454), so a @@ -562,9 +672,47 @@ def row(self, kind: str = "sibling") -> str: f"{spelled(self.branch)}") +def refused_member(root: Path, row: dict, refusal: Refusal) -> Sibling: + """The row of a member that `place` or `dispatch` refused, for a path of + its own this command cannot read (`member-unreadable`, #430). The + members after it are still placed, or their verb run: a refusal never + stops the ones after it, and `make park` exists to save work.""" + project = str(row.get("project") or "?") + sibling = Sibling(project, root.parent / project) + sibling.state = "CANNOT BE READ" + sibling.refusal = refusal + return sibling + + +def said_findings_and_refusals(findings: list[str], + refusals: list[Refusal]) -> None: + """Each FINDING, then each refusal whole, its remedy included (#430), + on stderr: the lines before a run's summary.""" + for finding in findings: + print(f"FINDING {finding}", file=sys.stderr) + for refusal in refusals: + print(str(refusal), file=sys.stderr) + + def project_id(path: Path) -> str | None: + """The `id:` of the clone at `path`, or None where it has no + `project.yaml`, or one that does not load. + + ONE IT CANNOT REACH IS REFUSED (#430): where the `stat` of `project.yaml` + is refused, this raises `member-unreadable`, since the id is how + `verify_sibling` tells this member's clone from another's. On Python + 3.14 it was None, so the clone was fetched into, or a verb run in it, + with its id never read.""" manifest = path / "project.yaml" - if not manifest.is_file(): + try: + present = _is_a(manifest, stat.S_ISREG) + except OSError as exc: + raise member_unreadable( + spelled(manifest), exc, + "This command never fetches into, or runs a verb in, a clone it " + "cannot identify, and the clone's own `project.yaml` is how it " + "is identified: nothing was fetched or run there.") from exc + if not present: return None try: data = load_yaml(manifest) @@ -676,10 +824,27 @@ def run_member_bootstrap(path: Path, sibling: Sibling) -> None: Not reimplemented here for the same reason the family's `bootstrap.py` does not reimplement it: each member is a whole project and knows how. + + A SCRIPT THAT CANNOT BE REACHED IS REFUSED, NOT "NONE" (#430): where its + `stat` is refused this raises `member-unreadable`, whose remedy says the + clone is there and how to run its bootstrap, since a run of `make + siblings` after it only fetches. On Python 3.14 it was "no + scripts/bootstrap.py", and the run said `siblings ok`. """ script = path / "scripts" / "bootstrap.py" project = spelled(sibling.project) - if not script.is_file(): + try: + present = _is_a(script, stat.S_ISREG) + except OSError as exc: + raise member_unreadable( + spelled(script), exc, + f"The clone of {project} at {spelled(path)} was made, and its " + "own bootstrap, which puts its legs on their branches, was not " + "run.", + f"Then run `{PYTHON} scripts/bootstrap.py` in that clone: a " + "second `make siblings` only fetches a clone that is " + "there.") from exc + if not present: print(f" [{project}] no scripts/bootstrap.py; cloned and " "left as it is. A member without one is not misconfigured — it " "simply has nothing for this command to call.") @@ -964,7 +1129,16 @@ def dispatch(root: Path, row: dict, urls: dict[str, str], target: str, if not verify_sibling(row, path, url, repository, sibling): return sibling - if not (path / "Makefile").is_file(): + makefile = path / "Makefile" + try: + present = _is_a(makefile, stat.S_ISREG) + except OSError as exc: + # One it cannot reach is no "no Makefile" (#430), which Python 3.14 + # said of it: the member is refused by name, and the others run. + raise member_unreadable( + spelled(makefile), exc, + f"`{spelling}` was not run in {at}.") from exc + if not present: sibling.state = "SKIPPED no Makefile" sibling.finding = ( f"{who}: {at} has no Makefile, so there is no `{spelling}` " @@ -1044,8 +1218,11 @@ def dispatch_all(root: Path, name: str, rows: list[dict], print(" no members declared. A family with none is empty, not wrong: " "`family.py add` is what puts one here.") for row in rows: - dispatched.append(dispatch(root, row, urls, target, make_args, make, - dry_run)) + try: + dispatched.append(dispatch(root, row, urls, target, make_args, + make, dry_run)) + except Refusal as exc: + dispatched.append(refused_member(root, row, exc)) print("\n(b) what ran") print(f" {'what':<9} {'name':<18} {'state':<22} branch") @@ -1053,16 +1230,16 @@ def dispatch_all(root: Path, name: str, rows: list[dict], print(sibling.row()) findings = [s.finding for s in dispatched if s.finding] + refusals = [s.refusal for s in dispatched if s.refusal] print() sys.stdout.flush() - if findings: - for finding in findings: - print(f"FINDING {finding}", file=sys.stderr) + if findings or refusals: + said_findings_and_refusals(findings, refusals) print(f"siblings: `make {target}` did not run everywhere — " - f"{len(dispatched) - len(findings)} of {len(dispatched)} " - "member(s) ran and exited 0. Nothing was cloned or fetched.", - file=sys.stderr) - return 1 + f"{len(dispatched) - len(findings) - len(refusals)} of " + f"{len(dispatched)} member(s) ran and exited 0. Nothing was " + "cloned or fetched.", file=sys.stderr) + return 2 if refusals else 1 if dry_run: print(f"--dry-run: nothing was run. `make {target}` would run in " f"{len(dispatched)} member(s).") @@ -1334,18 +1511,33 @@ def _place_all_members(root: Path, rows: list[dict], urls: dict[str, str], """Every member, placed, in the order `family.yaml` lists them. Split from `main` for #137 — "(a) the members, beside the holder". + A member `place` refuses, for a path of its own this command cannot + read, is that member's row, and the next is placed (#430). """ if not rows: print(" no members declared. A family with none is empty, not wrong: " "`family.py add` is what puts one here.") - return [place(root, row, urls, prefix, token, dry_run) for row in rows] + placed: list[Sibling] = [] + for row in rows: + try: + placed.append(place(root, row, urls, prefix, token, dry_run)) + except Refusal as exc: + placed.append(refused_member(root, row, exc)) + return placed -def _print_pinned_row(root: Path, row: dict) -> None: +def _print_pinned_row(root: Path, row: dict) -> Sibling: """One `members/` row of the layout table: the PINNED copy, - detached, that `make bootstrap` and `make validate` read. + detached, that `make bootstrap` and `make validate` read. Returns it, + so a refusal on it reaches the report. Split from `main` for #137 — the body of its second layout-table loop. + + A MOUNT THAT CANNOT BE ASKED ABOUT IS REFUSED, NOT "NOT CHECKED OUT" + (#430): where the `stat` of its `.git` is refused, as in a `members/` + this user cannot search, the row says `CANNOT BE READ` and the run is + refused `member-unreadable`. On Python 3.14 it said `not checked out`, + `make bootstrap`, of a checked-out copy, and the run said `siblings ok`. """ mounted = member_path(root, row) # The row's OWN `path:` as the label, never a path computed back out @@ -1353,18 +1545,30 @@ def _print_pinned_row(root: Path, row: dict) -> None: # under this root, and a table is not the place to discover it. pinned = Sibling(str(row.get("path") or f"members/{row.get('project')}"), mounted) - if (mounted / ".git").exists(): - pinned.state = "pinned in the holder" - pinned.branch = branch_of(mounted) + try: + checked_out = _is_a(mounted / ".git", _anything) + except OSError as exc: + pinned.state = "CANNOT BE READ" + pinned.refusal = member_unreadable( + spelled(f"{pinned.project}/.git"), exc, + "The pinned copy under members/ is what `make bootstrap` and " + "`make validate` read, and whether it is checked out could not " + "be asked.") else: - pinned.state = "not checked out" - pinned.branch = "make bootstrap" + if checked_out: + pinned.state = "pinned in the holder" + pinned.branch = branch_of(mounted) + else: + pinned.state = "not checked out" + pinned.branch = "make bootstrap" print(pinned.row("pinned")) + return pinned def _print_layout_table(root: Path, siblings: list[Sibling], - rows: list[dict]) -> None: + rows: list[dict]) -> list[Sibling]: """"(b) the layout": the holder, every sibling, then every PINNED mount. + Returns the pinned rows, for the report (#430). Split from `main` for #137. """ @@ -1379,24 +1583,30 @@ def _print_layout_table(root: Path, siblings: list[Sibling], # THE SECOND COPY, said out loud in the same table: the pinned mount is # what `make bootstrap` and `make validate` read, and it is detached # because a pin is a commit and not a branch. - for row in rows: - _print_pinned_row(root, row) + return [_print_pinned_row(root, row) for row in rows] -def _report_placement(root: Path, siblings: list[Sibling], - dry_run: bool) -> int: +def _report_placement(root: Path, siblings: list[Sibling], dry_run: bool, + pinned: list[Sibling] | None = None) -> int: """The findings (or the ok / dry-run line), and the exit code that matches. Split from `main` for #137. + + A path of a member's this command cannot read, a sibling's or a pinned + copy's (`pinned`), is a refusal and the run exits 2 (#430). """ findings = [s.finding for s in siblings if s.finding] + refusals = [s.refusal for s in [*siblings, *(pinned or [])] + if s.refusal] print() sys.stdout.flush() - if findings: - for finding in findings: - print(f"FINDING {finding}", file=sys.stderr) - print(f"siblings: {len(findings)} finding(s); nothing was moved and " + if findings or refusals: + said_findings_and_refusals(findings, refusals) + counted = [f"{len(findings)} finding(s)"] if findings else [] + if refusals: + counted.append(f"{len(refusals)} path(s) this user cannot read") + print(f"siblings: {' and '.join(counted)}; nothing was moved and " "no existing clone was touched", file=sys.stderr) - return 1 + return 2 if refusals else 1 if dry_run: print("--dry-run: nothing was cloned, fetched or moved.") return 0 @@ -1462,9 +1672,9 @@ def _run_placement(root: Path, name: str, rows: list[dict], manifest: dict, f"{source})") siblings = _place_all_members(root, rows, urls, prefix, token, dry_run) - _print_layout_table(root, siblings, rows) + pinned = _print_layout_table(root, siblings, rows) - return _report_placement(root, siblings, dry_run) + return _report_placement(root, siblings, dry_run, pinned) def main(argv: list[str] | None = None) -> int: diff --git a/tests/test_family_siblings_bootstrap_unreadable.py b/tests/test_family_siblings_bootstrap_unreadable.py new file mode 100644 index 00000000..fe774046 --- /dev/null +++ b/tests/test_family_siblings_bootstrap_unreadable.py @@ -0,0 +1,528 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The family root's `scripts/siblings.py` and `scripts/bootstrap.py`: a +member's path this user cannot read is refused by name, never read as +absent (#430, slice F of lane 1's plan). + +THE DEFECT. Each tool asked `Path.is_file()` or `Path.exists()` of a +member's path. Up to Python 3.13 those raise a refused `stat` +(`PermissionError`), so a `members/` this user cannot search ended +`make siblings` and `make bootstrap` in that traceback; from 3.14 they +answer False to ANY `OSError`, and the path read as not there: + +- `siblings.py` `tracking_branch`: the pinned copy's `project.yaml` was "not + checked out", and the working clone was made on the remote's DEFAULT + branch, not the member's tracking branch, `siblings ok`; +- `project_id`: a clone's `project.yaml` was no id, so the clone was + fetched into, or `make park` run in it, with its id never read; +- `run_member_bootstrap`: the member's `scripts/bootstrap.py` was "none", so + its legs were never placed, `siblings ok`; +- `dispatch`: the clone's `Makefile` was "no Makefile"; +- `_print_pinned_row`: a checked-out pinned copy was `not checked out`, + `make bootstrap`; +- `bootstrap.py` `unfetchable` and `run_member_target`: the member was NOT + CHECKED OUT, beside the advice to set a credential, or had no Makefile, + skipped, and the run said `family bootstrap ok`; +- `members_of`: `family.yaml` "does not exist" (`family-manifest-missing`). + +Each is now asked by `stat`ing the path (each file's own `_is_a`, as +`bump-leg.py`'s): nothing at the name (`ENOENT`, `ENOTDIR`, ...) is the +absent case it always was, and any other `OSError` is refused, +`member-unreadable` (`family-manifest-unreadable` for `family.yaml`), +`: cannot be read ()`, exit 2, with `chmod` for a permission +and `validate-pins.py`'s `OTHER_REASON` for any other cause. + +EACH SITE HAS THREE ROWS: nothing there (the control, unchanged); a refused +`stat` as Python 3.14 answers it, in process, on every platform and as root +(`refused_stat`); and a real mode, where one can be made (`needs_a_mode`: +not on Windows, not as root). A mode on `members/` reaches the +pinned copy, its `.git` and its `project.yaml` through the command itself; +a working clone's `Makefile`, a member's `Makefile` and `family.yaml` are +reached by a link into a folder at mode 000, the only way to refuse their +`stat` while the folder they are in, and its `.git`, can still be read. +""" + +from __future__ import annotations + +import errno +import shutil +import stat +from pathlib import Path + +import pytest + +from test_bump_leg_unopenable import ( + made_unopenable, needs_a_mode, refusal_of, refused_stat, +) +from test_doctor_unreadable_pin import IO_ERROR, refusing +from test_family import ( # noqa: F401 + NAME, allow_file_protocol, estate, family, holder, siblings, +) +from test_family_bootstrap_gate_order import bootstrap +from test_family_gitmodules_values import by_path +from test_family_siblings_placement import placement # noqa: F401 + +DENIED = "Permission denied" +#: The two codes this slice refuses with. +MEMBER_UNREADABLE = "member-unreadable" +MANIFEST_UNREADABLE = "family-manifest-unreadable" +#: A row whose member is nowhere: the control's absent case. +NOBODY = {"project": "Nobody", "id": "nobody", "path": "members/Nobody"} +#: A row whose path runs through a file (`ENOTDIR`): absent too. +THROUGH_A_FILE = {"project": "Nobody", "id": "nobody", + "path": "family.yaml/Nobody"} +ABSENT_ROWS = pytest.mark.parametrize( + "row", [NOBODY, THROUGH_A_FILE], + ids=["no-such-path", "a-file-on-the-path"]) +#: The words a permission's remedy and any other cause's begin with. +CHMOD = "`chmod u+rx` on a directory (and on every directory above it)" +OTHER_CAUSE = "Where the operating system gave another reason" + + +@pytest.fixture(scope="module") +def boot(): + """The template's `bootstrap.py`, loaded by path, no bytecode.""" + return by_path("bootstrap.py", "unreadable_family_bootstrap") + + +def row_of(boot, root: Path, project: str) -> dict: + """`project`'s row of the holder's `family.yaml`, as bootstrap reads it.""" + _, rows = boot.members_of(root) + return next(row for row in rows if row.get("project") == project) + + +def refusal_line(code: str, named: str, reason: str = DENIED) -> str: + return f"REFUSED {code}: {named}: cannot be read ({reason})" + + +def no_traceback(result) -> None: + assert "Traceback" not in result.stderr, result.stderr + + +def flat_rows(stdout: str) -> list[str]: + """Each line of a table, its runs of blanks made one.""" + return [" ".join(line.split()) for line in stdout.splitlines()] + + +def linked_into_locked(directory: Path, name: str) -> str: + """`directory/name` made a link to `directory/locked/name`, its bytes + moved there: once `locked` is at mode 000 the `stat` of `name` is + refused while `directory` and its `.git` can still be read. Returns + `locked`, relative to `directory`, for `made_unopenable`.""" + locked = directory / "locked" + locked.mkdir() + (directory / name).rename(locked / name) + (directory / name).symlink_to(Path("locked") / name) + return "locked" + + +@pytest.fixture(scope="module") +def placed_folder(family, tmp_path_factory) -> Path: + """`/InkRouter/` with the holder and both members placed beside it + by the holder's own `make siblings`, once for the module.""" + folder = tmp_path_factory.mktemp("placed") / NAME + folder.mkdir() + shutil.copytree(family["root"], folder / NAME, symlinks=True) + result = siblings(folder / NAME) + assert result.returncode == 0, result.stdout + result.stderr + return folder + + +@pytest.fixture +def placed(placed_folder, tmp_path) -> Path: + """A private copy of `placed_folder`; returns its holder.""" + folder = tmp_path / NAME + shutil.copytree(placed_folder, folder, symlinks=True) + return folder / NAME + + +# --- siblings.py: `tracking_branch`, the pinned copy's project.yaml ---------- + +@ABSENT_ROWS +def test_a_pinned_copy_not_there_keeps_the_default_branch(placement, family, + row): + """The control: no pinned copy is the default branch it always was.""" + assert placement.tracking_branch(family["root"], row) == ( + None, f"{row['path']}/project.yaml is not checked out, so its " + "tracking branch is not known here (`make bootstrap` checks " + "it out)") + + +def test_a_pinned_copy_whose_stat_is_refused_is_refused(placement, family, + monkeypatch): + """On every Python: not the default branch, but `member-unreadable`, + naming the copy's `project.yaml` relative to the holder.""" + row = row_of(placement, family["root"], "IRSS") + refused_stat(monkeypatch, "project.yaml") + refusal = refusal_of(placement, placement.tracking_branch, + family["root"], row) + assert refusal.code == MEMBER_UNREADABLE + assert refusal.detail == (f"members/IRSS/project.yaml: cannot be read " + f"({DENIED})") + assert "nothing was cloned for it" in refusal.remediation + assert CHMOD in refusal.remediation + + +@needs_a_mode +@pytest.mark.parametrize("mode", [0o000, 0o444], ids=["000", "444"]) +def test_make_siblings_clones_no_member_whose_pinned_copy_it_cannot_search( + estate, mode): + """`members/IRSS` at 000 or 444, nothing beside the holder yet: 3.12 + ended in a traceback after IRRS was cloned, and 3.14 cloned IRSS on + its remote's default branch with `siblings ok`. Now IRSS is refused by + name, nothing is cloned for it, IRRS is still placed, exit 2.""" + holder = estate["holder"] + with made_unopenable(holder, "members/IRSS", mode=mode): + result = siblings(holder) + no_traceback(result) + assert result.returncode == 2, result.stdout + result.stderr + assert refusal_line(MEMBER_UNREADABLE, "members/IRSS/project.yaml") \ + in result.stderr + assert "sibling IRSS CANNOT BE READ -" in flat_rows(result.stdout) + assert not (estate["folder"] / "IRSS").exists() + assert (estate["folder"] / "IRRS" / ".git").is_dir() + assert "default branch" not in result.stdout + + +# --- siblings.py: `project_id`, a clone's own project.yaml ------------------- + +def test_a_clone_with_no_project_yaml_has_no_id(placement, tmp_path): + """The control: no `project.yaml` is no id, as it always was.""" + assert placement.project_id(tmp_path) is None + + +def test_a_clone_whose_project_yaml_stat_is_refused_is_refused( + placement, family, monkeypatch): + """On every Python: no "no id", which let the clone be fetched into, or + a verb run in it, with its id never read.""" + clone = family["root"] / "members" / "IRSS" + refused_stat(monkeypatch, "project.yaml") + refusal = refusal_of(placement, placement.project_id, clone) + assert refusal.code == MEMBER_UNREADABLE + assert refusal.detail == (f"{clone / 'project.yaml'}: cannot be read " + f"({DENIED})") + assert "nothing was fetched or run there" in refusal.remediation + + +@needs_a_mode +def test_a_clone_it_cannot_search_is_refused_not_unidentified(placement, + holder): + """A real mode, in process: the clone's folder at 000.""" + clone = holder / "members" / "IRSS" + with made_unopenable(holder, "members/IRSS"): + refusal = refusal_of(placement, placement.project_id, clone) + assert refusal.detail == (f"{clone / 'project.yaml'}: cannot be read " + f"({DENIED})") + + +@pytest.mark.skipif(shutil.which("make") is None, + reason="needs `make` on PATH") +def test_make_park_refuses_every_clone_it_cannot_identify_and_goes_on( + placement, placed_folder, monkeypatch, capsys): + """Through `dispatch_all`, on every Python: each member is refused on + its own row, the one after it is still tried, and the run exits 2.""" + holder = placed_folder / NAME + _, rows = placement.members_of(holder) + refused_stat(monkeypatch, "project.yaml") + code = placement.dispatch_all(holder, NAME, rows, "park", [], True) + said = capsys.readouterr() + assert code == 2, said.out + said.err + table = flat_rows(said.out) + for project in ("IRRS", "IRSS"): + assert f"sibling {project} CANNOT BE READ -" in table, said.out + assert said.err.count(f"REFUSED {MEMBER_UNREADABLE}: ") == 2, said.err + assert "0 of 2 member(s) ran" in said.err + + +# --- siblings.py: `run_member_bootstrap`, the clone's scripts/bootstrap.py --- + +def test_a_clone_with_no_bootstrap_is_left_as_it_is(placement, tmp_path, + capsys): + """The control: a member with no `scripts/bootstrap.py` is cloned and + left as it is, no finding.""" + sibling = placement.Sibling("Nobody", tmp_path) + placement.run_member_bootstrap(tmp_path, sibling) + assert "[Nobody] no scripts/bootstrap.py; cloned and left as it is." \ + in capsys.readouterr().out + assert sibling.finding is None + + +def test_a_bootstrap_whose_stat_is_refused_is_refused_not_skipped( + placement, family, monkeypatch): + """On every Python: no "no scripts/bootstrap.py" and `siblings ok`. The + remedy says the clone is there and how to run its bootstrap, since a + second `make siblings` only fetches it.""" + clone = family["root"] / "members" / "IRSS" + refused_stat(monkeypatch, "bootstrap.py") + refusal = refusal_of(placement, placement.run_member_bootstrap, clone, + placement.Sibling("IRSS", clone)) + assert refusal.code == MEMBER_UNREADABLE + assert refusal.detail == (f"{clone / 'scripts' / 'bootstrap.py'}: cannot " + f"be read ({DENIED})") + assert (f"Then run `{placement.PYTHON} scripts/bootstrap.py` in that " + "clone") in refusal.remediation + assert "Then run the same command again" not in refusal.remediation + + +@needs_a_mode +def test_a_bootstrap_it_cannot_search_for_is_refused(placement, holder): + """A real mode, in process: the clone's `scripts/` at 000.""" + clone = holder / "members" / "IRSS" + with made_unopenable(clone, "scripts"): + refusal = refusal_of(placement, placement.run_member_bootstrap, + clone, placement.Sibling("IRSS", clone)) + assert refusal.detail == (f"{clone / 'scripts' / 'bootstrap.py'}: cannot " + f"be read ({DENIED})") + + +# --- siblings.py: `dispatch`, the clone's Makefile --------------------------- + +def dispatched(placement, holder: Path, project: str): + """`dispatch` of `make park --dry-run` for `project`, in process.""" + row = row_of(placement, holder, project) + return placement.dispatch(holder, row, placement.submodule_urls(holder), + "park", [], "make", True) + + +def test_a_clone_with_no_makefile_is_still_skipped_by_name(placement, placed): + """The control: a clone with no `Makefile` is `SKIPPED no Makefile`.""" + sibling_dir = placed.parent / "IRSS" + (sibling_dir / "Makefile").rename(sibling_dir / "Makefile.aside") + sibling = dispatched(placement, placed, "IRSS") + assert sibling.state == "SKIPPED no Makefile" + assert "has no Makefile" in sibling.finding + + +def test_a_makefile_whose_stat_is_refused_is_refused_not_absent( + placement, placed_folder, monkeypatch): + """On every Python: no "has no Makefile".""" + holder = placed_folder / NAME + refused_stat(monkeypatch, "Makefile") + refusal = refusal_of(placement, dispatched, placement, holder, "IRSS") + assert refusal.code == MEMBER_UNREADABLE + assert refusal.detail == (f"{placed_folder / 'IRSS' / 'Makefile'}: cannot " + f"be read ({DENIED})") + assert "`make park` was not run in" in refusal.remediation + + +@needs_a_mode +def test_make_park_refuses_a_makefile_it_cannot_reach_and_runs_the_rest( + placed): + """A real mode: IRSS's `Makefile` a link into a folder at 000. 3.12 + ended `make park` in a traceback after IRRS, and 3.14 said IRSS `has no + Makefile`. Now IRSS is refused by name, IRRS still runs, exit 2.""" + clone = placed.parent / "IRSS" + with made_unopenable(clone, linked_into_locked(clone, "Makefile")): + result = siblings(placed, "--make", "park", "--dry-run") + no_traceback(result) + assert result.returncode == 2, result.stdout + result.stderr + named = placed.resolve().parent / "IRSS" / "Makefile" + assert refusal_line(MEMBER_UNREADABLE, str(named)) in result.stderr + assert "sibling IRRS would make park main" in flat_rows(result.stdout) + assert "has no Makefile" not in result.stderr + + +# --- siblings.py: `_print_pinned_row`, the pinned copy's .git -------------- + +def test_a_pinned_copy_not_there_is_still_not_checked_out(placement, + family, capsys): + """The control: no mount is `not checked out`, `make bootstrap`.""" + placement._print_pinned_row(family["root"], NOBODY) + assert "pinned members/Nobody not checked out make bootstrap" in \ + flat_rows(capsys.readouterr().out) + + +def test_a_pinned_mount_whose_stat_is_refused_is_refused(placement, family, + monkeypatch, capsys): + """On every Python: no `not checked out` of a checked-out copy.""" + row = row_of(placement, family["root"], "IRSS") + refused_stat(monkeypatch, ".git") + pinned = placement._print_pinned_row(family["root"], row) + assert pinned.state == "CANNOT BE READ" + assert pinned.refusal.detail == (f"members/IRSS/.git: cannot be read " + f"({DENIED})") + assert "pinned members/IRSS CANNOT BE READ -" in flat_rows( + capsys.readouterr().out) + + +@needs_a_mode +def test_make_siblings_refuses_a_pinned_copy_beside_a_placed_clone(placed): + """A real mode, the members placed already: `members/IRSS` at 000. 3.12 + ended in a traceback at the layout table, and 3.14 said `not checked + out`, `make bootstrap` and `siblings ok`. Now the pinned row is + refused, the clones are still fetched, exit 2.""" + with made_unopenable(placed, "members/IRSS"): + result = siblings(placed) + no_traceback(result) + assert result.returncode == 2, result.stdout + result.stderr + assert refusal_line(MEMBER_UNREADABLE, "members/IRSS/.git") \ + in result.stderr + assert "pinned members/IRSS CANNOT BE READ -" in flat_rows(result.stdout) + assert "sibling IRSS present, fetched main" in flat_rows(result.stdout) + assert "siblings ok" not in result.stdout + + +# --- bootstrap.py: `checked_out`, in `unfetchable` and `run_member_target` - + +@ABSENT_ROWS +def test_a_member_not_there_is_still_not_checked_out(boot, family, row, + capsys): + """The control: no mount, or a path through a file, is the member NOT + CHECKED OUT it always was, in (a) and in (b).""" + left, _, _ = boot.unfetchable(family["root"], [row], True) + assert left == [(row, row["path"], boot.NO_GITLINK)] + assert boot.run_member_target(family["root"], row, "help") is None + assert "[Nobody] Nobody: NOT CHECKED OUT; skipped." in \ + capsys.readouterr().out + + +@pytest.mark.parametrize("ask", ["unfetchable", "run_member_target"]) +def test_a_mount_whose_stat_is_refused_is_refused(boot, family, monkeypatch, + ask): + """On every Python, in (a) and in (b): no NOT CHECKED OUT, skipped.""" + row = row_of(boot, family["root"], "IRRS") + refused_stat(monkeypatch, ".git") + args = ([row], True) if ask == "unfetchable" else (row, "help") + refusal = refusal_of(boot, getattr(boot, ask), family["root"], *args) + assert refusal.code == MEMBER_UNREADABLE + assert refusal.detail == f"members/IRRS/.git: cannot be read ({DENIED})" + assert "the run stopped there" in refusal.remediation + + +@needs_a_mode +@pytest.mark.parametrize("relative, mode, named", [ + ("members/IRSS", 0o000, "members/IRSS/.git"), + ("members", 0o444, "members/IRRS/.git"), +], ids=["a-mount-at-000", "members-at-444"]) +def test_make_bootstrap_refuses_a_mount_it_cannot_search(holder, relative, + mode, named): + """A real mode: 3.12 ended (a) in a traceback, and 3.14 said NOT + CHECKED OUT, set SHAPE_LEGS_TOKEN, and `family bootstrap ok`. Now (a) + refuses the mount by name, no member's `make` is run, exit 2.""" + with made_unopenable(holder, relative, mode=mode): + result = bootstrap(holder, allow_file_protocol()) + no_traceback(result) + assert result.returncode == 2, result.stdout + result.stderr + assert refusal_line(MEMBER_UNREADABLE, named) in result.stderr + assert "family bootstrap ok" not in result.stdout + assert ": make bootstrap ---" not in result.stdout + + +@needs_a_mode +def test_make_validates_member_half_refuses_a_mount_it_cannot_search(holder): + """`make validate`'s second step, `--no-fetch`: (b) runs IRRS's target, + then refuses IRSS's mount by name, exit 2, where 3.14 skipped it and + said `family bootstrap ok`.""" + with made_unopenable(holder, "members/IRSS"): + result = bootstrap(holder, allow_file_protocol(), "--no-fetch", + "--members-make", "help") + no_traceback(result) + assert result.returncode == 2, result.stdout + result.stderr + assert "--- IRRS: make help ---" in result.stdout + assert refusal_line(MEMBER_UNREADABLE, "members/IRSS/.git") \ + in result.stderr + + +# --- bootstrap.py: `run_member_target`, the member's Makefile ---------------- + +def test_a_member_with_no_makefile_is_still_skipped(boot, tmp_path, capsys): + """The control: a mount with a `.git` and no `Makefile` is skipped.""" + (tmp_path / "members" / "Ghost").mkdir(parents=True) + (tmp_path / "members" / "Ghost" / ".git").write_bytes(b"gitdir: x\n") + row = {"project": "Ghost", "path": "members/Ghost"} + assert boot.run_member_target(tmp_path, row, "help") is None + assert "[Ghost] no Makefile; skipped." in capsys.readouterr().out + + +def test_a_member_makefile_whose_stat_is_refused_is_refused(boot, family, + monkeypatch): + """On every Python: no "no Makefile; skipped".""" + row = row_of(boot, family["root"], "IRSS") + refused_stat(monkeypatch, "Makefile") + refusal = refusal_of(boot, boot.run_member_target, family["root"], row, + "help") + assert refusal.code == MEMBER_UNREADABLE + assert refusal.detail == (f"members/IRSS/Makefile: cannot be read " + f"({DENIED})") + assert "`make help` was not run in IRSS" in refusal.remediation + + +@needs_a_mode +def test_make_bootstrap_refuses_a_member_makefile_it_cannot_reach(holder): + """A real mode: IRRS's `Makefile` a link into a folder at 000. 3.12 + ended in a traceback, and 3.14 said `no Makefile; skipped` and + `family bootstrap ok`.""" + member = holder / "members" / "IRRS" + with made_unopenable(member, linked_into_locked(member, "Makefile")): + result = bootstrap(holder, allow_file_protocol(), "--no-fetch", + "--members-make", "help") + no_traceback(result) + assert result.returncode == 2, result.stdout + result.stderr + assert refusal_line(MEMBER_UNREADABLE, "members/IRRS/Makefile") \ + in result.stderr + assert "--- IRRS: make help ---" not in result.stdout + + +# --- bootstrap.py: `members_of`, family.yaml -------------------------------- + +def test_a_holder_with_no_family_yaml_is_still_missing(boot, tmp_path): + """The control: no `family.yaml` is `family-manifest-missing`.""" + refusal = refusal_of(boot, boot.members_of, tmp_path) + assert refusal.code == "family-manifest-missing" + + +def test_a_family_yaml_whose_stat_is_refused_is_not_called_missing( + boot, family, monkeypatch): + """On every Python: `family-manifest-unreadable`, never `does not + exist` and its remedy for another kind of root.""" + refused_stat(monkeypatch, "family.yaml") + refusal = refusal_of(boot, boot.members_of, family["root"]) + assert refusal.code == MANIFEST_UNREADABLE + assert refusal.detail == f"family.yaml: cannot be read ({DENIED})" + assert "`chmod u+r` on a file" in refusal.remediation + + +@needs_a_mode +@pytest.mark.parametrize("script", ["bootstrap.py", "siblings.py"]) +def test_both_scripts_refuse_a_family_yaml_they_cannot_reach(holder, script): + """A real mode: `family.yaml` a link into a folder at 000. Both scripts + read it through `members_of`, and both refuse it by name, exit 2.""" + with made_unopenable(holder, linked_into_locked(holder, "family.yaml")): + result = bootstrap(holder, allow_file_protocol()) \ + if script == "bootstrap.py" else siblings(holder) + no_traceback(result) + assert result.returncode == 2, result.stdout + result.stderr + assert refusal_line(MANIFEST_UNREADABLE, "family.yaml") in result.stderr + assert "family-manifest-missing" not in result.stderr + + +# --- the cause: a permission is told `chmod`, any other cause is not --------- + +@pytest.mark.parametrize("which", ["siblings", "bootstrap"]) +def test_a_stat_that_fails_for_another_reason_is_refused_sans_chmod( + placement, boot, family, monkeypatch, which): + """An I/O error is no permission: the same refusal, the system's own + reason on the line, and `validate-pins.py`'s `OTHER_REASON`, with no + `chmod`, in each file's own copy of the words.""" + module = placement if which == "siblings" else boot + pinned = row_of(boot, family["root"], "IRSS") + refusing(monkeypatch, "stat", ".git", OSError(errno.EIO, IO_ERROR)) + if which == "siblings": + refusal = placement._print_pinned_row(family["root"], pinned).refusal + else: + refusal = refusal_of(boot, boot.checked_out, family["root"], pinned) + assert refusal.detail == (f"members/IRSS/.git: cannot be read " + f"({IO_ERROR})") + assert OTHER_CAUSE in refusal.remediation + assert "chmod" not in refusal.remediation + assert module.OTHER_REASON in refusal.remediation + + +def test_every_absent_errno_is_absent_and_every_other_is_raised(boot, + placement): + """Each file's own `_is_a` answers as `bump-leg.py`'s does: the four + errnos of nothing there are False, any other `OSError` is raised.""" + for module in (boot, placement): + assert module._ABSENT_ERRNOS == (errno.ENOENT, errno.ENOTDIR, + errno.EBADF, errno.ELOOP) + assert module._ABSENT_WINERRORS == (21, 123, 1921) + assert module._is_a(Path("no-such-name-here"), stat.S_ISREG) is False From 32a5899feafde7b3ac6f5360157822b1d7556d91 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 16:20:31 +0000 Subject: [PATCH 19/20] validate-family.py: a hand mount's text quotes both of git's refusals of a gitlink staged at a commit HEAD does not record there, and a gitlink HEAD records at another commit is tested (#879) Fix round 3 for #889: lane 1's NITs on 29083a5f (issuecomment-6252129372), both taken. - NIT 1: `STAGED_AT` and `_staged_over`'s docstring quote both of git's refusals, each exit 1: "the following file has changes staged in the index", and, where what is on disk differs from what is staged too, "... has staged content different from both the file and the HEAD" (measured on git 2.43 in seven states: three give the first, four the second). - NIT 2: a hand mount committed with its section, its checkout moved to HEAD~1 and `git add`-ed, is a row of the round-2 SHOULD test, with two rows that print git's second refusal; each refused row checks that the text quotes the words git printed. Tests: 3 new rows in tests/test_family_holder_remedies_compose.py. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- .../family-root/scripts/validate-family.py | 23 +++++-- tests/test_family_holder_remedies_compose.py | 65 +++++++++++++++---- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 85a1630e..cd5dd1a5 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -874,11 +874,16 @@ def _staged_over(root: Path, path: str, commit: str) -> bool: which `_member_gitlinks` reads at `commit`, as staged (`STAGED_AT`): the index stages it at a commit HEAD does not record there, and anything is on disk at it, an empty directory too, which `git rm - --cached` takes with no `-f` (`_forced`). Measured on git 2.43, with - the gitlink staged and not committed: nothing on disk, it took it; an - empty directory, and the checkout `git mv members/

members/

2` - moved, staged, it refused (`the following file has changes staged in - the index`, exit 1; lane 1's SHOULD on #889, round 2).""" + --cached` takes with no `-f` (`_forced`). Measured on git 2.43: with + nothing on disk it took one staged and not committed; it refused one, + exit 1, saying `the following file has changes staged in the index` + where what is on disk matches what is staged, an empty directory, the + clean checkout `git mv members/

members/

2` moved, or a checkout + moved to `HEAD~1` and `git add`-ed, where HEAD records another commit, + and `... has staged content different from both the file and the HEAD` + where it differs too, a commit or an untracked file in the checkout + `git mv` moved, a directory holding a file, or a file (lane 1's SHOULD + on #889, round 2, and its NITs 1 and 2, round 3).""" return (os.path.lexists(root / path) and _head_records(root, path) != commit) @@ -2520,10 +2525,14 @@ def _given_as_git_reads(root: Path) -> set[str]: #: What a hand mount's text says its `git rm` meets where the index stages #: the gitlink at a commit HEAD does not record there (`_mounted_by_hand`, -#: lane 1's SHOULD on #889, round 2), measured on git 2.43. +#: lane 1's SHOULD on #889, round 2): git's two refusals, each exit 1, the +#: second where what is on disk differs from what is staged too, as git +#: 2.43 gave them (`_staged_over`; lane 1's NIT 1 on #889, round 3). STAGED_AT = (", and while the index stages it at a commit HEAD does not " "record there, as it does here (the following file has changes " - "staged in the index, exit 1),") + "staged in the index, or, where what is on disk differs too, " + "has staged content different from both the file and the HEAD, " + "exit 1),") def _mounted_by_hand(root: Path, path: str, project: str | None, diff --git a/tests/test_family_holder_remedies_compose.py b/tests/test_family_holder_remedies_compose.py index af813a84..5cf8899b 100644 --- a/tests/test_family_holder_remedies_compose.py +++ b/tests/test_family_holder_remedies_compose.py @@ -1407,12 +1407,15 @@ def test_round2_nit_a_clone_at_no_case_variant_is_not_said_refused( #: What a hand mount's text says of its `git rm` where git refuses the -#: gitlink as staged (lane 1's SHOULD on #889, round 2), and elsewhere. +#: gitlink as staged (lane 1's SHOULD on #889, round 2), with git's two +#: refusals (its NIT 1, round 3), and elsewhere. STAGED_REFUSED = ( "take it out with the second, which refuses while its checkout holds " "changes, and while the index stages it at a commit HEAD does not record " "there, as it does here (the following file has changes staged in the " - "index, exit 1), and takes its .gitmodules section out too") + "index, or, where what is on disk differs too, has staged content " + "different from both the file and the HEAD, exit 1), and takes its " + ".gitmodules section out too") PLAIN_REFUSED = ("take it out with the second, which refuses while its " "checkout holds changes and takes its .gitmodules section " "out too") @@ -1441,17 +1444,55 @@ def committed_with_its_checkout(root: Path, env: dict, family) -> str: return f"members/{SPARE}" +def moved_with_a_commit(root: Path, env: dict, family) -> str: + """`moved_by_git_mv`'s state, then a commit in the checkout it moved. + Returns the new path.""" + path = moved_by_git_mv(root, env, family) + raw(root / path, env, "-c", "user.name=t", "-c", "user.email=t@t", + "commit", "-q", "--allow-empty", "-m", "Made in the moved checkout") + return path + + +def holding_a_file(where: Path) -> None: + """A directory at `where` that holds a file and no `.git`.""" + where.mkdir() + (where / "kept.txt").write_bytes(b"kept\n") + + +def moved_back_and_added(root: Path, env: dict, family) -> str: + """`committed_with_its_checkout`'s state, then its checkout moved to + `HEAD~1` and `git add`-ed: HEAD records another commit there (lane 1's + NIT 2 on #889, round 3). Returns the path.""" + path = committed_with_its_checkout(root, env, family) + raw(root / path, env, "checkout", "-q", "HEAD~1") + raw(root, env, "add", "--", path) + return path + + #: `(the state, whether a plain git rm refuses its gitlink as staged)`: #: measured on git 2.43, it took one staged over nothing on disk, and one -#: committed with its checkout. +#: committed with its checkout. Where it refuses, the first three say `has +#: changes staged in the index`, and the others, whose disk differs from +#: what is staged, `has staged content different from both the file and +#: the HEAD` (lane 1's NIT 1 on #889, round 3). STAGED_STATES = { "git-mv": (moved_by_git_mv, True), "an-empty-directory": (staged_by_hand(Path.mkdir), True), + "head-records-another-commit": (moved_back_and_added, True), + "git-mv-and-a-commit-in-it": (moved_with_a_commit, True), + "a-directory-holding-a-file": (staged_by_hand(holding_a_file), True), "nothing-on-disk": (staged_by_hand(lambda where: None), False), "committed-with-its-checkout": (committed_with_its_checkout, False), } +def git_refusal(stderr: bytes) -> str: + """What `git rm` said it refused, after "the following file ", up to + its colon, on one line: git wraps the longer of its two refusals.""" + said = stderr.decode("utf-8").split("the following file ", 1)[1] + return " ".join(said.split(":", 1)[0].split()) + + @pytest.mark.parametrize("make, refused", STAGED_STATES.values(), ids=STAGED_STATES) def test_round2_should_a_hand_mount_s_git_rm_says_it_refuses_a_staged_gitlink( @@ -1459,13 +1500,14 @@ def test_round2_should_a_hand_mount_s_git_rm_says_it_refuses_a_staged_gitlink( """Lane 1's SHOULD on #889, round 2: a gitlink no row names, staged at a commit HEAD does not record there, as `git mv members/IRRS members/IRRS2` leaves one, or as one staged by hand over an empty - directory. The hand mount's `git rm -- ` exits 1 there (`the - following file has changes staged in the index`), as before this - change, where its text said only that it refuses while its checkout holds - changes. It says that refusal too now, in every mode, and not where - nothing is on disk or HEAD records the gitlink as staged, which git - takes. No line takes such a gitlink out: `-f` would take the - checkout's changes with it.""" + directory. The hand mount's `git rm -- ` exits 1 there, as + before this change, where its text said only that it refuses while + its checkout holds changes. It says that refusal too now, in every + mode, and not where nothing is on disk or HEAD records the gitlink as + staged, which git takes. No line takes such a gitlink out: `-f` would + take the checkout's changes with it. Round 3 (lane 1's NITs 1 and 2): + the text quotes the refusal git prints, either of two, and a gitlink + HEAD records at another commit gets it too.""" env = cloning(offline, tmp_path, family) path = make(holder, env, family) @@ -1479,4 +1521,5 @@ def test_round2_should_a_hand_mount_s_git_rm_says_it_refuses_a_staged_gitlink( type_line(holder, env, switch) typed = raw(holder, env, *shlex.split(removal)[1:], check=False) assert typed.returncode == int(refused), typed.stderr - assert (b"has changes staged in the index" in typed.stderr) is refused + if refused: + assert git_refusal(typed.stderr) in mount, typed.stderr From 3c5f2eab52c53d4fe4d89164407b418a428e533f Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 18:27:32 +0000 Subject: [PATCH 20/20] the family root's bootstrap.py: step (a)'s `member-unreadable` comes after git's own words, `main` runs (b) through a helper and catches a refusal once, so its cognitive complexity is back to 6 from 18, and the chmod advice is `_unreadable_file`'s word for word; and tests of `_is_a`'s NUL branch, each errno and the placement summary (#430) #951's round 1, on lane 5's review (issuecomment-6253349287) and lane 4's plan (issuecomment-6253366871): - SHOULD 1: `main` had a `try/except Refusal` three levels deep in the (b) loop, and S3776's count went from 15 to 18. The loop is now `run_members`, which raises `member-unreadable`, and `main` catches a refusal from (a) or (b) once, flushing stdout first. Counted after S3776's rules: `main` 6, `run_members` 5, `unfetched` 3; `fetch_members` 13, unchanged. - SHOULD 2: `member-unreadable` is kept; lane 4 aligns slice E on it. - SHOULD 3: one in-process assert per file that `_is_a` answers False to a name holding a NUL (built with `chr(0)`), and a NUL row reads as no member there, never a traceback. - NIT 4: the placement summary's words are tested, in process and on the command line. - NIT 6: where (a) refuses a mount and git's update did not exit 0, git's own words are printed first, with no credential advice (`unfetched`); they were lost. Tested in process (git's update stood in for) and with a real mode. - NIT 7: the errno test is three whose names are true, in each file: each errno of nothing there is False, each of Windows' errors of nothing there is False whatever its errno, and every other errno the platform names is raised. - NIT 8: bootstrap.py says "(and on every directory above it inside the root)", `validate-pins.py`'s `_unreadable_file` exactly; siblings.py keeps its own, since its clones sit outside the holder. The new file is 57 tests; 31 mutants, lane 5's eleven among them, are each killed. Part of #430 (slice F; lane 1's plan, issuecomment-6251069236). Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- templates/family-root/scripts/bootstrap.py | 97 +++++++---- ...st_family_siblings_bootstrap_unreadable.py | 157 ++++++++++++++++-- 2 files changed, 210 insertions(+), 44 deletions(-) diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index ad945c11..c55bdb5d 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -281,13 +281,15 @@ def unreadable(code: str, named: str, exc: OSError, what: str) -> Refusal: `_cannot_read`, `: cannot be read ()`, with `named` relative to the holder and spelled; `what` says what was not done, and why the question matters. THE ADVICE FOLLOWS THE CAUSE, in - `validate-pins.py`'s words: a permission (`PermissionError`, which is - `EACCES` and `EPERM`) is put right by the mode or the owner, and any - other cause is `OTHER_REASON`, which is not told to `chmod`.""" + `validate-pins.py`'s `_unreadable_file`'s words, word for word, since + every path this command asks about is inside the holder: a permission + (`PermissionError`, which is `EACCES` and `EPERM`) is put right by the + mode or the owner, and any other cause is `OTHER_REASON`, which is not + told to `chmod`.""" advice = ( "Make what this user may not open readable by this user: `chmod u+r` " "on a file, `chmod u+rx` on a directory (and on every directory above " - "it), or fix who owns it. " + "it inside the root), or fix who owns it. " if isinstance(exc, PermissionError) else OTHER_REASON) return Refusal( code, @@ -343,7 +345,9 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: fetched only where there is none and git exited 0. The advice to set a credential is printed only where git asked for one (`credential_advice`, #839). Raises the `git-not-runnable` refusal - where git cannot be started (#694).""" + where git cannot be started (#694), and `member-unreadable` where a + row's mount cannot be asked about, after git's own words (`unfetched`, + #430).""" source, token = credential() args = ["git"] if token: @@ -362,7 +366,7 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: if refusal is None: raise raise refusal from exc - left, stopper, modules = unfetchable(root, rows, proc.returncode == 0) + left, stopper, modules = unfetched(root, rows, proc, source, token) if proc.returncode != 0: fetch_incomplete(proc, source, token, credential_advice(bool(left), stopper, modules)) @@ -386,6 +390,24 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: return [] +def unfetched(root: Path, rows: list[dict], + proc: subprocess.CompletedProcess, source: str, token: str + ) -> tuple[list[tuple[dict, str, str]], str | None, + Gitmodules]: + """`unfetchable`, asked after git's update `proc` (#430). Where it + refuses a mount it cannot ask about (`member-unreadable`) and git did + not exit 0, git's own words for the update are printed first + (`fetch_incomplete`), with no advice: what git said of that mount is + the same fact, the refusal after it names the cure, and no credential + is it. It refused before they were printed, and they were lost.""" + try: + return unfetchable(root, rows, proc.returncode == 0) + except Refusal: + if proc.returncode != 0: + fetch_incomplete(proc, source, token, "") + raise + + def fetch_incomplete(proc: subprocess.CompletedProcess, source: str, token: str, advice: str) -> None: """(a)'s report of a `git submodule update` that exited non-zero: git's @@ -1191,6 +1213,29 @@ def run_member_target(root: Path, row: dict, target: str) -> str | None: MAKE_ERROR_RE = re.compile(rb"\*\*\* \[[^\n]*\] Error (\d+)\s*$") +def run_members(root: Path, rows: list[dict], target: str, + skip: bool) -> list[str]: + """(b) Each member's own `make `, in row order: the failure lines + (`run_member_target`), or none where `skip` (`--skip-members`) or no row + is declared. Raises `member-unreadable` at the first member whose mount + or `Makefile` cannot be asked about (#430), and no member after it is + run: `main` refuses the run there, exit 2.""" + print(f"\n(b) each member's own `make {spelled(target)}`") + if skip: + print(" skipped (--skip-members)") + return [] + if not rows: + print(" no members declared. A family with none is empty, not wrong: " + "`family.py add` is what puts one here.") + return [] + failed = [] + for row in rows: + failure = run_member_target(root, row, target) + if failure: + failed.append(failure) + return failed + + def run_make(make: str, target: str, cwd: Path) -> tuple[int, int | None]: """(b) `make ` in a member, as `(make's exit code, the exit code of the member's command it names)`, the second None where make printed @@ -1266,34 +1311,20 @@ def main(argv: list[str] | None = None) -> int: print("\n(a) members and their legs") failed: list[str] = [] - if args.no_fetch: - print(" skipped (--no-fetch)") - else: - try: + try: + if args.no_fetch: + print(" skipped (--no-fetch)") + else: failed += fetch_members(root, rows) - except Refusal as exc: - print(str(exc), file=sys.stderr) - return 2 - - print(f"\n(b) each member's own `make {spelled(args.members_make)}`") - if args.skip_members: - print(" skipped (--skip-members)") - elif not rows: - print(" no members declared. A family with none is empty, not wrong: " - "`family.py add` is what puts one here.") - else: - for row in rows: - try: - failure = run_member_target(root, row, args.members_make) - except Refusal as exc: - # Only `member-unreadable` reaches here (#430). Stdout - # first, so a log of both streams reads in the order the run - # went. - sys.stdout.flush() - print(str(exc), file=sys.stderr) - return 2 - if failure: - failed.append(failure) + failed += run_members(root, rows, args.members_make, + args.skip_members) + except Refusal as exc: + # `git-not-runnable` from (a) (#694), or `member-unreadable` from + # (a) or (b) (#430). Stdout first, so a log of both streams reads + # in the order the run went. + sys.stdout.flush() + print(str(exc), file=sys.stderr) + return 2 print() if failed: diff --git a/tests/test_family_siblings_bootstrap_unreadable.py b/tests/test_family_siblings_bootstrap_unreadable.py index fe774046..94ab18cd 100644 --- a/tests/test_family_siblings_bootstrap_unreadable.py +++ b/tests/test_family_siblings_bootstrap_unreadable.py @@ -44,8 +44,10 @@ from __future__ import annotations import errno +import os import shutil import stat +import subprocess from pathlib import Path import pytest @@ -73,9 +75,22 @@ ABSENT_ROWS = pytest.mark.parametrize( "row", [NOBODY, THROUGH_A_FILE], ids=["no-such-path", "a-file-on-the-path"]) -#: The words a permission's remedy and any other cause's begin with. +#: The words a permission's remedy and any other cause's begin with: +#: siblings.py's, whose clones sit outside the holder, and bootstrap.py's, +#: `validate-pins.py`'s `_unreadable_file` word for word. CHMOD = "`chmod u+rx` on a directory (and on every directory above it)" +CHMOD_IN_ROOT = ("`chmod u+rx` on a directory (and on every directory above " + "it inside the root)") OTHER_CAUSE = "Where the operating system gave another reason" +#: The summary `make siblings` ends on where it refused a path (#430). +SUMMARY_TAIL = "; nothing was moved and no existing clone was touched" +#: The `stat` errnos that mean nothing is there, Windows' errors that mean +#: the same ("not ready", "invalid name", "cannot resolve"), and every +#: other errno this platform names. +ABSENT_ERRNOS = [errno.ENOENT, errno.ENOTDIR, errno.EBADF, errno.ELOOP] +ABSENT_WINERRORS = [21, 123, 1921] +OTHER_ERRNOS = sorted(set(errno.errorcode) - set(ABSENT_ERRNOS)) +BOTH_FILES = pytest.mark.parametrize("which", ["siblings", "bootstrap"]) @pytest.fixture(scope="module") @@ -359,6 +374,35 @@ def test_make_siblings_refuses_a_pinned_copy_beside_a_placed_clone(placed): assert "pinned members/IRSS CANNOT BE READ -" in flat_rows(result.stdout) assert "sibling IRSS present, fetched main" in flat_rows(result.stdout) assert "siblings ok" not in result.stdout + assert result.stderr.splitlines()[-1] == ( + f"siblings: 1 path(s) this user cannot read{SUMMARY_TAIL}") + + +@pytest.mark.parametrize("findings, refusals, code, counted", [ + (1, 0, 1, "1 finding(s)"), + (0, 2, 2, "2 path(s) this user cannot read"), + (1, 2, 2, "1 finding(s) and 2 path(s) this user cannot read"), +], ids=["a-finding", "two-refusals", "both"]) +def test_the_placement_summary_counts_what_it_could_not_read( + placement, tmp_path, capsys, findings, refusals, code, counted): + """`_report_placement`'s last line counts the findings, as it did, and + the paths it could not read, a sibling's or a pinned copy's, and the + run exits 2 where there is any.""" + rows = [] + for number in range(findings): + row = placement.Sibling(f"F{number}", tmp_path) + row.finding = f"F{number}: a finding" + rows.append(row) + pinned = [] + for number in range(refusals): + row = placement.Sibling(f"members/R{number}", tmp_path) + row.refusal = placement.member_unreadable( + f"members/R{number}/.git", + PermissionError(errno.EACCES, DENIED), "What was not done.") + pinned.append(row) + assert placement._report_placement(tmp_path, rows, False, pinned) == code + assert capsys.readouterr().err.splitlines()[-1] == ( + f"siblings: {counted}{SUMMARY_TAIL}") # --- bootstrap.py: `checked_out`, in `unfetchable` and `run_member_target` - @@ -386,6 +430,35 @@ def test_a_mount_whose_stat_is_refused_is_refused(boot, family, monkeypatch, assert refusal.code == MEMBER_UNREADABLE assert refusal.detail == f"members/IRRS/.git: cannot be read ({DENIED})" assert "the run stopped there" in refusal.remediation + assert CHMOD_IN_ROOT in refusal.remediation + + +@pytest.mark.parametrize("exit_code", [1, 0], ids=["git-failed", "git-ok"]) +def test_gits_own_words_come_before_a_mount_it_refuses( + boot, family, monkeypatch, capsys, exit_code): + """Where (a) refuses a mount and git's update did not exit 0, git's own + words are printed first, with no credential advice, then the refusal; + they were lost. Where git exited 0 there is nothing of git's to say. + In process, with git's update stood in for, on every platform.""" + _, rows = boot.members_of(family["root"]) + real = subprocess.run + + def git_update(args, *more, **kwargs): + if "submodule" in args: + return subprocess.CompletedProcess( + args, exit_code, "", "fatal: the update stopped here\n") + return real(args, *more, **kwargs) + + for name in boot.CREDENTIAL_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(boot.subprocess, "run", git_update) + refused_stat(monkeypatch, ".git") + refusal = refusal_of(boot, boot.fetch_members, family["root"], rows) + said = capsys.readouterr().err + assert refusal.detail == f"members/IRRS/.git: cannot be read ({DENIED})" + assert said == (" FETCH INCOMPLETE (git exit 1) using credential " + "source 'none':\nfatal: the update stopped here\n" + if exit_code else "") @needs_a_mode @@ -402,9 +475,14 @@ def test_make_bootstrap_refuses_a_mount_it_cannot_search(holder, relative, result = bootstrap(holder, allow_file_protocol()) no_traceback(result) assert result.returncode == 2, result.stdout + result.stderr - assert refusal_line(MEMBER_UNREADABLE, named) in result.stderr + refused = refusal_line(MEMBER_UNREADABLE, named) + assert refused in result.stderr assert "family bootstrap ok" not in result.stdout assert ": make bootstrap ---" not in result.stdout + assert CHMOD_IN_ROOT in result.stderr + assert "SHAPE_LEGS_TOKEN" not in result.stderr + fetch = result.stderr.find(" FETCH INCOMPLETE (git exit ") + assert -1 < fetch < result.stderr.index(refused), result.stderr @needs_a_mode @@ -517,12 +595,69 @@ def test_a_stat_that_fails_for_another_reason_is_refused_sans_chmod( assert module.OTHER_REASON in refusal.remediation -def test_every_absent_errno_is_absent_and_every_other_is_raised(boot, - placement): - """Each file's own `_is_a` answers as `bump-leg.py`'s does: the four - errnos of nothing there are False, any other `OSError` is raised.""" - for module in (boot, placement): - assert module._ABSENT_ERRNOS == (errno.ENOENT, errno.ENOTDIR, - errno.EBADF, errno.ELOOP) - assert module._ABSENT_WINERRORS == (21, 123, 1921) - assert module._is_a(Path("no-such-name-here"), stat.S_ISREG) is False +@BOTH_FILES +@pytest.mark.parametrize("code", ABSENT_ERRNOS, + ids=[errno.errorcode[code] for code in ABSENT_ERRNOS]) +def test_each_errno_of_nothing_there_is_absent(placement, boot, monkeypatch, + which, code): + """Each file's own `_is_a` answers as `bump-leg.py`'s does: a `stat` + that says nothing is at the name is False, as `is_file()` said it.""" + module = placement if which == "siblings" else boot + assert tuple(ABSENT_ERRNOS) == module._ABSENT_ERRNOS + refusing(monkeypatch, "stat", "probe", OSError(code, os.strerror(code))) + assert module._is_a(Path("probe"), stat.S_ISREG) is False + + +@BOTH_FILES +@pytest.mark.parametrize("winerror", ABSENT_WINERRORS) +def test_each_windows_error_of_nothing_there_is_absent( + placement, boot, monkeypatch, which, winerror): + """So is a `stat` whose Windows error says nothing is there, whatever + its errno: here `EINVAL`, which is raised without one. In process, on + every platform.""" + module = placement if which == "siblings" else boot + assert tuple(ABSENT_WINERRORS) == module._ABSENT_WINERRORS + real = Path.stat + + def refused(self, *args, **kwargs): + if self.name != "probe": + return real(self, *args, **kwargs) + error = OSError(errno.EINVAL, os.strerror(errno.EINVAL), str(self)) + error.winerror = winerror + raise error + + monkeypatch.setattr(Path, "stat", refused) + assert module._is_a(Path("probe"), stat.S_ISREG) is False + + +@BOTH_FILES +def test_every_other_errno_is_raised(placement, boot, monkeypatch, which): + """And a `stat` refused with any other errno this platform names + (`EACCES`, `EPERM`, `EIO`, `ENAMETOOLONG` and `EINVAL` among them) is + raised, for the caller to refuse by name.""" + module = placement if which == "siblings" else boot + raised = [] + for code in OTHER_ERRNOS: + refusing(monkeypatch, "stat", "probe", + OSError(code, os.strerror(code))) + try: + module._is_a(Path("probe"), stat.S_ISREG) + except OSError as exc: + raised.append(exc.errno) + assert raised == OTHER_ERRNOS + + +@BOTH_FILES +def test_a_name_the_system_cannot_take_is_no_file(placement, boot, family, + which): + """A NUL in a name is a `ValueError` the system raises before any + `stat`, and `_is_a` answers False, as `exists()` did: a row whose + `path:` holds one reads as no member there, never a traceback.""" + module = placement if which == "siblings" else boot + nul = chr(0) + assert module._is_a(Path(f"a{nul}b"), stat.S_ISREG) is False + row = {"project": "IRSS", "path": f"members/IR{nul}SS"} + if which == "siblings": + assert placement.tracking_branch(family["root"], row)[0] is None + else: + assert boot.checked_out(family["root"], row) is False