public_git/bambuddy
1
0
Fork 0
mirror of https://github.com/maziggy/bambuddy.git synced 2026-10-19 07:33:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(backup): halve the provider round-trips, and stop losing commit metadata (#2656)

Browse Source 
The three remaining review items, all in the read path.

E1 — four provider calls where two would do. preview() called list_commits
twice: once inside _resolve_ref to turn HEAD into a SHA, once more at limit=20
purely to find the entry describing that same SHA. And list_tree's recursive
tree GET was thrown away, so fetch_files immediately fetched the identical tree
again to map path -> blob SHA. _resolve_ref now returns the entry it already
has, and list_tree returns its blob_shas map for fetch_files to take as an
optional argument. GitLab reads files by path and ignores it.

E2 — `commit: null` for a ref outside the 20 most recent. Two causes, and the
second is the one that actually bit: REF_PATTERN accepts a 7-character ref while
providers return the full 40, so the exact `==` in the scan never matched an
abbreviated SHA *even when the commit was in the window*. Fixed by prefix
comparison, plus a get_commit(ref) on the GitHub and GitLab backends for the
genuinely-outside-the-window case. Gitea and Forgejo inherit GitHub's. Still
best-effort: it is a subject line and a date, so a failed lookup renders the
preview without them rather than failing it.

E7 — the two tree readers disagreed, and each was wrong in the other's
direction. GitHub's recursive trees endpoint is not paginated and signals
overflow with truncated=true, which _blob_shas_at hard-fails on. Gitea and
Forgejo *do* page that endpoint, and inherited that single GET unchanged — so a
large backup repo returned only the first page and every category beyond it
looked absent from the commit. GiteaBackend now has its own paging
_blob_shas_at. GitLab had the mirror-image bug the review did not name: at its
50-page cap it exited through the while condition and returned success: True
with a silently partial path list. Both now fail loudly, which is what the
GitHub version was always doing.

Both halves of E7 are the same failure the module already refuses to allow: a
restore that skips categories and calls it "not present in this backup commit".

24 new or changed tests, all failing against this commit's parent.
This commit is contained in:
jmoore-skild
2026-08-04 08:57:34 -04:00
parent 945f50a1ab
commit 158301ac8a
7 changed files with 554 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/git_providers/gitlab.py
+73 -7
View File
@@ -185,6 +185,48 @@ class GitLabBackend(GitProviderBackend):
logger.exception("list_commits failed for %s branch=%s", repo_url, branch)
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "commits": []}
async def get_commit(self, repo_url: str, token: str, ref: str, client: httpx.AsyncClient) -> dict:
"""Read one commit's metadata directly, for refs outside the list window."""
try:
api_base = self.get_api_base(repo_url)
headers = self.get_headers(token)
encoded_path = self._encoded_project(repo_url)
response = await client.get(
f"{api_base}/projects/{encoded_path}/repository/commits/{urllib.parse.quote(ref, safe='')}",
headers=headers,
)
if response.status_code == 404:
return {"success": False, "message": f"Commit '{ref}' not found in the repository", "commit": None}
if response.status_code != 200:
msg = f"Failed to read commit (HTTP {response.status_code}): {self._truncated_response_text(response)}"
logger.warning("get_commit %s ref=%s: %s", repo_url, ref, msg)
return {"success": False, "message": msg, "commit": None}
try:
data = response.json()
except ValueError:
return {"success": False, "message": "Non-JSON response reading commit", "commit": None}
sha = data.get("id") if isinstance(data, dict) else None
if not isinstance(sha, str) or not sha:
return {"success": False, "message": "Commit response carried no SHA", "commit": None}
# GitLab flattens author/date onto the commit, as in list_commits.
return {
"success": True,
"message": "OK",
"commit": {
"sha": sha,
"message": data.get("message") or "",
"author": data.get("author_name") or "",
"date": data.get("committed_date") or data.get("created_at") or "",
},
}
except Exception as e:
logger.exception("get_commit failed for %s ref=%s", repo_url, ref)
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "commit": None}
async def list_tree(
self,
repo_url: str,
@@ -200,9 +242,11 @@ class GitLabBackend(GitProviderBackend):
paths: list[str] = []
page = 1
complete = False
# GitLab's tree endpoint paginates instead of exposing a "truncated"
# flag, so walk pages until one comes back short. The page cap stops
# a malformed X-Next-Page loop from spinning forever.
# a malformed X-Next-Page loop from spinning forever — and reaching
# it is a failure, not a result: see the check after the loop.
while page <= 50:
response = await client.get(
f"{api_base}/projects/{encoded_path}/repository/tree",
@@ -214,20 +258,21 @@ class GitLabBackend(GitProviderBackend):
"success": False,
"message": f"Commit or tree '{ref}' not found in the repository",
"paths": [],
"blob_shas": {},
}
if response.status_code != 200:
msg = (
f"Failed to list tree (HTTP {response.status_code}): {self._truncated_response_text(response)}"
)
logger.warning("list_tree %s ref=%s: %s", repo_url, ref, msg)
return {"success": False, "message": msg, "paths": []}
return {"success": False, "message": msg, "paths": [], "blob_shas": {}}
try:
data = response.json()
except ValueError:
return {"success": False, "message": "Non-JSON response listing tree", "paths": []}
return {"success": False, "message": "Non-JSON response listing tree", "paths": [], "blob_shas": {}}
if not isinstance(data, list):
return {"success": False, "message": "Unexpected shape listing tree", "paths": []}
return {"success": False, "message": "Unexpected shape listing tree", "paths": [], "blob_shas": {}}
for item in data:
if isinstance(item, dict) and item.get("type") == "blob":
@@ -236,14 +281,29 @@ class GitLabBackend(GitProviderBackend):
paths.append(path)
if len(data) < 100:
complete = True
break
page += 1
return {"success": True, "message": "OK", "paths": sorted(paths)}
if not complete:
# Falling out of the loop means the last page was full and there
# are more. Returning success here would hand the restore a
# silently partial path list, and it would then report the
# categories it could not see as "not present in this commit" —
# the same failure GitHub's truncated=true check refuses to allow.
msg = (
"Repository tree exceeds the listing limit (more than 5000 files), so the backup "
"contents cannot be enumerated reliably. Rotate the backup repository."
)
logger.warning("list_tree %s ref=%s: %s", repo_url, ref, msg)
return {"success": False, "message": msg, "paths": [], "blob_shas": {}}
# GitLab reads files by path, so there is no blob-SHA map to share.
return {"success": True, "message": "OK", "paths": sorted(paths), "blob_shas": {}}
except Exception as e:
logger.exception("list_tree failed for %s ref=%s", repo_url, ref)
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "paths": []}
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "paths": [], "blob_shas": {}}
async def fetch_files(
self,
@@ -252,8 +312,14 @@ class GitLabBackend(GitProviderBackend):
ref: str,
paths: list[str],
client: httpx.AsyncClient,
blob_shas: dict[str, str] | None = None,
) -> dict:
"""Read ``paths`` at ``ref`` via /repository/files/{path}."""
"""Read ``paths`` at ``ref`` via /repository/files/{path}.
``blob_shas`` is accepted for interface parity and ignored: this backend
addresses files by path, so it never needed the tree listing that makes
the map worth passing.
"""
try:
api_base = self.get_api_base(repo_url)
headers = self.get_headers(token)