feat(forge): GitHub adapter — second implementation keeps the seam a contract (#2693, milestone 288 step 8)
CI & Build / Python lint (push) Successful in 4s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / integration (push) Successful in 27s
CI & Build / TypeScript typecheck (push) Successful in 34s
CI & Build / Python tests (push) Successful in 1m8s
CI & Build / Build & push image (push) Successful in 45s
CI & Build / Python lint (push) Successful in 4s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / integration (push) Successful in 27s
CI & Build / TypeScript typecheck (push) Successful in 34s
CI & Build / Python tests (push) Successful in 1m8s
CI & Build / Build & push image (push) Successful in 45s
ForgeAdapter is now a named base class carrying the shared plumbing (host join, error taxonomy, contents decoding, archive, default_branch, latest_commit); GiteaForge keeps its exact behavior and GitHubForge joins with the real differences: api.github.com / GHE /api/v3 host mapping, Bearer auth, a commits call for the provenance stamp (GitHub's contents payload only carries the blob sha), and the codeload tarball redirect. The contract grew latest_commit, and with it the cached-SHA short-circuit in pull-time freshness: a stored provenance commit that still heads the recorded path confirms 'current' without a content transfer — the economy that fits pulls inside GitHub's rate limits; every surprise falls back to the full fetch. Webhook deliveries now also accept X-Hub-Signature-256 (sha256=<hex>); the payload shape was already common. Settings card copy covers both forges' token scopes; the kind selector already flowed from the server. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -33,7 +33,7 @@ import re
|
||||
import tarfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from scribe.services.forge import GiteaForge, get_forge
|
||||
from scribe.services.forge import ForgeAdapter, get_forge
|
||||
from scribe.services.repo_bindings import keys_for_project
|
||||
from scribe.services.settings import get_setting, set_setting
|
||||
|
||||
@@ -252,7 +252,7 @@ async def _recorded_locations(user_id: int, project_id: int) -> list[tuple[str,
|
||||
|
||||
|
||||
async def compute_coverage(
|
||||
user_id: int, project_id: int, *, forge: GiteaForge | None = None
|
||||
user_id: int, project_id: int, *, forge: ForgeAdapter | None = None
|
||||
) -> dict | None:
|
||||
"""Measure a project's pattern-library coverage against its bound repos.
|
||||
|
||||
@@ -299,7 +299,7 @@ async def compute_coverage(
|
||||
|
||||
|
||||
async def refresh_coverage(
|
||||
user_id: int, project_id: int, *, forge: GiteaForge | None = None
|
||||
user_id: int, project_id: int, *, forge: ForgeAdapter | None = None
|
||||
) -> dict | None:
|
||||
"""Compute and cache. The only writer of the cache key."""
|
||||
coverage = await compute_coverage(user_id, project_id, forge=forge)
|
||||
|
||||
+185
-47
@@ -16,8 +16,9 @@ Design constraints, in force everywhere below:
|
||||
method to misuse. The token an operator mints for it only ever needs read
|
||||
scope, and the docs say so.
|
||||
- The contract stays as small as its consumers (steps 5-7): read_file /
|
||||
default_branch / resolve_repo / check. GitHub later implements this same
|
||||
contract (step 8); resist widening it speculatively.
|
||||
latest_commit / archive / default_branch / resolve_repo / check. Two
|
||||
implementations (Gitea, GitHub — step 8) keep it honest; resist widening
|
||||
it speculatively.
|
||||
- Repo identity is the repo-binding key — `normalize_repo_key`'s
|
||||
host/owner/repo — so the join between a snippet's recorded repo and the
|
||||
forge needs no new identity scheme. The host segment selects whether THIS
|
||||
@@ -50,8 +51,8 @@ FORGE_KIND_KEY = "forge_kind"
|
||||
FORGE_BASE_URL_KEY = "forge_base_url"
|
||||
FORGE_TOKEN_KEY = "forge_token"
|
||||
|
||||
# Kinds an instance can configure. GitHub joins in step 8 of milestone 288.
|
||||
FORGE_KINDS = ("gitea",)
|
||||
# Kinds an instance can configure. Matches _FORGE_CLASSES below.
|
||||
FORGE_KINDS = ("gitea", "github")
|
||||
|
||||
# Total budget per forge call. Consumers either have a cache to fall back to
|
||||
# (step 5) or a user watching a button (the test probe) — neither tolerates a
|
||||
@@ -89,15 +90,19 @@ def _host_of(url: str) -> str:
|
||||
return (urlsplit(url).hostname or "").lower()
|
||||
|
||||
|
||||
class GiteaForge:
|
||||
"""The Gitea implementation of the forge contract, over its REST API.
|
||||
class ForgeAdapter:
|
||||
"""The shared plumbing of the forge contract; adapters supply the API
|
||||
base, auth headers, and any endpoint that differs.
|
||||
|
||||
`transport` exists for tests: httpx.MockTransport makes the contract
|
||||
testable without a live server or a new dependency. Production callers
|
||||
never pass it.
|
||||
"""
|
||||
|
||||
kind = "gitea"
|
||||
kind = ""
|
||||
# One "newest commit for this path" page — the endpoint is shared but the
|
||||
# page-size parameter is not, so each adapter names its own.
|
||||
_commit_page_params: dict = {}
|
||||
|
||||
def __init__(self, base_url: str, token: str, *, transport=None) -> None:
|
||||
self.base_url = (base_url or "").rstrip("/")
|
||||
@@ -125,11 +130,22 @@ class GiteaForge:
|
||||
return None
|
||||
return rest
|
||||
|
||||
def _api_base(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def _headers(self) -> dict:
|
||||
raise NotImplementedError
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
kwargs: dict = {
|
||||
"base_url": f"{self.base_url}/api/v1",
|
||||
"headers": {"Authorization": f"token {self._token}"},
|
||||
"base_url": self._api_base(),
|
||||
"headers": self._headers(),
|
||||
"timeout": _TIMEOUT,
|
||||
# GitHub serves tarballs via a 302 to codeload. httpx drops the
|
||||
# Authorization header on the cross-host hop, and GitHub's
|
||||
# redirect target carries its own short-lived token in the URL —
|
||||
# so following is both necessary there and harmless on Gitea.
|
||||
"follow_redirects": True,
|
||||
}
|
||||
if self._transport is not None:
|
||||
kwargs["transport"] = self._transport
|
||||
@@ -150,6 +166,87 @@ class GiteaForge:
|
||||
raise ForgeError(f"forge returned HTTP {resp.status_code} for {url}")
|
||||
return resp
|
||||
|
||||
def _decode_contents(self, payload, path: str) -> str:
|
||||
"""Both forges speak the same contents-API dialect: a base64 file
|
||||
object, a list for a directory."""
|
||||
if isinstance(payload, list):
|
||||
raise ForgeNotFound(f"{path} is a directory on the forge, not a file")
|
||||
if payload.get("type") != "file":
|
||||
raise ForgeNotFound(
|
||||
f"{path} is a {payload.get('type', 'non-file')} on the forge"
|
||||
)
|
||||
if payload.get("encoding") != "base64" or payload.get("content") is None:
|
||||
raise ForgeError(f"forge returned no readable content for {path}")
|
||||
try:
|
||||
return base64.b64decode(payload["content"]).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError) as exc:
|
||||
raise ForgeError(f"forge content for {path} is not utf-8 text") from exc
|
||||
|
||||
async def _newest_commit(
|
||||
self, client: httpx.AsyncClient, repo: str, path: str, ref: str
|
||||
) -> str:
|
||||
params: dict = {**self._commit_page_params, "path": path}
|
||||
if ref:
|
||||
params["sha"] = ref
|
||||
resp = await self._get(client, f"/repos/{repo}/commits", params=params)
|
||||
payload = resp.json()
|
||||
# Tolerant parse on purpose: the caller uses this as an optimization
|
||||
# and falls back to read_file, so a surprising payload must read as
|
||||
# "don't know", never break a pull.
|
||||
if isinstance(payload, list) and payload and isinstance(payload[0], dict):
|
||||
return str(payload[0].get("sha") or "")
|
||||
return ""
|
||||
|
||||
async def latest_commit(self, repo: str, path: str, ref: str = "") -> str:
|
||||
"""The newest commit touching ``path`` — "" when it can't be told.
|
||||
|
||||
The cached-SHA short-circuit (#2693): when a snippet's provenance
|
||||
already names a commit, this one small call can prove the file
|
||||
hasn't moved since — no content transfer, which is what keeps
|
||||
pull-time freshness inside GitHub's rate limits.
|
||||
"""
|
||||
async with self._client() as client:
|
||||
return await self._newest_commit(client, repo, path, ref)
|
||||
|
||||
def _archive_url(self, repo: str, ref: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
async def archive(self, repo: str, ref: str) -> bytes:
|
||||
"""The repo's content at ``ref`` as a gzipped tarball, in one request.
|
||||
|
||||
Coverage measurement (step 7) needs every source file's text; per-file
|
||||
reads would mean one API call per file, so the archive endpoint is the
|
||||
only shape that scales past toy repos. Callers must never run this in
|
||||
a request path — it moves the whole repo.
|
||||
"""
|
||||
async with self._client() as client:
|
||||
resp = await self._get(
|
||||
client, self._archive_url(repo, ref), timeout=_ARCHIVE_TIMEOUT
|
||||
)
|
||||
return resp.content
|
||||
|
||||
async def default_branch(self, repo: str) -> str:
|
||||
async with self._client() as client:
|
||||
resp = await self._get(client, f"/repos/{repo}")
|
||||
branch = (resp.json() or {}).get("default_branch") or ""
|
||||
if not branch:
|
||||
raise ForgeError(f"forge reported no default branch for {repo}")
|
||||
return branch
|
||||
|
||||
|
||||
class GiteaForge(ForgeAdapter):
|
||||
"""The Gitea implementation of the forge contract, over its REST API."""
|
||||
|
||||
kind = "gitea"
|
||||
# stat/verification/files add per-commit work Gitea skips when told to.
|
||||
_commit_page_params = {"limit": 1, "stat": "false"}
|
||||
|
||||
def _api_base(self) -> str:
|
||||
return f"{self.base_url}/api/v1"
|
||||
|
||||
def _headers(self) -> dict:
|
||||
return {"Authorization": f"token {self._token}"}
|
||||
|
||||
async def read_file(self, repo: str, path: str, ref: str = "") -> ForgeFile:
|
||||
"""Read one file's current content, with the commit it was served at.
|
||||
|
||||
@@ -164,18 +261,7 @@ class GiteaForge:
|
||||
params=params,
|
||||
)
|
||||
payload = resp.json()
|
||||
if isinstance(payload, list):
|
||||
raise ForgeNotFound(f"{path} is a directory on the forge, not a file")
|
||||
if payload.get("type") != "file":
|
||||
raise ForgeNotFound(
|
||||
f"{path} is a {payload.get('type', 'non-file')} on the forge"
|
||||
)
|
||||
if payload.get("encoding") != "base64" or payload.get("content") is None:
|
||||
raise ForgeError(f"forge returned no readable content for {path}")
|
||||
try:
|
||||
content = base64.b64decode(payload["content"]).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError) as exc:
|
||||
raise ForgeError(f"forge content for {path} is not utf-8 text") from exc
|
||||
content = self._decode_contents(payload, path)
|
||||
return ForgeFile(
|
||||
content=content,
|
||||
# last_commit_sha is the commit that last touched the file — the
|
||||
@@ -185,29 +271,8 @@ class GiteaForge:
|
||||
path=payload.get("path") or path,
|
||||
)
|
||||
|
||||
async def archive(self, repo: str, ref: str) -> bytes:
|
||||
"""The repo's content at ``ref`` as a gzipped tarball, in one request.
|
||||
|
||||
Coverage measurement (step 7) needs every source file's text; per-file
|
||||
reads would mean one API call per file, so the archive endpoint is the
|
||||
only shape that scales past toy repos. Callers must never run this in
|
||||
a request path — it moves the whole repo.
|
||||
"""
|
||||
async with self._client() as client:
|
||||
resp = await self._get(
|
||||
client,
|
||||
f"/repos/{repo}/archive/{quote(ref, safe='')}.tar.gz",
|
||||
timeout=_ARCHIVE_TIMEOUT,
|
||||
)
|
||||
return resp.content
|
||||
|
||||
async def default_branch(self, repo: str) -> str:
|
||||
async with self._client() as client:
|
||||
resp = await self._get(client, f"/repos/{repo}")
|
||||
branch = (resp.json() or {}).get("default_branch") or ""
|
||||
if not branch:
|
||||
raise ForgeError(f"forge reported no default branch for {repo}")
|
||||
return branch
|
||||
def _archive_url(self, repo: str, ref: str) -> str:
|
||||
return f"/repos/{repo}/archive/{quote(ref, safe='')}.tar.gz"
|
||||
|
||||
async def check(self) -> dict:
|
||||
"""Health probe for the settings test button: reach the forge AND
|
||||
@@ -222,6 +287,78 @@ class GiteaForge:
|
||||
}
|
||||
|
||||
|
||||
class GitHubForge(ForgeAdapter):
|
||||
"""The GitHub implementation — the second one, which is the point (#2693):
|
||||
it proves the seam is a contract rather than a Gitea-shaped hole. Works
|
||||
against github.com and GitHub Enterprise; the token is a fine-grained PAT
|
||||
with Contents: Read-only (or a classic token with `repo` read)."""
|
||||
|
||||
kind = "github"
|
||||
_commit_page_params = {"per_page": 1}
|
||||
|
||||
_API_VERSION = "2022-11-28"
|
||||
|
||||
def _api_base(self) -> str:
|
||||
# github.com's API lives on its own host; GitHub Enterprise serves
|
||||
# the same API under the instance at /api/v3.
|
||||
if self.host == "github.com":
|
||||
return "https://api.github.com"
|
||||
return f"{self.base_url}/api/v3"
|
||||
|
||||
def _headers(self) -> dict:
|
||||
return {
|
||||
"Authorization": f"Bearer {self._token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": self._API_VERSION,
|
||||
}
|
||||
|
||||
async def read_file(self, repo: str, path: str, ref: str = "") -> ForgeFile:
|
||||
"""Same contents-API dialect as Gitea, minus one field: GitHub's
|
||||
payload carries only the blob sha — a content address, not a point in
|
||||
history — so the provenance stamp costs one extra commits call. ""
|
||||
when even that can't be told; consumers already treat an empty stamp
|
||||
as "don't restamp"."""
|
||||
params = {"ref": ref} if ref else None
|
||||
async with self._client() as client:
|
||||
resp = await self._get(
|
||||
client,
|
||||
f"/repos/{repo}/contents/{quote(path, safe='/')}",
|
||||
params=params,
|
||||
)
|
||||
payload = resp.json()
|
||||
content = self._decode_contents(payload, path)
|
||||
try:
|
||||
commit_sha = await self._newest_commit(client, repo, path, ref)
|
||||
except ForgeError:
|
||||
commit_sha = ""
|
||||
return ForgeFile(
|
||||
content=content,
|
||||
commit_sha=commit_sha,
|
||||
path=payload.get("path") or path,
|
||||
)
|
||||
|
||||
def _archive_url(self, repo: str, ref: str) -> str:
|
||||
return f"/repos/{repo}/tarball/{quote(ref, safe='')}"
|
||||
|
||||
async def check(self) -> dict:
|
||||
"""GitHub has no /version endpoint; proving the token against /user
|
||||
is the whole probe, and the pinned API version stands in as the
|
||||
version string."""
|
||||
async with self._client() as client:
|
||||
user = (await self._get(client, "/user")).json() or {}
|
||||
return {
|
||||
"ok": True,
|
||||
"version": f"GitHub API {self._API_VERSION}",
|
||||
"username": user.get("login") or "",
|
||||
}
|
||||
|
||||
|
||||
_FORGE_CLASSES: dict[str, type[ForgeAdapter]] = {
|
||||
"gitea": GiteaForge,
|
||||
"github": GitHubForge,
|
||||
}
|
||||
|
||||
|
||||
async def forge_config() -> dict:
|
||||
"""The instance's forge configuration, DB-first with env fallback.
|
||||
|
||||
@@ -241,11 +378,12 @@ async def forge_config() -> dict:
|
||||
}
|
||||
|
||||
|
||||
async def get_forge(*, transport=None) -> GiteaForge | None:
|
||||
async def get_forge(*, transport=None) -> ForgeAdapter | None:
|
||||
"""The configured forge adapter, or None — and None means "behave exactly
|
||||
as if this module did not exist", which every consumer must honor."""
|
||||
cfg = await forge_config()
|
||||
if cfg["kind"] not in FORGE_KINDS:
|
||||
cls = _FORGE_CLASSES.get(cfg["kind"])
|
||||
if cls is None:
|
||||
if cfg["kind"]:
|
||||
# A kind we don't implement is a misconfiguration, not "off" —
|
||||
# say so once per lookup rather than silently reading as absent.
|
||||
@@ -256,4 +394,4 @@ async def get_forge(*, transport=None) -> GiteaForge | None:
|
||||
if not cfg["base_url"].startswith(("http://", "https://")):
|
||||
logger.warning("forge base URL %r has no http(s) scheme — forge disabled", cfg["base_url"])
|
||||
return None
|
||||
return GiteaForge(cfg["base_url"], cfg["token"], transport=transport)
|
||||
return cls(cfg["base_url"], cfg["token"], transport=transport)
|
||||
|
||||
@@ -1046,10 +1046,27 @@ async def attach_live_body(note, data: dict) -> None:
|
||||
data["body_freshness"] = "repo-not-on-this-forge"
|
||||
return
|
||||
|
||||
stored_prov_sha = (fields.get("provenance") or {}).get("commit_sha") or ""
|
||||
|
||||
async def _probe():
|
||||
# Cached-SHA short-circuit (#2693): provenance names the commit the
|
||||
# cached code was last confirmed at, so one cheap "newest commit
|
||||
# touching this path" call can prove the file hasn't moved since —
|
||||
# no content transfer. That economy is what fits pull-time freshness
|
||||
# inside GitHub's rate limits; it's merely nice on a self-hosted
|
||||
# Gitea. Any surprise (error, empty, mismatch) falls through to the
|
||||
# full fetch, which stays the authoritative path.
|
||||
if stored_prov_sha:
|
||||
try:
|
||||
head = await forge.latest_commit(repo, loc["path"])
|
||||
except ForgeError:
|
||||
head = ""
|
||||
if head and head == stored_prov_sha:
|
||||
return None
|
||||
return await forge.read_file(repo, loc["path"])
|
||||
|
||||
try:
|
||||
fetched = await asyncio.wait_for(
|
||||
forge.read_file(repo, loc["path"]), timeout=PULL_FETCH_BUDGET_S
|
||||
)
|
||||
fetched = await asyncio.wait_for(_probe(), timeout=PULL_FETCH_BUDGET_S)
|
||||
except ForgeNotFound:
|
||||
data["body_source"] = "cache"
|
||||
data["body_freshness"] = "missing"
|
||||
@@ -1064,6 +1081,14 @@ async def attach_live_body(note, data: dict) -> None:
|
||||
data["body_freshness"] = "unreachable"
|
||||
return
|
||||
|
||||
if fetched is None:
|
||||
# Unchanged since the provenance commit — confirmed against the
|
||||
# source without moving the file. Same stamp, so nothing to persist
|
||||
# (the same-sha rule); the body already reflects that commit.
|
||||
data["body_source"] = "forge"
|
||||
data["body_freshness"] = "current"
|
||||
return
|
||||
|
||||
cached = _normalized_code(fields.get("code") or "")
|
||||
if cached and cached in _normalized_code(fetched.content):
|
||||
data["body_source"] = "forge"
|
||||
|
||||
Reference in New Issue
Block a user