"""Forge adapter — optional server-side READ access to the operator's git forge. Step 4 of milestone 288 (#2689, decision #2686). The recorded location of a snippet is the source of truth for its code and the stored body is a cache; this module is the seam that lets the SERVER read that source of truth, so the cache can be refreshed at pull time (step 5), drift can be flagged from push webhooks (step 6), and coverage can be measured (step 7). Design constraints, in force everywhere below: - OPTIONAL per user (rule #115, sharpened by #2778). Connections are per-user keyring rows resolved by repo host on the PROJECT OWNER's keyring; `get_forges()` returns an empty selector when the owner has nothing configured, and every consumer must treat that as "keep today's behavior". A user who never configures a forge is not degraded — that is the baseline. - READ-ONLY by construction. The adapter exposes reads; there is no write method to misuse. The token an operator mints for it only ever needs read scope, and the docs say so. - The contract stays as small as its consumers (steps 5-7): read_file / latest_commit / archive / default_branch / resolve_repo / check. Two implementations (Gitea, GitHub — step 8) keep it honest; resist widening it speculatively. - Repo identity is the repo-binding key — `normalize_repo_key`'s host/owner/repo — so the join between a snippet's recorded repo and the forge needs no new identity scheme. The host segment selects whether THIS forge can serve the repo; the remainder is the API path. - Errors carry no token, ever, and failures are exceptions the caller handles — a consumer decides whether to fall back (pull-time fetch) or surface (settings test button); this module never silently swallows. This is also the codebase's first outbound-HTTP client with a real timeout convention (oauth.py predates it): short total timeout, no retries — every consumer has a fallback, so a slow forge must cost bounded time. """ from __future__ import annotations import base64 import binascii import logging from dataclasses import dataclass from urllib.parse import quote, urlsplit import httpx from sqlalchemy import select from scribe.config import Config from scribe.services.repo_bindings import normalize_repo_key logger = logging.getLogger(__name__) # Kinds a connection can use. Matches _FORGE_CLASSES below. FORGE_KINDS = ("gitea", "github") # Total budget per forge call. Consumers either have a cache to fall back to # (step 5) or a user watching a button (the test probe) — neither tolerates a # hung socket, and there is no retry: the fallback IS the retry policy. _TIMEOUT = httpx.Timeout(5.0) # Archive downloads move a whole-repo tarball and only ever run off the # request path (coverage recompute, step 7), so they get a bigger budget than # the per-file reads — but still a bound, because a hung background task # holds a connection slot as surely as a foreground one. _ARCHIVE_TIMEOUT = httpx.Timeout(60.0) class ForgeError(RuntimeError): """A forge call failed (network, auth, unexpected payload). Token-free.""" class ForgeNotFound(ForgeError): """The repo, path, or ref does not exist on the forge — the one failure consumers treat differently, because for a recorded snippet location it is itself a finding (the recorded path is gone).""" @dataclass(frozen=True) class ForgeFile: """One file read from the forge at a specific point in history.""" content: str # The commit the content was served at — what provenance stores (#2688). commit_sha: str path: str def host_of(url: str) -> str: """The lowercase hostname of a URL — the keyring's lookup key (#2778).""" return (urlsplit(url).hostname or "").lower() class ForgeAdapter: """The shared plumbing of the forge contract; adapters supply the API base, auth headers, and any endpoint that differs. `transport` exists for tests: httpx.MockTransport makes the contract testable without a live server or a new dependency. Production callers never pass it. """ kind = "" # One "newest commit for this path" page — the endpoint is shared but the # page-size parameter is not, so each adapter names its own. _commit_page_params: dict = {} def __init__(self, base_url: str, token: str, *, transport=None) -> None: self.base_url = (base_url or "").rstrip("/") self._token = token or "" self._transport = transport @property def host(self) -> str: return host_of(self.base_url) def resolve_repo(self, repo_or_url: str) -> str | None: """The forge-API repo path for a recorded repo — or None if this forge does not serve it. Accepts anything `normalize_repo_key` accepts (a raw remote URL or an already-normalized key). None is a NORMAL answer, not an error: a snippet recorded against github.com on an instance whose forge is a self-hosted Gitea is simply out of this forge's reach. """ key = normalize_repo_key(repo_or_url or "") if not key or "/" not in key: return None host, _, rest = key.partition("/") if host != self.host or "/" not in rest: return None return rest def _api_base(self) -> str: raise NotImplementedError def _headers(self) -> dict: raise NotImplementedError def _client(self) -> httpx.AsyncClient: kwargs: dict = { "base_url": self._api_base(), "headers": self._headers(), "timeout": _TIMEOUT, # GitHub serves tarballs via a 302 to codeload. httpx drops the # Authorization header on the cross-host hop, and GitHub's # redirect target carries its own short-lived token in the URL — # so following is both necessary there and harmless on Gitea. "follow_redirects": True, } if self._transport is not None: kwargs["transport"] = self._transport return httpx.AsyncClient(**kwargs) async def _get(self, client: httpx.AsyncClient, url: str, **kw) -> httpx.Response: try: resp = await client.get(url, **kw) except httpx.HTTPError as exc: # str(exc) on transport errors names hosts and timeouts, never # headers — safe, and the detail is what makes the test button useful. raise ForgeError(f"forge unreachable: {exc}") from exc if resp.status_code == 404: raise ForgeNotFound(f"not found on forge: {url}") if resp.status_code in (401, 403): raise ForgeError("forge rejected the token (check its read scope)") if resp.status_code >= 400: raise ForgeError(f"forge returned HTTP {resp.status_code} for {url}") return resp def _decode_contents(self, payload, path: str) -> str: """Both forges speak the same contents-API dialect: a base64 file object, a list for a directory.""" if isinstance(payload, list): raise ForgeNotFound(f"{path} is a directory on the forge, not a file") if payload.get("type") != "file": raise ForgeNotFound( f"{path} is a {payload.get('type', 'non-file')} on the forge" ) if payload.get("encoding") != "base64" or payload.get("content") is None: raise ForgeError(f"forge returned no readable content for {path}") try: return base64.b64decode(payload["content"]).decode("utf-8") except (binascii.Error, UnicodeDecodeError) as exc: raise ForgeError(f"forge content for {path} is not utf-8 text") from exc async def _newest_commit( self, client: httpx.AsyncClient, repo: str, path: str, ref: str ) -> str: params: dict = {**self._commit_page_params, "path": path} if ref: params["sha"] = ref resp = await self._get(client, f"/repos/{repo}/commits", params=params) payload = resp.json() # Tolerant parse on purpose: the caller uses this as an optimization # and falls back to read_file, so a surprising payload must read as # "don't know", never break a pull. if isinstance(payload, list) and payload and isinstance(payload[0], dict): return str(payload[0].get("sha") or "") return "" async def latest_commit(self, repo: str, path: str, ref: str = "") -> str: """The newest commit touching ``path`` — "" when it can't be told. The cached-SHA short-circuit (#2693): when a snippet's provenance already names a commit, this one small call can prove the file hasn't moved since — no content transfer, which is what keeps pull-time freshness inside GitHub's rate limits. """ async with self._client() as client: return await self._newest_commit(client, repo, path, ref) def _archive_url(self, repo: str, ref: str) -> str: raise NotImplementedError async def archive(self, repo: str, ref: str) -> bytes: """The repo's content at ``ref`` as a gzipped tarball, in one request. Coverage measurement (step 7) needs every source file's text; per-file reads would mean one API call per file, so the archive endpoint is the only shape that scales past toy repos. Callers must never run this in a request path — it moves the whole repo. """ async with self._client() as client: resp = await self._get( client, self._archive_url(repo, ref), timeout=_ARCHIVE_TIMEOUT ) return resp.content async def default_branch(self, repo: str) -> str: async with self._client() as client: resp = await self._get(client, f"/repos/{repo}") branch = (resp.json() or {}).get("default_branch") or "" if not branch: raise ForgeError(f"forge reported no default branch for {repo}") return branch class GiteaForge(ForgeAdapter): """The Gitea implementation of the forge contract, over its REST API.""" kind = "gitea" # stat/verification/files add per-commit work Gitea skips when told to. _commit_page_params = {"limit": 1, "stat": "false"} def _api_base(self) -> str: return f"{self.base_url}/api/v1" def _headers(self) -> dict: return {"Authorization": f"token {self._token}"} async def read_file(self, repo: str, path: str, ref: str = "") -> ForgeFile: """Read one file's current content, with the commit it was served at. `repo` is the API path from resolve_repo ("owner/repo"); `ref` is a branch, tag, or commit — empty means the default branch. """ params = {"ref": ref} if ref else None async with self._client() as client: resp = await self._get( client, f"/repos/{repo}/contents/{quote(path, safe='/')}", params=params, ) payload = resp.json() content = self._decode_contents(payload, path) return ForgeFile( content=content, # last_commit_sha is the commit that last touched the file — the # honest provenance stamp. The blob sha is a content address, not # a point in history, so it is deliberately not surfaced. commit_sha=payload.get("last_commit_sha") or "", path=payload.get("path") or path, ) def _archive_url(self, repo: str, ref: str) -> str: return f"/repos/{repo}/archive/{quote(ref, safe='')}.tar.gz" async def check(self) -> dict: """Health probe for the settings test button: reach the forge AND prove the token is accepted. Returns {"ok", "version", "username"}.""" async with self._client() as client: version = (await self._get(client, "/version")).json() or {} user = (await self._get(client, "/user")).json() or {} return { "ok": True, "version": version.get("version") or "", "username": user.get("login") or user.get("username") or "", } class GitHubForge(ForgeAdapter): """The GitHub implementation — the second one, which is the point (#2693): it proves the seam is a contract rather than a Gitea-shaped hole. Works against github.com and GitHub Enterprise; the token is a fine-grained PAT with Contents: Read-only (or a classic token with `repo` read).""" kind = "github" _commit_page_params = {"per_page": 1} _API_VERSION = "2022-11-28" def _api_base(self) -> str: # github.com's API lives on its own host; GitHub Enterprise serves # the same API under the instance at /api/v3. if self.host == "github.com": return "https://api.github.com" return f"{self.base_url}/api/v3" def _headers(self) -> dict: return { "Authorization": f"Bearer {self._token}", "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": self._API_VERSION, } async def read_file(self, repo: str, path: str, ref: str = "") -> ForgeFile: """Same contents-API dialect as Gitea, minus one field: GitHub's payload carries only the blob sha — a content address, not a point in history — so the provenance stamp costs one extra commits call. "" when even that can't be told; consumers already treat an empty stamp as "don't restamp".""" params = {"ref": ref} if ref else None async with self._client() as client: resp = await self._get( client, f"/repos/{repo}/contents/{quote(path, safe='/')}", params=params, ) payload = resp.json() content = self._decode_contents(payload, path) try: commit_sha = await self._newest_commit(client, repo, path, ref) except ForgeError: commit_sha = "" return ForgeFile( content=content, commit_sha=commit_sha, path=payload.get("path") or path, ) def _archive_url(self, repo: str, ref: str) -> str: return f"/repos/{repo}/tarball/{quote(ref, safe='')}" async def check(self) -> dict: """GitHub has no /version endpoint; proving the token against /user is the whole probe, and the pinned API version stands in as the version string.""" async with self._client() as client: user = (await self._get(client, "/user")).json() or {} return { "ok": True, "version": f"GitHub API {self._API_VERSION}", "username": user.get("login") or "", } _FORGE_CLASSES: dict[str, type[ForgeAdapter]] = { "gitea": GiteaForge, "github": GitHubForge, } def build_adapter( kind: str, base_url: str, token: str, *, transport=None ) -> ForgeAdapter | None: """One validated adapter from raw connection values, or None. None means "this connection cannot serve reads" — the same contract the old instance-wide lookup had, applied per keyring row. Misconfigurations are logged, never raised: a bad row must not break the reads the good rows can still serve. """ kind = (kind or "").strip().lower() base_url = (base_url or "").rstrip("/") cls = _FORGE_CLASSES.get(kind) if cls is None: if kind: # A kind we don't implement is a misconfiguration, not "off" — # say so once per lookup rather than silently reading as absent. logger.warning("unknown forge kind %r configured — connection disabled", kind) return None if not base_url or not token: return None if not base_url.startswith(("http://", "https://")): logger.warning("forge base URL %r has no http(s) scheme — connection disabled", base_url) return None return cls(base_url, token, transport=transport) @dataclass(frozen=True) class ForgeSelector: """The forge reads available to one project owner (#2778). Consumers ask it to serve a REPO, not to hand over "the forge": resolve() walks the owner's adapters and returns the (adapter, api_repo) pair for the first one whose host serves the repo — or None, which every consumer treats exactly as the old "no forge configured" state. An empty selector IS rule #115's baseline. """ adapters: tuple[ForgeAdapter, ...] = () @property def configured(self) -> bool: return bool(self.adapters) def resolve(self, repo_or_url: str) -> tuple[ForgeAdapter, str] | None: for adapter in self.adapters: repo = adapter.resolve_repo(repo_or_url) if repo is not None: return adapter, repo return None async def get_forges( owner_id: int, project_id: int | None = None, *, transport=None ) -> ForgeSelector: """The forge selector for reads on behalf of ``owner_id``'s records. The keyring model (#2778): every server-side forge read for a record runs on the PROJECT OWNER's connections, resolved by repo host — a forge token is a user's credential, and one user's reads must never ride another user's token. Pass the record's ``project_id`` so the per-project pin applies: a pinned project uses ONLY its pinned connection (explicit and auditable); a pin that no longer belongs to the owner (ownership moved) is ignored with a warning rather than honored across users. The env config (FORGE_KIND/FORGE_BASE_URL/FORGE_TOKEN) survives as an implicit keyring entry for ADMIN owners only — it is the operator's token, so it must not serve other users' reads — and a stored row for the same host beats it, because the UI writes rows. """ from scribe.models import async_session from scribe.models.forge_connection import ForgeConnection from scribe.models.project import Project from scribe.models.user import User async with async_session() as session: pinned_id = None if project_id: pinned_id = ( await session.execute( select(Project.forge_connection_id).where(Project.id == project_id) ) ).scalar_one_or_none() rows = list( ( await session.execute( select(ForgeConnection) .where(ForgeConnection.user_id == owner_id) .order_by(ForgeConnection.id) ) ).scalars().all() ) role = "" if Config.FORGE_KIND and Config.FORGE_BASE_URL and Config.FORGE_TOKEN: role = ( await session.execute(select(User.role).where(User.id == owner_id)) ).scalar_one_or_none() or "" if pinned_id: pin = next((r for r in rows if r.id == pinned_id), None) if pin is not None: adapter = build_adapter(pin.kind, pin.base_url, pin.token, transport=transport) return ForgeSelector((adapter,) if adapter is not None else ()) logger.warning( "project %s pins forge connection %s the owner (%s) does not hold — pin ignored", project_id, pinned_id, owner_id, ) adapters: list[ForgeAdapter] = [] for row in rows: adapter = build_adapter(row.kind, row.base_url, row.token, transport=transport) if adapter is not None: adapters.append(adapter) if role == "admin": env = build_adapter( Config.FORGE_KIND, Config.FORGE_BASE_URL, Config.FORGE_TOKEN, transport=transport, ) if env is not None and all(a.host != env.host for a in adapters): adapters.append(env) return ForgeSelector(tuple(adapters))