diff --git a/postgit/README.md b/postgit/README.md index a6f64f7..8f73dbe 100644 --- a/postgit/README.md +++ b/postgit/README.md @@ -4,3 +4,13 @@ Tool for transferring data between Git (Git's Object DB) and Postgres. - Use git history to populate tables and audit tables - Use audit tables to populate git history - Repo settings, notes, and other misc. info can be stored (and read by distgit) + +Fundamentally, a git repo unpacks into a single schema with the following tables: +- objects +- refs +And the following views: +- blobs +- trees +- commits +- tags (Note this refers to Annotated Tags, not a reference in `refs/tags/`.) +You can pack as many such schemas into a database as you want, but the default behavior of PostGit's PG Export is to create a new database and export to the "public" schema. diff --git a/postgit/__init__.py b/postgit/__init__.py new file mode 100644 index 0000000..b9e2a30 --- /dev/null +++ b/postgit/__init__.py @@ -0,0 +1,31 @@ +from .adapters import ( + GitObjectType, + OidBinaryDumper, + OidDumper, + object_data_from_pygit2, + object_type_from_pygit2, + objects_row_insert, + oid_from_db, + postgit_dict_row, + postgit_tuple_row, + ref_storage_columns, + register_adapters, + register_adapters_async, + regtype_git_object_type, +) + +__all__ = [ + "GitObjectType", + "OidBinaryDumper", + "OidDumper", + "object_data_from_pygit2", + "object_type_from_pygit2", + "objects_row_insert", + "oid_from_db", + "postgit_dict_row", + "postgit_tuple_row", + "ref_storage_columns", + "register_adapters", + "register_adapters_async", + "regtype_git_object_type", +] diff --git a/postgit/adapters.py b/postgit/adapters.py new file mode 100644 index 0000000..8df8831 --- /dev/null +++ b/postgit/adapters.py @@ -0,0 +1,218 @@ +""" +Psycopg 3 adapters and row factories for PostGit tables (see schema.sql). + +- ``pygit2.Oid`` binds to ``bytea`` (``Oid.raw``) and, via row factories, is + reconstructed from ``oid`` / ``target_oid`` columns (not from arbitrary + ``bytea``, e.g. ``object_data``). +- ``git_object_type`` maps to ``GitObjectType`` (``IntEnum`` aligned with + libgit2 / ``pygit2.GIT_OBJECT_*``). +""" + +from __future__ import annotations + +from enum import IntEnum +from typing import Any, Sequence + +import pygit2 +from psycopg import _oids +from psycopg._cursor_base import BaseCursor +from psycopg.abc import AdaptContext +from psycopg.adapt import Buffer, Dumper +from psycopg.connection import Connection +from psycopg.connection_async import AsyncConnection +from psycopg.pq import Format +from psycopg.rows import RowMaker, dict_row, tuple_row +from psycopg.types.enum import EnumInfo, register_enum +from psycopg.types.string import BytesDumper + +# libgit2 GIT_REFERENCE_DIRECT / GIT_REFERENCE_SYMBOLIC +_GIT_REFERENCE_OID = 1 +_GIT_REFERENCE_SYMBOLIC = 2 + +_OID_COLUMN_NAMES = frozenset({"oid", "target_oid"}) + + +class GitObjectType(IntEnum): + """Matches ``git_object_type`` in PostgreSQL and ``pygit2.GIT_OBJECT_*``.""" + + COMMIT = 1 + TREE = 2 + BLOB = 3 + TAG = 4 + + +_GIT_OBJECT_TYPE_PG_MAP = { + GitObjectType.COMMIT: "commit", + GitObjectType.TREE: "tree", + GitObjectType.BLOB: "blob", + GitObjectType.TAG: "tag", +} + + +class OidDumper(Dumper): + """Encode ``pygit2.Oid`` as PostgreSQL ``bytea`` (text parameter format).""" + + oid = _oids.BYTEA_OID + + def __init__(self, cls: type, context: AdaptContext | None = None): + super().__init__(cls, context) + self._bytes = BytesDumper(bytes, context) + + def dump(self, obj: pygit2.Oid) -> Buffer | None: + return self._bytes.dump(obj.raw) + + +class OidBinaryDumper(Dumper): + """Encode ``pygit2.Oid`` as PostgreSQL ``bytea`` (binary parameter format).""" + + format = Format.BINARY + oid = _oids.BYTEA_OID + + def dump(self, obj: pygit2.Oid) -> Buffer | None: + return obj.raw + + +def register_adapters( + conn: Connection[Any], + *, + git_object_type_regtype: str = "git_object_type", +) -> None: + """ + Register dumpers/loaders on ``conn`` for ``pygit2.Oid`` and ``git_object_type``. + + Call once per sync connection after the schema exists (enum type present). + + :param git_object_type_regtype: argument to :meth:`EnumInfo.fetch` (e.g. + :func:`regtype_git_object_type` if the type is not on ``search_path``). + """ + info = EnumInfo.fetch(conn, git_object_type_regtype) + if info is None: + raise LookupError( + "PostgreSQL type git_object_type not found; apply schema.sql first " + "(use the regtype your search_path resolves, e.g. public.git_object_type)." + ) + register_enum( + info, + conn, + enum=GitObjectType, + mapping=_GIT_OBJECT_TYPE_PG_MAP, + ) + conn.adapters.register_dumper(pygit2.Oid, OidDumper) + conn.adapters.register_dumper(pygit2.Oid, OidBinaryDumper) + + +async def register_adapters_async( + conn: AsyncConnection[Any], + *, + git_object_type_regtype: str = "git_object_type", +) -> None: + """Async variant of :func:`register_adapters`.""" + info = await EnumInfo.fetch(conn, git_object_type_regtype) + if info is None: + raise LookupError( + "PostgreSQL type git_object_type not found; apply schema.sql first." + ) + register_enum( + info, + conn, + enum=GitObjectType, + mapping=_GIT_OBJECT_TYPE_PG_MAP, + ) + conn.adapters.register_dumper(pygit2.Oid, OidDumper) + conn.adapters.register_dumper(pygit2.Oid, OidBinaryDumper) + + +def oid_from_db(value: bytes | memoryview) -> pygit2.Oid: + """Build ``Oid`` from a ``bytea`` value (e.g. manual handling without a row factory).""" + return pygit2.Oid(raw=bytes(value)) + + +def postgit_tuple_row(cursor: BaseCursor[Any, Any]) -> RowMaker[tuple[Any, ...]]: + """ + Like :func:`psycopg.rows.tuple_row`, but turns ``oid`` and ``target_oid`` + columns into ``pygit2.Oid``. + """ + if not cursor.description: + return tuple_row(cursor) + names = [d.name for d in cursor.description] + idxs = [i for i, n in enumerate(names) if n in _OID_COLUMN_NAMES] + if not idxs: + return tuple_row(cursor) + + def rowmaker(values: Sequence[Any]) -> tuple[Any, ...]: + out = list(values) + for i in idxs: + v = out[i] + if isinstance(v, (bytes, memoryview)): + out[i] = oid_from_db(v) + return tuple(out) + + return rowmaker + + +def postgit_dict_row(cursor: BaseCursor[Any, Any]) -> RowMaker[dict[str, Any]]: + """ + Like :func:`psycopg.rows.dict_row`, but turns ``oid`` and ``target_oid`` + values into ``pygit2.Oid``. + """ + inner = dict_row(cursor) + if cursor.description is None: + return inner + names = [d.name for d in cursor.description] + idxs = [i for i, n in enumerate(names) if n in _OID_COLUMN_NAMES] + if not idxs: + return inner + + def rowmaker(values: Sequence[Any]) -> dict[str, Any]: + d = dict(zip(names, values)) + for k in _OID_COLUMN_NAMES & d.keys(): + v = d[k] + if isinstance(v, (bytes, memoryview)): + d[k] = oid_from_db(v) + return d + + return rowmaker + + +def ref_storage_columns( + ref: pygit2.Reference, +) -> tuple[pygit2.Oid | None, str | None]: + """ + Values for ``(target_oid, symbolic_target)`` on the ``refs`` table. + + Matches :func:`register_adapters` expectations: direct refs supply ``Oid``; + symbolic refs supply the target ref name as ``str``. + """ + if ref.type == _GIT_REFERENCE_SYMBOLIC: + t = ref.target + if not isinstance(t, str): + t = str(t) + return None, t + if ref.type == _GIT_REFERENCE_OID: + tgt = ref.target + if not isinstance(tgt, pygit2.Oid): + tgt = pygit2.Oid(hex=str(tgt)) + return tgt, None + raise TypeError(f"unsupported reference type: {ref.type!r}") + + +def object_type_from_pygit2(obj: pygit2.Object) -> GitObjectType: + """Map ``Object.type`` (libgit2 kind int) to :class:`GitObjectType`.""" + return GitObjectType(obj.type) + + +def object_data_from_pygit2(obj: pygit2.Object) -> bytes: + """Object payload for ``objects.object_data`` (same as ``Object.read_raw()``).""" + return obj.read_raw() + + +def objects_row_insert( + obj: pygit2.Object, +) -> tuple[pygit2.Oid, GitObjectType, bytes]: + """``(oid, object_type, object_data)`` for inserting into ``objects``.""" + return obj.id, object_type_from_pygit2(obj), object_data_from_pygit2(obj) + + +def regtype_git_object_type(schema: str = "public") -> str: + """Qualified type name for :func:`EnumInfo.fetch` if ``search_path`` is not set.""" + return f"{schema}.git_object_type" diff --git a/postgit/requirements.txt b/postgit/requirements.txt new file mode 100644 index 0000000..2e832ac --- /dev/null +++ b/postgit/requirements.txt @@ -0,0 +1,2 @@ +psycopg[binary]>=3.1 +pygit2>=1.12.0 diff --git a/postgit/schema.sql b/postgit/schema.sql new file mode 100644 index 0000000..336fda0 --- /dev/null +++ b/postgit/schema.sql @@ -0,0 +1,45 @@ +-- PostGit: one Git repository maps to a PostgreSQL schema (e.g. public). +CREATE TYPE git_object_type AS ENUM ('commit', 'tree', 'blob', 'tag'); +COMMENT ON TYPE git_object_type IS 'libgit2 object types, 1=commit, 2=tree, 3=blob, 4=(annotated) tag.'; + +CREATE TABLE objects ( + oid bytea NOT NULL + CHECK (octet_length(oid) IN (20, 32)) + PRIMARY KEY, + object_type git_object_type NOT NULL, + object_data bytea NOT NULL +); + +CREATE INDEX objects_object_type_idx ON objects (object_type); + +CREATE TABLE refs ( + ref_name text NOT NULL PRIMARY KEY, + target_oid bytea NULL + CHECK (target_oid IS NULL OR octet_length(target_oid) IN (20, 32)) + REFERENCES objects (oid) ON DELETE RESTRICT, + symbolic_target text NULL, + CONSTRAINT refs_direct_or_symbolic_chk CHECK ( + (symbolic_target IS NULL AND target_oid IS NOT NULL) + OR (symbolic_target IS NOT NULL AND target_oid IS NULL) + ) +); + +CREATE VIEW blobs AS +SELECT oid, object_type, object_data +FROM objects +WHERE object_type = 'blob'; + +CREATE VIEW trees AS +SELECT oid, object_type, object_data +FROM objects +WHERE object_type = 'tree'; + +CREATE VIEW commits AS +SELECT oid, object_type, object_data +FROM objects +WHERE object_type = 'commit'; + +CREATE VIEW tags AS +SELECT oid, object_type, object_data +FROM objects +WHERE object_type = 'tag';