Migration module

The v1-to-v2 upgrade behind research-tracker migrate. See Migration for the command and the backup directory.

research_tracker.migrate.MigrationError

Bases: RuntimeError

Raised when a store cannot be migrated rather than left half-rewritten.

Source code in src/research_tracker/migrate.py

class MigrationError(RuntimeError):
    """Raised when a store cannot be migrated rather than left half-rewritten."""

research_tracker.migrate.migrate_store(root, *, dry_run=False, backup=True, progress=None)

Upgrade the schema-version-1 Parquet store at root to the SQLite backend.

Steps, in order: refuse anything that is not a v1 store; read and validate every Parquet table; when backup is set move the Parquet files into parquet-v1/; build and bulk-load a v2 store in a staging directory (so ExperimentStore never sees the v1 marker at root); then publish the database and the v2 marker; finally read the row counts back.

With dry_run the tables are read, validated, and loaded into the staging store exactly as for a real run, and that staging store is then discarded, so nothing under root is modified. Any failure while publishing restores the Parquet files and the original marker, so a raised error never leaves a partially-migrated store.

Source code in src/research_tracker/migrate.py

def migrate_store(
    root: Path,
    *,
    dry_run: bool = False,
    backup: bool = True,
    progress: Callable[[str], None] | None = None,
) -> dict[str, int]:
    """
    Upgrade the schema-version-1 Parquet store at `root` to the SQLite backend.

    Steps, in order: refuse anything that is not a v1 store; read and validate
    every Parquet table; when `backup` is set move the Parquet files into
    `parquet-v1/`; build and bulk-load a v2 store in a staging directory (so
    `ExperimentStore` never sees the v1 marker at `root`); then publish the
    database and the v2 marker; finally read the row counts back.

    With `dry_run` the tables are read, validated, and loaded into the staging
    store exactly as for a real run, and that staging store is then discarded,
    so nothing under `root` is modified. Any failure while publishing restores
    the Parquet files and the original marker, so a raised error never leaves a
    partially-migrated store.
    """
    root = Path(root)

    if not root.is_dir():
        raise MigrationError(f"{root} is not a directory; nothing to migrate.")

    version = _read_schema_version(root)

    if version != 1:
        raise MigrationError(
            f"{root} declares schema version {version}; only a schema-version-1 "
            "Parquet store can be migrated. This store needs no migration."
        )

    if _db_path(root).exists():
        raise MigrationError(
            f"{root} already contains store.sqlite. Refusing to migrate over an "
            "existing SQLite store; move or remove it first."
        )

    schema_bytes = _schema_path(root).read_bytes()

    frames = {kind: _read_v1_table(root, kind) for kind in _TABLE_KINDS}
    counts = {kind: len(frame) for kind, frame in frames.items()}

    _report(
        progress,
        "Read " + ", ".join(f"{kind}s={count}" for kind, count in counts.items()),
    )

    # A real run stages inside the store root so publishing the database is a
    # same-filesystem rename, which `os.replace` performs atomically; a store
    # on another volume would otherwise fail to migrate. A dry run writes
    # nothing back, so it stages in the system temp directory and therefore
    # never needs write access to the store root.
    staging_dir = None if dry_run else root
    staging_root = Path(tempfile.mkdtemp(prefix=".migrate-", dir=staging_dir))

    try:
        staging = ExperimentStore(staging_root)

        # The staging store is deleted before its atexit flush could re-create
        # a lock file inside the removed directory.
        atexit.unregister(staging.flush)

        return _migrate_frames(
            root,
            staging,
            frames,
            counts,
            schema_bytes,
            dry_run=dry_run,
            backup=backup,
            progress=progress,
        )
    finally:
        shutil.rmtree(staging_root, ignore_errors=True)

research_tracker.migrate.is_v1_store(root)

True when root declares schema version 1 (the Parquet layout).

Source code in src/research_tracker/migrate.py

def is_v1_store(root: Path) -> bool:
    """True when `root` declares schema version 1 (the Parquet layout)."""
    try:
        return _read_schema_version(Path(root)) == 1
    except MigrationError:
        return False

research_tracker.migrate.is_v2_store(root)

True when root is a complete schema-version-2 SQLite store.

Source code in src/research_tracker/migrate.py

def is_v2_store(root: Path) -> bool:
    """True when `root` is a complete schema-version-2 SQLite store."""
    root = Path(root)

    try:
        version = _read_schema_version(root)
    except MigrationError:
        return False

    return version == SCHEMA_VERSION and _db_path(root).is_file()

research_tracker.migrate.PARQUET_DIR_NAME = 'parquet-v1' module-attribute