Source code for advanced_alchemy.utils.fixtures

"""Utilities for loading and writing fixture data in JSON, JSONL, and CSV formats."""

import contextlib
import csv
import gzip
import io
import re
import warnings
import zipfile
import zlib
from collections.abc import AsyncIterable, Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, Optional, Union

from advanced_alchemy.utils.deprecation import deprecated
from advanced_alchemy.utils.serialization import decode_json, encode_json
from advanced_alchemy.utils.sync_tools import async_

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Generator, Iterable, Iterator
    from contextlib import AbstractContextManager
    from typing import IO, BinaryIO, TextIO

    from anyio import Path as AsyncPath
    from typing_extensions import TypeGuard

__all__ = (
    "FixtureCompression",
    "FixtureFormat",
    "iter_fixture_async",
    "iter_fixture_sync",
    "open_fixture",
    "open_fixture_async",
    "open_fixture_sync",
    "write_fixture_async",
    "write_fixture_sync",
)

FixtureFormat = Literal["json", "jsonl", "csv"]
"""Supported fixture file serialization format (``"json"``, ``"jsonl"``, or ``"csv"``)."""
FixtureCompression = Literal["gzip", "zip"]
"""Supported fixture archive compression format (``"gzip"``, ``"zip"``, or ``None``)."""

_FORMAT_PRIORITY: "tuple[FixtureFormat, ...]" = ("json", "jsonl", "csv")
_COMPRESSION_SUFFIXES: "dict[FixtureCompression, str]" = {"gzip": ".gz", "zip": ".zip"}
_SUFFIX_COMPRESSIONS: "dict[str, FixtureCompression]" = {".gz": "gzip", ".zip": "zip"}
_SHARD_INDEX_WIDTH = 3
_TEXT_ENCODING = "utf-8-sig"
_READ_ERRORS = (OSError, zipfile.BadZipFile, gzip.BadGzipFile, EOFError, zlib.error)


[docs] def open_fixture_sync(fixtures_path: "Union[Path, AsyncPath]", fixture_name: str) -> Any: """Load a JSON, JSONL, or CSV fixture synchronously. Supports plain files, gzip-compressed files (``.gz``), and zip archives (``.zip``). Formats are resolved in priority order: ``json`` > ``jsonl`` > ``csv``. Both lowercase and uppercase fixture names are checked according to format and compression rules. Text is decoded using ``utf-8-sig``. Unsharded ``json`` fixtures return the decoded top-level JSON value unchanged. Unsharded ``jsonl`` and ``csv`` fixtures return a list of decoded records or dictionaries. Args: fixtures_path: Directory path containing fixture files. Can be a :class:`pathlib.Path` or :class:`anyio.Path` instance. fixture_name: Fixture base name without file extension. Raises: FileNotFoundError: If no matching fixture file exists. OSError: If an I/O or decompression error occurs while reading the file. ValueError: If a zip archive contains no matching member, a JSONL line contains invalid JSON, or a JSON fixture read as records is not an array. Returns: Any: Decoded JSON data or a list of records. Examples: >>> from pathlib import Path >>> fixtures_path = Path("./fixtures") >>> data = open_fixture_sync(fixtures_path, "users") >>> print(data) [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] """ files = _FixtureLocator(Path(fixtures_path), fixture_name).resolve() if len(files) == 1 and files[0].shard_index is None: return files[0].read() return [record for fixture_file in files for record in fixture_file.iter_records()]
[docs] async def open_fixture_async(fixtures_path: "Union[Path, AsyncPath]", fixture_name: str) -> Any: """Load a JSON, JSONL, or CSV fixture asynchronously. Offloads synchronous fixture resolution, decompression, and parsing to a worker thread via :func:`~advanced_alchemy.utils.sync_tools.async_`. Args: fixtures_path: Directory path containing fixture files. Can be a :class:`pathlib.Path` or :class:`anyio.Path` instance. fixture_name: Fixture base name without file extension. Raises: FileNotFoundError: If no matching fixture file exists. OSError: If an I/O or decompression error occurs while reading the file. ValueError: If a zip archive contains no matching member, a JSONL line contains invalid JSON, or a JSON fixture read as records is not an array. Returns: Any: Decoded JSON data or a list of records. Examples: >>> from pathlib import Path >>> fixtures_path = Path("./fixtures") >>> data = await open_fixture_async(fixtures_path, "users") >>> print(data) [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] """ return await async_(open_fixture_sync)(Path(str(fixtures_path)), fixture_name)
[docs] @deprecated("1.13.0", removal_in="2.0", alternative="open_fixture_sync", kind="function") def open_fixture(fixtures_path: "Union[Path, AsyncPath]", fixture_name: str) -> Any: """Load a JSON, JSONL, or CSV fixture synchronously. Deprecated since 1.13.0 and scheduled for removal in 2.0. Use :func:`open_fixture_sync` instead. Args: fixtures_path: Directory path containing fixture files. Can be a :class:`pathlib.Path` or :class:`anyio.Path` instance. fixture_name: Fixture base name without file extension. Raises: FileNotFoundError: If no matching fixture file exists. OSError: If an I/O or decompression error occurs while reading the file. ValueError: If a zip archive contains no matching member, a JSONL line contains invalid JSON, or a JSON fixture read as records is not an array. Returns: Any: Decoded JSON data or a list of records. """ return open_fixture_sync(fixtures_path, fixture_name)
[docs] def iter_fixture_sync( fixtures_path: "Union[Path, AsyncPath]", fixture_name: str, *, batch_size: int = 1000 ) -> "Iterator[list[Any]]": """Stream decoded fixture records in non-empty batches synchronously. Validates ``batch_size`` and resolves the fixture file or numbered shards eagerly at call time, then yields non-empty batches of at most ``batch_size`` records across shard boundaries. Closing the returned iterator early releases any open file handle immediately. Args: fixtures_path: Directory path containing fixture files. Can be a :class:`pathlib.Path` or :class:`anyio.Path` instance. fixture_name: Fixture base name without file extension. batch_size: Maximum number of records per yielded batch. Must be at least 1. Raises: FileNotFoundError: If no matching fixture file exists. OSError: If an I/O or decompression error occurs while reading a file. ValueError: If ``batch_size`` is less than 1, both an unsharded file and numbered shards exist, shard validation fails, a zip archive contains no matching member, a JSONL line contains invalid JSON, or a JSON fixture is not an array. Returns: Iterator[list[Any]]: Iterator yielding non-empty lists of records. """ return _FixtureLocator(Path(fixtures_path), fixture_name).iter_batches(batch_size)
[docs] async def iter_fixture_async( fixtures_path: "Union[Path, AsyncPath]", fixture_name: str, *, batch_size: int = 1000 ) -> "AsyncIterator[list[Any]]": """Stream decoded fixture records in non-empty batches asynchronously. Offloads fixture resolution, batch reads, and generator teardown to worker threads via :func:`~advanced_alchemy.utils.sync_tools.async_`. Validation and resolution run on the first iteration step. Callers exiting early must call ``aclose()`` or consume the iterator completely so underlying file handles are closed promptly. Args: fixtures_path: Directory path containing fixture files. Can be a :class:`pathlib.Path` or :class:`anyio.Path` instance. fixture_name: Fixture base name without file extension. batch_size: Maximum number of records per yielded batch. Must be at least 1. Raises: FileNotFoundError: If no matching fixture file exists. OSError: If an I/O or decompression error occurs while reading a file. ValueError: If ``batch_size`` is less than 1, both an unsharded file and numbered shards exist, shard validation fails, a zip archive contains no matching member, a JSONL line contains invalid JSON, or a JSON fixture is not an array. Yields: list[Any]: Non-empty batch of decoded records. """ locator = _FixtureLocator(Path(str(fixtures_path)), fixture_name) batches = await async_(locator.iter_batches)(batch_size) try: while True: batch = await async_(_FixtureLocator.next_batch)(batches) if batch is None: return yield batch finally: await async_(batches.close)()
[docs] def write_fixture_sync( fixtures_path: "Union[Path, AsyncPath]", fixture_name: str, records: "Iterable[Any]", *, file_format: FixtureFormat = "json", compression: "Optional[FixtureCompression]" = None, max_bytes: "Optional[int]" = None, max_records: "Optional[int]" = None, ) -> "list[Path]": """Write fixture records synchronously in JSON, JSONL, or CSV format. Streams encoded records into temporary ``.tmp`` files, atomically renames temporary files into place upon completion, and removes any stale fixture files matching ``fixture_name`` across all formats, compressions, and shard layouts. Args: fixtures_path: Target directory path for fixture files. Created if it does not exist. Can be a :class:`pathlib.Path` or :class:`anyio.Path`. fixture_name: Fixture base name without file extension. records: Iterable of records to write. file_format: Output format (``"json"``, ``"jsonl"``, or ``"csv"``). compression: Optional compression (``"gzip"``, ``"zip"``, or ``None``). max_bytes: Optional maximum on-disk byte size per shard. Must be at least 1. max_records: Optional maximum record count per shard. Must be at least 1. Raises: ValueError: If ``file_format``, ``compression``, ``max_bytes``, or ``max_records`` is invalid, or if CSV records introduce keys absent from the header. TypeError: If ``file_format`` is ``"csv"`` and a record is not a mapping. OSError: If writing, deleting stale files, or renaming files fails. Returns: list[Path]: Final written fixture paths in shard order. Examples: >>> from pathlib import Path >>> fixtures_path = Path("./fixtures") >>> paths = write_fixture_sync( ... fixtures_path, ... "users", ... [{"id": 1, "name": "Alice"}], ... file_format="jsonl", ... compression="gzip", ... ) """ writer = _FixtureWriter( Path(str(fixtures_path)), fixture_name, file_format=file_format, compression=compression, max_bytes=max_bytes, max_records=max_records, ) try: writer.write_many(records) return writer.commit() except BaseException: writer.abort() raise
[docs] async def write_fixture_async( fixtures_path: "Union[Path, AsyncPath]", fixture_name: str, records: "Union[Iterable[Any], AsyncIterable[Any]]", *, file_format: FixtureFormat = "json", compression: "Optional[FixtureCompression]" = None, max_bytes: "Optional[int]" = None, max_records: "Optional[int]" = None, batch_size: int = 1000, ) -> "list[Path]": """Write fixture records asynchronously in JSON, JSONL, or CSV format. Streams records from a synchronous or asynchronous iterable in batches of ``batch_size`` into temporary ``.tmp`` files, offloads writer construction, batch writes, commit, and abort to worker threads via :func:`~advanced_alchemy.utils.sync_tools.async_`, atomically renames temporary files into place on completion, and removes stale fixture files matching ``fixture_name``. Args: fixtures_path: Target directory path for fixture files. Created if it does not exist. Can be a :class:`pathlib.Path` or :class:`anyio.Path`. fixture_name: Fixture base name without file extension. records: Synchronous or asynchronous iterable of records to write. file_format: Output format (``"json"``, ``"jsonl"``, or ``"csv"``). compression: Optional compression (``"gzip"``, ``"zip"``, or ``None``). max_bytes: Optional maximum on-disk byte size per shard. Must be at least 1. max_records: Optional maximum record count per shard. Must be at least 1. batch_size: Maximum number of records buffered before offloading a batch write to a worker thread. Must be at least 1. Raises: ValueError: If ``batch_size``, ``file_format``, ``compression``, ``max_bytes``, or ``max_records`` is invalid, or if CSV records introduce keys absent from the header. TypeError: If ``file_format`` is ``"csv"`` and a record is not a mapping. OSError: If writing, deleting stale files, or renaming files fails. Returns: list[Path]: Final written fixture paths in shard order. Examples: >>> from pathlib import Path >>> async def stream_users(): ... yield {"id": 1, "name": "Alice"} >>> paths = await write_fixture_async( ... Path("./fixtures"), ... "users", ... stream_users(), ... file_format="jsonl", ... compression="gzip", ... ) """ if batch_size < 1: msg = f"batch_size must be at least 1, got {batch_size}" raise ValueError(msg) writer = await async_(_FixtureWriter)( Path(str(fixtures_path)), fixture_name, file_format=file_format, compression=compression, max_bytes=max_bytes, max_records=max_records, ) try: batch: list[Any] = [] if isinstance(records, AsyncIterable): async for record in records: batch.append(record) if len(batch) >= batch_size: await async_(writer.write_many)(batch) batch = [] else: for record in records: batch.append(record) if len(batch) >= batch_size: await async_(writer.write_many)(batch) batch = [] if batch: await async_(writer.write_many)(batch) return await async_(writer.commit)() except BaseException: await async_(writer.abort)() raise
class _FixtureFile: """Streaming reader for a single plain, gzipped, or zipped fixture file.""" __slots__ = ("compression", "file_format", "fixture_name", "path", "shard_index") def __init__( self, path: "Path", fixture_name: str, file_format: FixtureFormat, compression: "Optional[FixtureCompression]", shard_index: "Optional[int]", ) -> None: """Initialize a fixture file descriptor. Args: path: Path to the fixture file on disk. fixture_name: Logical fixture name without extension. file_format: Fixture format (``"json"``, ``"jsonl"``, or ``"csv"``). compression: Compression kind (``"gzip"``, ``"zip"``, or ``None``). shard_index: Optional 1-based shard index, or ``None`` when unsharded. """ self.path = path self.fixture_name = fixture_name self.file_format = file_format self.compression = compression self.shard_index = shard_index def read(self) -> Any: """Read and decode the entire fixture file. Raises: OSError: If reading or decompressing the fixture file fails. ValueError: If a zip archive has no matching member or JSONL content is invalid. Returns: Any: The decoded JSON value or a list of records for JSONL/CSV files. """ if self.file_format in ("jsonl", "csv"): return list(self.iter_records()) try: with self._open_text() as handle: content = handle.read() except _READ_ERRORS as exc: msg = f"Error reading fixture file {self.path}: {exc}" raise OSError(msg) from exc return decode_json(content) def iter_records(self) -> "Generator[Any, None, None]": """Stream decoded records from the fixture file one at a time. Raises: OSError: If reading or decompressing the fixture file fails. ValueError: If a ``json`` file does not contain a top-level array, a zip archive has no matching member, or a JSONL line is invalid JSON. Yields: Any: Each decoded record in the fixture file. """ if self.file_format == "json": value = self.read() if not isinstance(value, list): msg = f"Fixture {self.fixture_name!r} file {self.path} must contain a JSON array, got {type(value).__name__}" raise ValueError(msg) yield from value return try: with self._open_text() as handle: if self.file_format == "csv": yield from csv.DictReader(handle) else: yield from self._iter_jsonl(handle) except _READ_ERRORS as exc: msg = f"Error reading fixture file {self.path}: {exc}" raise OSError(msg) from exc def _iter_jsonl(self, handle: "TextIO") -> "Generator[Any, None, None]": """Yield decoded JSON values from each non-blank JSONL line in ``handle``.""" for lineno, line in enumerate(handle, start=1): if line.strip(): yield self._decode_line(line, lineno) def _open_text(self) -> "AbstractContextManager[TextIO]": """Open the fixture file as a streaming UTF-8-SIG text handle. Returns: AbstractContextManager[TextIO]: A text stream context manager. """ newline = "" if self.file_format == "csv" else None if self.compression == "gzip": return gzip.open(self.path, mode="rt", encoding=_TEXT_ENCODING, newline=newline) if self.compression == "zip": with zipfile.ZipFile(self.path, mode="r") as archive: member = archive.open(self._select_zip_member(archive), mode="r") return io.TextIOWrapper(member, encoding=_TEXT_ENCODING, newline=newline) return self.path.open(mode="r", encoding=_TEXT_ENCODING, newline=newline) def _select_zip_member(self, archive: zipfile.ZipFile) -> str: """Select the matching member name inside a zip fixture archive. Args: archive: Open zip archive handle. Raises: ValueError: If the archive contains no file ending with the fixture suffix. Returns: str: Selected zip member name. """ suffix = f".{self.file_format}" members = [name for name in archive.namelist() if name.endswith(suffix)] if not members: msg = f"No {self.file_format.upper()} files found in zip archive: {self.path}" raise ValueError(msg) for candidate in (self.path.stem, f"{self.fixture_name}{suffix}"): if candidate in members: return candidate return members[0] def _decode_line(self, line: str, lineno: int) -> Any: """Decode one JSONL line and attach file path and line number on failure. Args: line: Raw line text from the JSONL file. lineno: 1-based line number within the file. Raises: ValueError: If the line is not valid JSON. Returns: Any: The decoded JSON value for the line. """ try: return decode_json(line) except Exception as exc: msg = f"Invalid JSON in fixture file {self.path} at line {lineno}" raise ValueError(msg) from exc class _FixtureLocator: """Locator that resolves fixture files by format priority and compression.""" __slots__ = ("base_path", "display_name", "fixture_name") def __init__(self, base_path: "Path", fixture_name: str) -> None: """Initialize a fixture locator. Args: base_path: Directory path containing fixture files. fixture_name: Logical fixture name or relative subpath without extension. Raises: ValueError: If ``fixture_name`` is empty, absolute, or traverses parent directories. """ self.display_name = fixture_name self.base_path, self.fixture_name = self.split_target(base_path, fixture_name) @staticmethod def split_target(base_path: "Path", fixture_name: str) -> "tuple[Path, str]": """Resolve a fixture name or relative subpath into its directory and leaf name. Args: base_path: Root directory path for fixtures. fixture_name: Fixture name or relative subpath without extension. Raises: ValueError: If ``fixture_name`` is empty, ends with a path separator, is absolute, or contains ``..`` traversal segments. Returns: tuple[Path, str]: Resolved target directory and leaf fixture name. """ if not fixture_name or fixture_name.endswith(("/", "\\")): msg = f"Invalid fixture name {fixture_name!r}" raise ValueError(msg) relative = Path(fixture_name) if relative.is_absolute() or not relative.name or relative.name in {".", ".."} or ".." in relative.parts: msg = f"Invalid fixture name {fixture_name!r}" raise ValueError(msg) return base_path / relative.parent, relative.name @staticmethod def next_batch(batches: "Iterator[list[Any]]") -> "Optional[list[Any]]": """Return the next batch from a batch iterator, or ``None`` when exhausted. Args: batches: Synchronous batch iterator. Returns: Optional[list[Any]]: Next batch of records, or ``None`` at end of stream. """ return next(batches, None) @staticmethod def path_identity(path: "Path") -> "Union[tuple[int, int], str]": """Return a filesystem identity key for deduplicating equivalent paths. Args: path: Existing file path. Returns: Union[tuple[int, int], str]: Device/inode pair when available, else path string. """ stat = path.stat() if stat.st_ino: return (stat.st_dev, stat.st_ino) return str(path) def resolve(self) -> "list[_FixtureFile]": """Resolve the highest-priority fixture file or shards for the configured name. Raises: FileNotFoundError: If no matching fixture file or shard is found. ValueError: If both an unsharded file and numbered shards exist for the resolved format, or if shard validation fails. Returns: list[_FixtureFile]: Ordered list containing the resolved fixture file or shards. """ for file_format in _FORMAT_PRIORITY: candidates = self._unsharded_candidates(file_format) shards = self._shards(file_format) if candidates and shards: paths = ", ".join(str(fixture_file.path) for fixture_file in [*candidates, *shards]) msg = f"Fixture {self.display_name!r} has both an unsharded file and numbered shards: {paths}" raise ValueError(msg) if shards: return shards if candidates: return candidates[:1] msg = ( f"Could not find the {self.display_name} fixture " "(tried .json, .jsonl, .csv with optional .gz/.zip compression, numbered shards, and case variations)" ) raise FileNotFoundError(msg) def iter_batches(self, batch_size: int) -> "Generator[list[Any], None, None]": """Validate ``batch_size`` and resolve files eagerly, then stream record batches. Args: batch_size: Maximum number of records per yielded batch. Must be at least 1. Raises: ValueError: If ``batch_size`` is less than 1, or if fixture/shard resolution fails. FileNotFoundError: If no matching fixture file or shard is found. Returns: Generator[list[Any], None, None]: Generator yielding non-empty record batches. """ if batch_size < 1: msg = f"batch_size must be at least 1, got {batch_size}" raise ValueError(msg) return self._batches(self.resolve(), batch_size) def existing_paths(self) -> "list[Path]": """Return every existing unsharded or sharded path matching the fixture name. Skips contiguous index validation so broken, gapped, or duplicate shard sets (anchored by shard ``001``) can still be discovered for cleanup when replacing a fixture, while ignoring unrelated dotted-numeric siblings. Returns: list[Path]: Sorted deduplicated fixture file paths in ``base_path``. """ if not self.base_path.is_dir(): return [] seen: set[Union[tuple[int, int], str]] = set() paths: list[Path] = [] for file_format in _FORMAT_PRIORITY: for fixture_file in [*self._unsharded_candidates(file_format), *self._shard_matches(file_format)]: identity = self.path_identity(fixture_file.path) if identity not in seen: seen.add(identity) paths.append(fixture_file.path) return sorted(paths) def _unsharded_candidates(self, file_format: FixtureFormat) -> "list[_FixtureFile]": """Collect existing unsharded fixture candidates for a format in priority order. Args: file_format: Fixture format to probe. Returns: list[_FixtureFile]: Existing deduplicated candidate fixture files. """ upper_name = self.fixture_name.upper() variants: list[tuple[str, Optional[FixtureCompression]]] = [ (f"{self.fixture_name}.{file_format}", None), ] if file_format == "csv": variants.append((f"{upper_name}.{file_format}", None)) for comp, suffix in _COMPRESSION_SUFFIXES.items(): variants.append((f"{upper_name}.{file_format}{suffix}", comp)) variants.append((f"{self.fixture_name}.{file_format}{suffix}", comp)) candidates: list[_FixtureFile] = [] seen: set[Union[tuple[int, int], str]] = set() for name, compression in variants: path = self.base_path / name if not path.is_file(): continue identity = self.path_identity(path) if identity in seen: continue seen.add(identity) candidates.append(_FixtureFile(path, self.fixture_name, file_format, compression, None)) return candidates def _shard_matches(self, file_format: FixtureFormat) -> "list[_FixtureFile]": """Collect canonical numbered shard files for a format when shard ``001`` is present. Args: file_format: Fixture format to probe. Returns: list[_FixtureFile]: Deduplicated matching shard descriptors, or ``[]`` when no shard ``001`` exists. """ if not self.base_path.is_dir(): return [] pattern = re.compile( rf"^(?P<name>{re.escape(self.fixture_name)}|{re.escape(self.fixture_name.upper())})" rf"\.(?P<index>\d+)\.(?P<fmt>json|jsonl|csv)(?P<comp>\.gz|\.zip)?$" ) matches: list[_FixtureFile] = [] seen: set[Union[tuple[int, int], str]] = set() for path in sorted(self.base_path.iterdir()): match = pattern.match(path.name) if match is None or match["fmt"] != file_format or not self._is_canonical_index(match["index"]): continue if not path.is_file(): continue identity = self.path_identity(path) if identity in seen: continue seen.add(identity) compression = _SUFFIX_COMPRESSIONS.get(match["comp"] or "") matches.append(_FixtureFile(path, self.fixture_name, file_format, compression, int(match["index"]))) if not any(fixture_file.shard_index == 1 for fixture_file in matches): return [] return matches def _shards(self, file_format: FixtureFormat) -> "list[_FixtureFile]": """Validate and return numbered shards for a format in numeric index order. Args: file_format: Fixture format to validate. Raises: ValueError: If duplicate shard indexes exist, index 0 is present, or any shard index in ``1..N`` is missing. Returns: list[_FixtureFile]: Validated shard descriptors sorted by shard index. """ by_index: dict[int, list[_FixtureFile]] = {} for fixture_file in self._shard_matches(file_format): if fixture_file.shard_index is None: continue by_index.setdefault(fixture_file.shard_index, []).append(fixture_file) if not by_index: return [] indexes = sorted(by_index) for index in indexes: group = by_index[index] if len(group) > 1: paths = ", ".join(str(fixture_file.path) for fixture_file in group) msg = f"Fixture {self.display_name!r} has multiple shards with index {index}: {paths}" raise ValueError(msg) if indexes[0] == 0: paths = ", ".join(str(fixture_file.path) for fixture_file in by_index[0]) msg = f"Fixture {self.display_name!r} shard indexes start at 1, found index 0: {paths}" raise ValueError(msg) for expected, actual in enumerate(indexes, start=1): if actual != expected: paths = ", ".join(str(by_index[index][0].path) for index in indexes) msg = f"Fixture {self.display_name!r} is missing shard index(es) {expected}: {paths}" raise ValueError(msg) return [by_index[index][0] for index in indexes] @staticmethod def _batches(files: "list[_FixtureFile]", batch_size: int) -> "Generator[list[Any], None, None]": """Yield non-empty batches of at most ``batch_size`` records across resolved files. Args: files: Ordered fixture file descriptors to stream. batch_size: Positive maximum batch size. Yields: list[Any]: Non-empty list of decoded records. """ batch: list[Any] = [] for fixture_file in files: records = fixture_file.iter_records() try: for record in records: batch.append(record) if len(batch) == batch_size: yield batch batch = [] finally: records.close() if batch: yield batch @staticmethod def _is_canonical_index(digits: str) -> bool: """Return whether ``digits`` matches the writer's zero-padded shard index format.""" return len(digits) >= _SHARD_INDEX_WIDTH and digits == f"{int(digits):0{_SHARD_INDEX_WIDTH}d}" class _FixtureWriter: """Streaming writer for plain, gzipped, and zipped fixture files.""" __slots__ = ( "_archive", "_closed", "_final_paths", "_force_rollover", "_header", "_pending", "_raw", "_record_count", "_shard_records", "_sink", "_temp_paths", "base_path", "compression", "display_name", "file_format", "fixture_name", "max_bytes", "max_records", ) def __init__( self, base_path: "Path", fixture_name: str, *, file_format: FixtureFormat, compression: "Optional[FixtureCompression]", max_bytes: "Optional[int]", max_records: "Optional[int]", ) -> None: """Initialize a streaming fixture writer and validate options before disk access. Args: base_path: Target directory path for written fixture files. fixture_name: Logical fixture name or relative subpath without extension. file_format: Output format (``"json"``, ``"jsonl"``, or ``"csv"``). compression: Optional compression (``"gzip"``, ``"zip"``, or ``None``). max_bytes: Optional maximum on-disk byte size per shard. max_records: Optional maximum record count per shard. Raises: ValueError: If any fixture name, format, compression, or shard limit argument is invalid. """ if file_format not in ("json", "jsonl", "csv"): msg = f"Unsupported fixture format {file_format!r}; expected 'json', 'jsonl', or 'csv'" raise ValueError(msg) if compression not in (None, "gzip", "zip"): msg = f"Unsupported fixture compression {compression!r}; expected 'gzip', 'zip', or None" raise ValueError(msg) if max_bytes is not None and max_bytes < 1: msg = f"max_bytes must be at least 1, got {max_bytes}" raise ValueError(msg) if max_records is not None and max_records < 1: msg = f"max_records must be at least 1, got {max_records}" raise ValueError(msg) self.display_name = fixture_name self.base_path, self.fixture_name = _FixtureLocator.split_target(base_path, fixture_name) self.file_format: FixtureFormat = file_format self.compression: Optional[FixtureCompression] = compression self.max_bytes = max_bytes self.max_records = max_records self.base_path.mkdir(parents=True, exist_ok=True) self._archive: Optional[zipfile.ZipFile] = None self._closed = False self._final_paths: list[Path] = [] self._force_rollover = False self._header: Optional[list[Any]] = None self._pending = 0 self._raw: Optional[BinaryIO] = None self._record_count = 0 self._shard_records = 0 self._sink: Optional[Union[IO[bytes], gzip.GzipFile]] = None self._temp_paths: list[Path] = [] @property def sharded(self) -> bool: """Return whether numbered shard output is enabled.""" return self.max_bytes is not None or self.max_records is not None def write(self, record: Any) -> None: """Encode and stream a single record to the current shard. Args: record: Record value to serialize and write. Raises: RuntimeError: If the writer has already been committed or aborted. TypeError: If writing CSV and ``record`` is not a mapping. ValueError: If writing CSV and ``record`` introduces keys absent from the header. """ if self._closed: msg = "Fixture writer is closed" raise RuntimeError(msg) if self._raw is None: self._open_shard() self._record_count += 1 chunk = self._encode(record, first_in_shard=self._shard_records == 0) if self._shard_records > 0 and self._roll_over_needed_after_flush(len(chunk)): self._close_shard() self._open_shard() chunk = self._encode(record, first_in_shard=True) if self._shard_records == 0 and self.max_bytes is not None: projected = self._projected_size(len(chunk)) if projected > self.max_bytes: warnings.warn( f"Record {self._record_count} of fixture {self.display_name!r} exceeds " f"max_bytes={self.max_bytes} on its own (projected {projected} bytes); " "writing it to its own shard", UserWarning, stacklevel=2, ) self._force_rollover = True self._require_sink().write(chunk) self._pending += len(chunk) self._shard_records += 1 def write_many(self, records: "Iterable[Any]") -> None: """Stream every record from an iterable into the writer. Args: records: Iterable of records to write. """ for record in records: self.write(record) def commit(self) -> "list[Path]": """Finalize output, rename temp files into place, and remove stale fixture files. Returns: list[Path]: Final written fixture file paths in shard order. """ if self._raw is None and not self._final_paths: self._open_shard() self._close_shard() stale_paths = _FixtureLocator(self.base_path, self.fixture_name).existing_paths() for temp, final in zip(self._temp_paths, self._final_paths): temp.replace(final) final_set = set(self._final_paths) final_identities = {_FixtureLocator.path_identity(final) for final in self._final_paths if final.exists()} for path in stale_paths: if path in final_set or not path.exists(): continue if _FixtureLocator.path_identity(path) in final_identities: continue path.unlink(missing_ok=True) self._closed = True return list(self._final_paths) def abort(self) -> None: """Close any open streams and delete temporary files idempotently.""" self._closed = True if self._sink is not None: with contextlib.suppress(Exception): self._sink.close() if self._archive is not None: with contextlib.suppress(Exception): self._archive.close() if self._raw is not None: with contextlib.suppress(Exception): self._raw.close() self._sink = None self._archive = None self._raw = None for temp in self._temp_paths: temp.unlink(missing_ok=True) def _final_path(self, index: int) -> "Path": """Compute the final destination path for a 1-based shard index.""" suffix = _COMPRESSION_SUFFIXES.get(self.compression, "") if self.compression is not None else "" if self.sharded: return self.base_path / f"{self.fixture_name}.{index:03d}.{self.file_format}{suffix}" return self.base_path / f"{self.fixture_name}.{self.file_format}{suffix}" def _open_shard(self) -> None: """Open the next temporary shard stream with deterministic compression settings.""" final = self._final_path(len(self._final_paths) + 1) temp = final.with_name(f"{final.name}.tmp") self._final_paths.append(final) self._temp_paths.append(temp) self._raw = temp.open("wb") if self.compression == "gzip": self._sink = gzip.GzipFile(filename="", mode="wb", fileobj=self._raw, mtime=0) elif self.compression == "zip": self._archive = zipfile.ZipFile(self._raw, mode="w", compression=zipfile.ZIP_DEFLATED) info = zipfile.ZipInfo(final.name[: -len(".zip")], date_time=(1980, 1, 1, 0, 0, 0)) info.compress_type = zipfile.ZIP_DEFLATED info.external_attr = 0o644 << 16 self._sink = self._archive.open(info, mode="w", force_zip64=True) else: self._sink = self._raw self._shard_records = 0 self._pending = 0 self._force_rollover = False def _close_shard(self) -> None: """Write closing delimiters if needed and close the active shard streams.""" sink = self._require_sink() if self.file_format == "json": sink.write(b"[]\n" if self._shard_records == 0 else b"\n]\n") raw = self._require_raw() if sink is not raw: sink.close() if self._archive is not None: self._archive.close() raw.close() self._sink = None self._archive = None self._raw = None def _encode(self, record: Any, *, first_in_shard: bool) -> bytes: """Encode one record into UTF-8 bytes without mutating shard counters.""" if self.file_format == "json": prefix = b"[\n" if first_in_shard else b",\n" return prefix + encode_json(record).encode("utf-8") if self.file_format == "jsonl": return (encode_json(record) + "\n").encode("utf-8") if not self._is_mapping(record): msg = f"CSV fixture records must be mappings, got {type(record).__name__}" raise TypeError(msg) if self._header is None: self._header = list(record.keys()) header_keys = set(self._header) extra = [key for key in record if key not in header_keys] if extra: msg = ( f"CSV record {self._record_count} of fixture {self.display_name!r} " f"has keys not in the header: {extra!r}" ) raise ValueError(msg) header_row = self._encode_csv_row(self._header) if first_in_shard else "" data_row = self._encode_csv_row([record.get(key, "") for key in self._header]) return (header_row + data_row).encode("utf-8") @staticmethod def _is_mapping(value: Any) -> "TypeGuard[Mapping[Any, Any]]": """Narrow an arbitrary record to a typed mapping for static type checkers.""" return isinstance(value, Mapping) @staticmethod def _encode_csv_row(values: "Iterable[Any]") -> str: """Format a single CSV row using Python's excel dialect.""" buffer = io.StringIO() csv.writer(buffer).writerow(values) return buffer.getvalue() def _require_sink(self) -> "Union[IO[bytes], gzip.GzipFile]": """Return the active byte sink handle or raise if the writer is closed.""" if self._sink is None: msg = "Fixture writer is closed" raise RuntimeError(msg) return self._sink def _require_raw(self) -> "BinaryIO": """Return the active underlying binary file handle or raise if closed.""" if self._raw is None: msg = "Fixture writer is closed" raise RuntimeError(msg) return self._raw def _projected_size(self, chunk_size: int) -> int: """Estimate the on-disk shard size in bytes after writing ``chunk_size`` bytes.""" raw_pos = self._require_raw().tell() if self.compression is None: return raw_pos + chunk_size + 64 pending = self._pending + chunk_size if self.compression == "gzip": reserve = 64 else: member_name = self._final_paths[-1].name[: -len(".zip")] reserve = 256 + 2 * len(member_name.encode("utf-8")) return raw_pos + pending + 5 * (pending // 16383 + 1) + reserve def _roll_over_needed_after_flush(self, chunk_size: int) -> bool: """Determine whether to open a new shard before writing, flushing gzip if needed.""" if self._force_rollover: return True if self.max_records is not None and self._shard_records >= self.max_records: return True if self.max_bytes is not None: projected = self._projected_size(chunk_size) if projected > self.max_bytes and self.compression == "gzip" and self._pending > 0: self._require_sink().flush() self._pending = 0 projected = self._projected_size(chunk_size) return projected > self.max_bytes return False