From de1a3c6fc221effc61f4116689ef79c5ab34bc6a Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 14 Mar 2024 18:28:25 +0800 Subject: [PATCH] record rich results in verfile and get rid of Result & VersData types Use RichResult to replace Result; Result was RichResult plus entry name. --- nvchecker/__main__.py | 2 +- nvchecker/core.py | 66 ++++++++++++++++++++++++++++++------------- nvchecker/tools.py | 4 +-- nvchecker/util.py | 11 +------- tests/conftest.py | 10 +++---- 5 files changed, 56 insertions(+), 37 deletions(-) diff --git a/nvchecker/__main__.py b/nvchecker/__main__.py index eb8e82d..7d72653 100755 --- a/nvchecker/__main__.py +++ b/nvchecker/__main__.py @@ -88,7 +88,7 @@ def main() -> None: if options.ver_files is not None: newverf = options.ver_files[1] vers = core.read_verfile(newverf) - vers.update({k: r.version for k, r in results.items()}) + vers.update(results) core.write_verfile(newverf, vers) if args.failures and has_failures: diff --git a/nvchecker/core.py b/nvchecker/core.py index 669a637..033b304 100644 --- a/nvchecker/core.py +++ b/nvchecker/core.py @@ -1,5 +1,5 @@ # MIT licensed -# Copyright (c) 2013-2020 lilydjwg , et al. +# Copyright (c) 2013-2020, 2024 lilydjwg , et al. from __future__ import annotations @@ -20,6 +20,7 @@ from importlib import import_module import re import contextvars import json +import dataclasses import structlog @@ -36,7 +37,7 @@ import platformdirs from .lib import nicelogger from . import slogconf from .util import ( - Entry, Entries, KeyManager, RawResult, RichResult, Result, VersData, + Entry, Entries, KeyManager, RawResult, RichResult, ResultData, FunctionWorker, GetVersionError, FileLoadError, EntryWaiter, ) @@ -126,7 +127,7 @@ def safe_overwrite(file: Path, data: Union[bytes, str], *, # if the above write failed (because disk is full etc), the old data should be kept os.rename(tmpname, resolved_path) -def read_verfile(file: Path) -> VersData: +def read_verfile(file: Path) -> ResultData: try: with open(file) as f: data = f.read() @@ -142,17 +143,35 @@ def read_verfile(file: Path) -> VersData: name, ver = l.rstrip().split(None, 1) v[name] = ver + if v.get('version') is None: + v = {k: RichResult(version=a) for k, a in v.items()} + elif v['version'] == 2: + v = {k: RichResult(**a) for k, a in v['data'].items()} + else: + raise Exception('unknown verfile version', v['version']) + return v -def write_verfile(file: Path, versions: VersData) -> None: - # sort and indent to make it friendly to human and git +def write_verfile(file: Path, versions: ResultData) -> None: + d = { + 'version': 2, + # sort and indent to make it friendly to human and git + 'data': dict(sorted(versions.items())), + } data = json.dumps( - dict(sorted(versions.items())), - indent=2, - ensure_ascii=False, + d, + indent = 2, + ensure_ascii = False, + default = json_encode, ) + '\n' safe_overwrite(file, data) +def json_encode(obj): + if isinstance(obj, RichResult): + d = {k: v for k, v in dataclasses.asdict(obj).items() if v is not None} + return d + raise TypeError(obj) + class Options(NamedTuple): ver_files: Optional[Tuple[Path, Path]] max_concurrency: int @@ -325,7 +344,7 @@ def apply_list_options( return versions[-1] -def _process_result(r: RawResult) -> Union[Result, Exception]: +def _process_result(r: RawResult) -> Union[RichResult, Exception]: version = r.version conf = r.conf name = r.name @@ -362,7 +381,12 @@ def _process_result(r: RawResult) -> Union[Result, Exception]: try: version_str = substitute_version(version_str, conf) - return Result(name, version_str, conf, url, gitref, revision) + return RichResult( + version = version_str, + url = url, + gitref = gitref, + revision = revision, + ) except (ValueError, re.error) as e: logger.exception('error occurred in version substitutions', name=name) return e @@ -371,15 +395,19 @@ def _process_result(r: RawResult) -> Union[Result, Exception]: return ValueError('no version returned') def check_version_update( - oldvers: VersData, - r: Result, + oldvers: ResultData, + name: str, + r: RichResult, verbose: bool, ) -> None: - oldver = oldvers.get(r.name, None) + if old_result := oldvers.get(name): + oldver = old_result.version + else: + oldver = None if not oldver or oldver != r.version: logger.info( 'updated', - name = r.name, + name = name, version = r.version, old_version = oldver, url = r.url, @@ -387,10 +415,10 @@ def check_version_update( else: # provide visible user feedback if it was the only entry level = logging.INFO if verbose else logging.DEBUG - logger.log(level, 'up-to-date', name=r.name, version=r.version, url=r.url) + logger.log(level, 'up-to-date', name=name, version=r.version, url=r.url) async def process_result( - oldvers: VersData, + oldvers: ResultData, result_q: Queue[RawResult], entry_waiter: EntryWaiter, verbose: bool = False, @@ -409,9 +437,9 @@ async def process_result( entry_waiter.set_exception(r.name, r1) has_failures = True continue - check_version_update(oldvers, r1, verbose) - entry_waiter.set_result(r1.name, r1.version) - ret[r1.name] = r1 + check_version_update(oldvers, r.name, r1, verbose) + entry_waiter.set_result(r.name, r1.version) + ret[r.name] = r1 except asyncio.CancelledError: return ret, has_failures diff --git a/nvchecker/tools.py b/nvchecker/tools.py index 5e43b5d..ecd0894 100644 --- a/nvchecker/tools.py +++ b/nvchecker/tools.py @@ -106,8 +106,8 @@ def cmp() -> None: oldverf = opt.ver_files[0] newverf = opt.ver_files[1] - oldvers = core.read_verfile(oldverf) - newvers = core.read_verfile(newverf) + oldvers = {k: v.version for k, v in core.read_verfile(oldverf).items()} + newvers = {k: v.version for k, v in core.read_verfile(newverf).items()} differences = [] diff --git a/nvchecker/util.py b/nvchecker/util.py index 033dc85..fbc3ff9 100644 --- a/nvchecker/util.py +++ b/nvchecker/util.py @@ -39,7 +39,6 @@ logger = structlog.get_logger(logger_name=__name__) Entry = Dict[str, Any] Entry.__doc__ = '''The configuration `dict` for an entry.''' Entries = Dict[str, Entry] -VersData = Dict[str, str] if sys.version_info[:2] >= (3, 11): from typing import LiteralString @@ -145,15 +144,7 @@ RawResult.name.__doc__ = 'The name (table name) of the entry.' RawResult.version.__doc__ = 'The result from the check.' RawResult.conf.__doc__ = 'The entry configuration (table content) of the entry.' -class Result(NamedTuple): - name: str - version: str - conf: Entry - url: Optional[str] - gitref: Optional[str] - revision: Optional[str] - -ResultData = Dict[str, Result] +ResultData = Dict[str, RichResult] class BaseWorker: '''The base class for defining `Worker` classes for source plugins. diff --git a/tests/conftest.py b/tests/conftest.py index d980a95..9d0a839 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,11 @@ # MIT licensed -# Copyright (c) 2020 lilydjwg , et al. +# Copyright (c) 2020, 2024 lilydjwg , et al. import asyncio import structlog import os from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict if TYPE_CHECKING: import tomli as tomllib @@ -20,13 +20,13 @@ import pytest_asyncio from nvchecker import core from nvchecker import __main__ as main -from nvchecker.util import Entries, VersData, RawResult +from nvchecker.util import Entries, ResultData, RawResult use_keyfile = False async def run( entries: Entries, max_concurrency: int = 20, -) -> VersData: +) -> Dict[str, str]: task_sem = asyncio.Semaphore(max_concurrency) result_q: asyncio.Queue[RawResult] = asyncio.Queue() keyfile = os.environ.get('KEYFILE') @@ -43,7 +43,7 @@ async def run( keymanager, entry_waiter, 1, {}, ) - oldvers: VersData = {} + oldvers: ResultData = {} result_coro = core.process_result(oldvers, result_q, entry_waiter) runner_coro = core.run_tasks(futures)