From 14b3863f11fe72f0cd753a6724c093d1661dfc00 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Tue, 11 Aug 2020 17:43:03 +0800 Subject: [PATCH] version 2: MVP --- NEW | 21 ++ mypy.ini | 13 + nvchecker-old/get_version.py | 57 +++ .../source/android_sdk.py | 0 {nvchecker => nvchecker-old}/source/anitya.py | 0 .../source/archpkg.py | 0 .../source/bitbucket.py | 0 {nvchecker => nvchecker-old}/source/cmd.py | 0 {nvchecker => nvchecker-old}/source/cpan.py | 0 .../source/cratesio.py | 0 .../source/debianpkg.py | 0 {nvchecker => nvchecker-old}/source/gems.py | 0 {nvchecker => nvchecker-old}/source/gitea.py | 0 {nvchecker => nvchecker-old}/source/github.py | 0 {nvchecker => nvchecker-old}/source/gitlab.py | 0 .../source/hackage.py | 0 {nvchecker => nvchecker-old}/source/manual.py | 0 {nvchecker => nvchecker-old}/source/npm.py | 0 .../source/packagist.py | 0 {nvchecker => nvchecker-old}/source/pacman.py | 0 {nvchecker => nvchecker-old}/source/pypi.py | 0 {nvchecker => nvchecker-old}/source/regex.py | 0 .../source/repology.py | 0 .../source/simple_json.py | 0 .../source/sparkle.py | 0 .../source/ubuntupkg.py | 0 {nvchecker => nvchecker-old}/source/vcs.py | 0 {nvchecker => nvchecker-old}/source/vcs.sh | 0 {nvchecker => nvchecker-old}/tools.py | 0 nvchecker/__init__.py | 2 +- nvchecker/__main__.py | 67 ++-- nvchecker/core.py | 338 +++++++++++------- nvchecker/get_version.py | 105 ------ nvchecker/{source => httpclient}/__init__.py | 7 - .../aiohttp_httpclient.py | 0 .../{source => httpclient}/httpclient.py | 0 .../tornado_httpclient.py | 6 +- nvchecker/lib/README.md | 2 +- nvchecker/lib/__init__.py | 3 + nvchecker/lib/nicelogger.py | 3 +- nvchecker/lib/notify.py | 102 ------ nvchecker/slogconf.py | 2 +- nvchecker/source/aur.py | 31 -- nvchecker/util.py | 72 ++++ nvchecker_source/aur.py | 84 +++++ nvchecker_source/none.py | 15 + setup.py | 4 +- tests/__init__.py | 3 + tests/conftest.py | 3 + 49 files changed, 537 insertions(+), 403 deletions(-) create mode 100644 NEW create mode 100644 mypy.ini create mode 100644 nvchecker-old/get_version.py rename {nvchecker => nvchecker-old}/source/android_sdk.py (100%) rename {nvchecker => nvchecker-old}/source/anitya.py (100%) rename {nvchecker => nvchecker-old}/source/archpkg.py (100%) rename {nvchecker => nvchecker-old}/source/bitbucket.py (100%) rename {nvchecker => nvchecker-old}/source/cmd.py (100%) rename {nvchecker => nvchecker-old}/source/cpan.py (100%) rename {nvchecker => nvchecker-old}/source/cratesio.py (100%) rename {nvchecker => nvchecker-old}/source/debianpkg.py (100%) rename {nvchecker => nvchecker-old}/source/gems.py (100%) rename {nvchecker => nvchecker-old}/source/gitea.py (100%) rename {nvchecker => nvchecker-old}/source/github.py (100%) rename {nvchecker => nvchecker-old}/source/gitlab.py (100%) rename {nvchecker => nvchecker-old}/source/hackage.py (100%) rename {nvchecker => nvchecker-old}/source/manual.py (100%) rename {nvchecker => nvchecker-old}/source/npm.py (100%) rename {nvchecker => nvchecker-old}/source/packagist.py (100%) rename {nvchecker => nvchecker-old}/source/pacman.py (100%) rename {nvchecker => nvchecker-old}/source/pypi.py (100%) rename {nvchecker => nvchecker-old}/source/regex.py (100%) rename {nvchecker => nvchecker-old}/source/repology.py (100%) rename {nvchecker => nvchecker-old}/source/simple_json.py (100%) rename {nvchecker => nvchecker-old}/source/sparkle.py (100%) rename {nvchecker => nvchecker-old}/source/ubuntupkg.py (100%) rename {nvchecker => nvchecker-old}/source/vcs.py (100%) rename {nvchecker => nvchecker-old}/source/vcs.sh (100%) rename {nvchecker => nvchecker-old}/tools.py (100%) delete mode 100644 nvchecker/get_version.py rename nvchecker/{source => httpclient}/__init__.py (72%) rename nvchecker/{source => httpclient}/aiohttp_httpclient.py (100%) rename nvchecker/{source => httpclient}/httpclient.py (100%) rename nvchecker/{source => httpclient}/tornado_httpclient.py (94%) delete mode 100644 nvchecker/lib/notify.py delete mode 100644 nvchecker/source/aur.py create mode 100644 nvchecker/util.py create mode 100644 nvchecker_source/aur.py create mode 100644 nvchecker_source/none.py diff --git a/NEW b/NEW new file mode 100644 index 0000000..c8a334f --- /dev/null +++ b/NEW @@ -0,0 +1,21 @@ +source file +| parse +entries: name, conf, global options +| dedupe +entries: names, conf, global options +| dispatch +one future per module +| run; token queue; result queue +result task, runner task +| receive +(names, result) +| transform back +(name, result) +| runner task done +| write record file +file + +TODO: +* dedupe & cache +* update tests +* update README diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..f3f5e64 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,13 @@ +[mypy] +warn_unused_configs = True +warn_redundant_casts = True +warn_unused_ignores = True +show_error_context = True +show_column_numbers = True +no_implicit_optional = True + +[mypy-structlog] +ignore_missing_imports = True + +[mypy-pyalpm] +ignore_missing_imports = True diff --git a/nvchecker-old/get_version.py b/nvchecker-old/get_version.py new file mode 100644 index 0000000..54c2d6c --- /dev/null +++ b/nvchecker-old/get_version.py @@ -0,0 +1,57 @@ +# MIT licensed +# Copyright (c) 2013-2017 lilydjwg , et al. + +from importlib import import_module + +import structlog + +logger = structlog.get_logger(logger_name=__name__) + +_handler_precedence = ( + 'github', 'aur', 'pypi', 'archpkg', 'debianpkg', 'ubuntupkg', + 'gems', 'pacman', + 'cmd', 'bitbucket', 'regex', 'manual', 'vcs', + 'cratesio', 'npm', 'hackage', 'cpan', 'gitlab', 'packagist', + 'repology', 'anitya', 'android_sdk', 'sparkle', 'gitea' +) + +_Task = namedtuple('_Task', 'batch_mode main args names') + +class Dispatcher: + def __init__(self): + self.futures = [] + self.mods = {} + self.tasks_dedupe = {} + + def add_task(self, name, conf, **kwargs): + for key in _handler_precedence: + if key not in conf: + continue + + value = self.mods.get(key) + if not value: + mod = import_module( + '.source.' + key, __package__) + batch_mode = getattr(mod, 'BATCH_MODE', False) + if batch_mode: + main = mod.Batcher() + else: + main = mod.get_version + get_cacheable_conf = getattr(mod, 'get_cacheable_conf', lambda name, conf: conf) + self.mods[key] = batch_mode, main, get_cacheable_conf + else: + batch_mode, main, get_cacheable_conf = value + + cacheable_conf = get_cacheable_conf(name, conf) + cache_key = tuple(sorted(cacheable_conf.items())) + task = self.tasks_dedupe.get(cache_key) + if task is None: + self.tasks_dedupe[cache_key] = _Task( + batch_mode, main, (cacheable_conf, kwargs), [name]) + else: + task.names.append(name) + + else: + logger.error( + 'no idea to get version info.', name=name) + diff --git a/nvchecker/source/android_sdk.py b/nvchecker-old/source/android_sdk.py similarity index 100% rename from nvchecker/source/android_sdk.py rename to nvchecker-old/source/android_sdk.py diff --git a/nvchecker/source/anitya.py b/nvchecker-old/source/anitya.py similarity index 100% rename from nvchecker/source/anitya.py rename to nvchecker-old/source/anitya.py diff --git a/nvchecker/source/archpkg.py b/nvchecker-old/source/archpkg.py similarity index 100% rename from nvchecker/source/archpkg.py rename to nvchecker-old/source/archpkg.py diff --git a/nvchecker/source/bitbucket.py b/nvchecker-old/source/bitbucket.py similarity index 100% rename from nvchecker/source/bitbucket.py rename to nvchecker-old/source/bitbucket.py diff --git a/nvchecker/source/cmd.py b/nvchecker-old/source/cmd.py similarity index 100% rename from nvchecker/source/cmd.py rename to nvchecker-old/source/cmd.py diff --git a/nvchecker/source/cpan.py b/nvchecker-old/source/cpan.py similarity index 100% rename from nvchecker/source/cpan.py rename to nvchecker-old/source/cpan.py diff --git a/nvchecker/source/cratesio.py b/nvchecker-old/source/cratesio.py similarity index 100% rename from nvchecker/source/cratesio.py rename to nvchecker-old/source/cratesio.py diff --git a/nvchecker/source/debianpkg.py b/nvchecker-old/source/debianpkg.py similarity index 100% rename from nvchecker/source/debianpkg.py rename to nvchecker-old/source/debianpkg.py diff --git a/nvchecker/source/gems.py b/nvchecker-old/source/gems.py similarity index 100% rename from nvchecker/source/gems.py rename to nvchecker-old/source/gems.py diff --git a/nvchecker/source/gitea.py b/nvchecker-old/source/gitea.py similarity index 100% rename from nvchecker/source/gitea.py rename to nvchecker-old/source/gitea.py diff --git a/nvchecker/source/github.py b/nvchecker-old/source/github.py similarity index 100% rename from nvchecker/source/github.py rename to nvchecker-old/source/github.py diff --git a/nvchecker/source/gitlab.py b/nvchecker-old/source/gitlab.py similarity index 100% rename from nvchecker/source/gitlab.py rename to nvchecker-old/source/gitlab.py diff --git a/nvchecker/source/hackage.py b/nvchecker-old/source/hackage.py similarity index 100% rename from nvchecker/source/hackage.py rename to nvchecker-old/source/hackage.py diff --git a/nvchecker/source/manual.py b/nvchecker-old/source/manual.py similarity index 100% rename from nvchecker/source/manual.py rename to nvchecker-old/source/manual.py diff --git a/nvchecker/source/npm.py b/nvchecker-old/source/npm.py similarity index 100% rename from nvchecker/source/npm.py rename to nvchecker-old/source/npm.py diff --git a/nvchecker/source/packagist.py b/nvchecker-old/source/packagist.py similarity index 100% rename from nvchecker/source/packagist.py rename to nvchecker-old/source/packagist.py diff --git a/nvchecker/source/pacman.py b/nvchecker-old/source/pacman.py similarity index 100% rename from nvchecker/source/pacman.py rename to nvchecker-old/source/pacman.py diff --git a/nvchecker/source/pypi.py b/nvchecker-old/source/pypi.py similarity index 100% rename from nvchecker/source/pypi.py rename to nvchecker-old/source/pypi.py diff --git a/nvchecker/source/regex.py b/nvchecker-old/source/regex.py similarity index 100% rename from nvchecker/source/regex.py rename to nvchecker-old/source/regex.py diff --git a/nvchecker/source/repology.py b/nvchecker-old/source/repology.py similarity index 100% rename from nvchecker/source/repology.py rename to nvchecker-old/source/repology.py diff --git a/nvchecker/source/simple_json.py b/nvchecker-old/source/simple_json.py similarity index 100% rename from nvchecker/source/simple_json.py rename to nvchecker-old/source/simple_json.py diff --git a/nvchecker/source/sparkle.py b/nvchecker-old/source/sparkle.py similarity index 100% rename from nvchecker/source/sparkle.py rename to nvchecker-old/source/sparkle.py diff --git a/nvchecker/source/ubuntupkg.py b/nvchecker-old/source/ubuntupkg.py similarity index 100% rename from nvchecker/source/ubuntupkg.py rename to nvchecker-old/source/ubuntupkg.py diff --git a/nvchecker/source/vcs.py b/nvchecker-old/source/vcs.py similarity index 100% rename from nvchecker/source/vcs.py rename to nvchecker-old/source/vcs.py diff --git a/nvchecker/source/vcs.sh b/nvchecker-old/source/vcs.sh similarity index 100% rename from nvchecker/source/vcs.sh rename to nvchecker-old/source/vcs.sh diff --git a/nvchecker/tools.py b/nvchecker-old/tools.py similarity index 100% rename from nvchecker/tools.py rename to nvchecker-old/tools.py diff --git a/nvchecker/__init__.py b/nvchecker/__init__.py index 9c1af1a..e0e86c6 100644 --- a/nvchecker/__init__.py +++ b/nvchecker/__init__.py @@ -1,4 +1,4 @@ # MIT licensed # Copyright (c) 2013-2020 lilydjwg , et al. -__version__ = '1.8dev' +__version__ = '2.0dev' diff --git a/nvchecker/__main__.py b/nvchecker/__main__.py index bc2889a..2e065de 100755 --- a/nvchecker/__main__.py +++ b/nvchecker/__main__.py @@ -1,33 +1,23 @@ #!/usr/bin/env python3 # MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. +# Copyright (c) 2013-2020 lilydjwg , et al. +from __future__ import annotations + +import sys import argparse import asyncio +from typing import Coroutine import structlog -from .lib import notify from . import core +from .util import VersData, RawResult logger = structlog.get_logger(logger_name=__name__) -notifications = [] -args = None - -class Source(core.Source): - def on_update(self, name, version, oldver): - if args.notify: - msg = '%s updated to version %s' % (name, version) - notifications.append(msg) - notify.update('nvchecker', '\n'.join(notifications)) - -def main(): - global args - +def main() -> None: parser = argparse.ArgumentParser(description='New version checker for software') - parser.add_argument('-n', '--notify', action='store_true', default=False, - help='show desktop notifications when a new version is available') parser.add_argument('-t', '--tries', default=1, type=int, metavar='N', help='try N times when errors occur') core.add_common_arguments(parser) @@ -36,12 +26,47 @@ def main(): return if not args.file: - return + try: + file = open(core.get_default_config()) + except FileNotFoundError: + sys.exit('version configuration file not given and default does not exist') + else: + file = args.file - s = Source(args.file, args.tries) + entries, options = core.load_file(file) + token_q = core.token_queue(options.max_concurrent) + result_q: asyncio.Queue[RawResult] = asyncio.Queue() + try: + futures = core.dispatch( + entries, token_q, result_q, + options.keymanager, args.tries, + ) + except ModuleNotFoundError as e: + sys.exit(f'Error: {e}') - ioloop = asyncio.get_event_loop() - ioloop.run_until_complete(s.check()) + if options.ver_files is not None: + oldvers = core.read_verfile(options.ver_files[0]) + else: + oldvers = {} + result_coro = core.process_result(oldvers, result_q) + runner_coro = core.run_tasks(futures) + + # asyncio.run doesn't work because it always creates new eventloops + loop = asyncio.get_event_loop() + newvers = loop.run_until_complete(run(result_coro, runner_coro)) + + if options.ver_files is not None: + core.write_verfile(options.ver_files[1], newvers) + +async def run( + result_coro: Coroutine[None, None, VersData], + runner_coro: Coroutine[None, None, None], +) -> VersData: + result_fu = asyncio.create_task(result_coro) + runner_fu = asyncio.create_task(runner_coro) + await runner_fu + result_fu.cancel() + return await result_fu if __name__ == '__main__': main() diff --git a/nvchecker/core.py b/nvchecker/core.py index 934ac12..d0295ba 100644 --- a/nvchecker/core.py +++ b/nvchecker/core.py @@ -1,25 +1,43 @@ # vim: se sw=2: # MIT licensed -# Copyright (c) 2013-2018 lilydjwg , et al. +# Copyright (c) 2013-2020 lilydjwg , et al. + +from __future__ import annotations import os import sys -import configparser import asyncio +from asyncio import Queue import logging +import argparse +from typing import ( + TextIO, Tuple, NamedTuple, Optional, List, Union, + cast, Dict, Awaitable, Sequence, +) +import types +from pathlib import Path +from importlib import import_module +import re import structlog +import toml from .lib import nicelogger -from .get_version import get_version -from .source import session from . import slogconf - +from .util import ( + Entry, Entries, KeyManager, RawResult, Result, VersData, +) from . import __version__ +from .sortversion import sort_version_keys logger = structlog.get_logger(logger_name=__name__) -def add_common_arguments(parser): +def get_default_config() -> str: + confdir = os.environ.get('XDG_CONFIG_DIR', os.path.expanduser('~/.config')) + file = os.path.join(confdir, 'nvchecker/nvchecker.toml') + return file + +def add_common_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument('-l', '--logging', choices=('debug', 'info', 'warning', 'error'), default='info', help='logging level (default: info)') @@ -31,10 +49,11 @@ def add_common_arguments(parser): help='specify fd to send json logs to. stdout by default') parser.add_argument('-V', '--version', action='store_true', help='show version and exit') - parser.add_argument('file', metavar='FILE', nargs='?', type=open, - help='software version source file') + parser.add_argument('-c', '--file', + metavar='FILE', type=open, + help='software version configuration file [default: %s]' % get_default_config) -def process_common_arguments(args): +def process_common_arguments(args: argparse.Namespace) -> bool: '''return True if should stop''' processors = [ slogconf.exc_info, @@ -71,8 +90,10 @@ def process_common_arguments(args): progname = os.path.basename(sys.argv[0]) print('%s v%s' % (progname, __version__)) return True + return False -def safe_overwrite(fname, data, *, method='write', mode='w', encoding=None): +def safe_overwrite(fname: str, data: Union[bytes, str], *, + method: str = 'write', mode: str = 'w', encoding: Optional[str] = None) -> None: # FIXME: directory has no read perm # FIXME: symlinks and hard links tmpname = fname + '.tmp' @@ -85,7 +106,7 @@ def safe_overwrite(fname, data, *, method='write', mode='w', encoding=None): # if the above write failed (because disk is full etc), the old data should be kept os.rename(tmpname, fname) -def read_verfile(file): +def read_verfile(file: Path) -> VersData: v = {} try: with open(file) as f: @@ -96,134 +117,199 @@ def read_verfile(file): pass return v -def write_verfile(file, versions): - # sort using only alphanums, as done by the sort command, and needed by - # comm command +def write_verfile(file: Path, versions: VersData) -> None: + # sort using only alphanums, as done by the sort command, + # and needed by comm command data = ['%s %s\n' % item for item in sorted(versions.items(), key=lambda i: (''.join(filter(str.isalnum, i[0])), i[1]))] - safe_overwrite(file, data, method='writelines') + safe_overwrite( + str(file), ''.join(data), method='writelines') -class Source: - oldver = newver = None - tries = 1 +class Options(NamedTuple): + ver_files: Optional[Tuple[Path, Path]] + max_concurrent: int + keymanager: KeyManager - def __init__(self, file, tries=1): - self.config = config = configparser.ConfigParser( - dict_type=dict, allow_no_value=True, interpolation=None, +def load_file( + file: TextIO, +) -> Tuple[Entries, Options]: + config = toml.load(file) + ver_files: Optional[Tuple[Path, Path]] = None + + if '__config__' in config: + c = config.pop('__config__') + d = Path(file.name).parent + + if 'oldver' in c and 'newver' in c: + oldver_s = os.path.expandvars( + os.path.expanduser(c.get('oldver'))) + oldver = d / oldver_s + newver_s = os.path.expandvars( + os.path.expanduser(c.get('newver'))) + newver = d / newver_s + ver_files = oldver, newver + + keyfile = c.get('keyfile') + if keyfile: + keyfile_s = os.path.expandvars( + os.path.expanduser(c.get('keyfile'))) + keyfile = d / keyfile_s + + max_concurrent = c.getint( + 'max_concurrent', 20) + keymanager = KeyManager(keyfile) + + else: + max_concurrent = 20 + keymanager = KeyManager(None) + + return cast(Entries, config), Options( + ver_files, max_concurrent, keymanager) + +def token_queue(maxsize: int) -> Queue[bool]: + token_q: Queue[bool] = Queue(maxsize=maxsize) + + for _ in range(maxsize): + token_q.put_nowait(True) + + return token_q + +def dispatch( + entries: Entries, + token_q: Queue[bool], + result_q: Queue[RawResult], + keymanager: KeyManager, + tries: int, +) -> List[asyncio.Future]: + mods: Dict[str, Tuple[types.ModuleType, List]] = {} + for name, entry in entries.items(): + source = entry.get('source', 'none') + if source not in mods: + mod = import_module('nvchecker_source.' + source) + tasks: List[Tuple[str, Entry]] = [] + mods[source] = mod, tasks + else: + tasks = mods[source][1] + tasks.append((name, entry)) + + ret = [] + for mod, tasks in mods.values(): + worker = mod.Worker( # type: ignore + token_q, result_q, tasks, + tries, keymanager, ) - self.name = file.name - self.tries = tries - config.read_file(file) - if '__config__' in config: - c = config['__config__'] + ret.append(worker.run()) - d = os.path.dirname(file.name) - if 'oldver' in c and 'newver' in c: - self.oldver = os.path.expandvars(os.path.expanduser( - os.path.join(d, c.get('oldver')))) - self.newver = os.path.expandvars(os.path.expanduser( - os.path.join(d, c.get('newver')))) + return ret - keyfile = c.get('keyfile') - if keyfile: - keyfile = os.path.expandvars(os.path.expanduser( - os.path.join(d, c.get('keyfile')))) +def substitute_version( + version: str, conf: Entry, +) -> str: + ''' + Substitute the version string via defined rules in the configuration file. + See README.rst#global-options for details. + ''' + prefix = conf.get('prefix') + if prefix: + if version.startswith(prefix): + version = version[len(prefix):] + return version - self.max_concurrent = c.getint('max_concurrent', 20) - self.keymanager = KeyManager(keyfile) - session.nv_config = config["__config__"] + from_pattern = conf.get('from_pattern') + if from_pattern: + to_pattern = conf.get('to_pattern') + if not to_pattern: + raise ValueError("from_pattern exists but to_pattern doesn't") - else: - self.max_concurrent = 20 - self.keymanager = KeyManager(None) + return re.sub(from_pattern, to_pattern, version) - async def check(self): - if self.oldver: - self.oldvers = read_verfile(self.oldver) - else: - self.oldvers = {} - self.curvers = self.oldvers.copy() + # No substitution rules found. Just return the original version string. + return version - tries = self.tries - token_q = asyncio.Queue(maxsize=self.max_concurrent) +def apply_list_options( + versions: List[str], conf: Entry, +) -> Optional[str]: + pattern = conf.get('include_regex') + if pattern: + re_pat = re.compile(pattern) + versions = [x for x in versions + if re_pat.fullmatch(x)] - for _ in range(self.max_concurrent): - await token_q.put(True) + pattern = conf.get('exclude_regex') + if pattern: + re_pat = re.compile(pattern) + versions = [x for x in versions + if not re_pat.fullmatch(x)] - async def worker(name, conf): - await token_q.get() - try: - for i in range(tries): - try: - ret = await get_version( - name, conf, keyman=self.keymanager) - return name, ret - except Exception as e: - if i + 1 < tries: - logger.warning('failed, retrying', - name=name, exc_info=e) - await asyncio.sleep(i) - else: - return name, e - finally: - await token_q.put(True) + ignored = set(conf.get('ignored', '').split()) + if ignored: + versions = [x for x in versions if x not in ignored] - config = self.config - futures = [] - for name in config.sections(): - if name == '__config__': + if not versions: + return None + + sort_version_key = sort_version_keys[ + conf.get("sort_version_key", "parse_version")] + versions.sort(key=sort_version_key) + + return versions[-1] + +def _process_result(r: RawResult) -> Optional[Result]: + version = r.version + conf = r.conf + name = r.name + + if isinstance(version, Exception): + logger.error('unexpected error happened', + name=r.name, exc_info=r.version) + return None + elif isinstance(version, list): + version_str = apply_list_options(version, conf) + else: + version_str = version + + if version_str: + version_str = version_str.replace('\n', ' ') + + try: + version_str = substitute_version(version_str, conf) + return Result(name, version_str, conf) + except (ValueError, re.error): + logger.exception('error occurred in version substitutions', name=name) + + return None + +def check_version_update( + oldvers: VersData, name: str, version: str, +) -> Optional[str]: + oldver = oldvers.get(name, None) + if not oldver or oldver != version: + logger.info('updated', name=name, version=version, old_version=oldver) + return version + else: + logger.debug('up-to-date', name=name, version=version) + return None + +async def process_result( + oldvers: VersData, + result_q: Queue[RawResult], +) -> VersData: + ret = {} + try: + while True: + r = await result_q.get() + r1 = _process_result(r) + if r1 is None: continue + v = check_version_update( + oldvers, r1.name, r1.version) + if v is not None: + ret[r1.name] = v + except asyncio.CancelledError: + return ret - conf = config[name] - conf['oldver'] = self.oldvers.get(name, None) - fu = asyncio.ensure_future(worker(name, conf)) - futures.append(fu) - - for fu in asyncio.as_completed(futures): - name, result = await fu - if isinstance(result, Exception): - logger.error('unexpected error happened', - name=name, exc_info=result) - self.on_exception(name, result) - elif result is not None: - self.print_version_update(name, result) - else: - conf = config[name] - if not conf.getboolean('missing_ok', False): - logger.warning('no-result', name=name) - self.on_no_result(name) - - if self.newver: - write_verfile(self.newver, self.curvers) - - def print_version_update(self, name, version): - oldver = self.oldvers.get(name, None) - if not oldver or oldver != version: - logger.info('updated', name=name, version=version, old_version=oldver) - self.curvers[name] = version - self.on_update(name, version, oldver) - else: - logger.debug('up-to-date', name=name, version=version) - - def on_update(self, name, version, oldver): - pass - - def on_no_result(self, name): - pass - - def on_exception(self, name, exc): - pass - - def __repr__(self): - return '' % self.name - -class KeyManager: - def __init__(self, file): - self.config = config = configparser.ConfigParser(dict_type=dict) - if file is not None: - config.read([file]) - else: - config.add_section('keys') - - def get_key(self, name): - return self.config.get('keys', name, fallback=None) +async def run_tasks( + futures: Sequence[Awaitable[None]] +) -> None: + for fu in asyncio.as_completed(futures): + await fu diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py deleted file mode 100644 index 3e7750b..0000000 --- a/nvchecker/get_version.py +++ /dev/null @@ -1,105 +0,0 @@ -# MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. - -import re -from importlib import import_module - -import structlog - -from .sortversion import sort_version_keys - -logger = structlog.get_logger(logger_name=__name__) - -handler_precedence = ( - 'github', 'aur', 'pypi', 'archpkg', 'debianpkg', 'ubuntupkg', - 'gems', 'pacman', - 'cmd', 'bitbucket', 'regex', 'manual', 'vcs', - 'cratesio', 'npm', 'hackage', 'cpan', 'gitlab', 'packagist', - 'repology', 'anitya', 'android_sdk', 'sparkle', 'gitea' -) - -def substitute_version(version, name, conf): - ''' - Substitute the version string via defined rules in the configuration file. - See README.rst#global-options for details. - ''' - prefix = conf.get('prefix') - if prefix: - if version.startswith(prefix): - version = version[len(prefix):] - return version - - from_pattern = conf.get('from_pattern') - if from_pattern: - to_pattern = conf.get('to_pattern') - if not to_pattern: - raise ValueError('%s: from_pattern exists but to_pattern doesn\'t', name) - - return re.sub(from_pattern, to_pattern, version) - - # No substitution rules found. Just return the original version string. - return version - -def apply_list_options(versions, conf): - pattern = conf.get('include_regex') - if pattern: - pattern = re.compile(pattern) - versions = [x for x in versions - if pattern.fullmatch(x)] - - pattern = conf.get('exclude_regex') - if pattern: - pattern = re.compile(pattern) - versions = [x for x in versions - if not pattern.fullmatch(x)] - - ignored = set(conf.get('ignored', '').split()) - if ignored: - versions = [x for x in versions if x not in ignored] - - if not versions: - return - - sort_version_key = sort_version_keys[ - conf.get("sort_version_key", "parse_version")] - versions.sort(key=sort_version_key) - - return versions[-1] - -_cache = {} - -async def get_version(name, conf, **kwargs): - for key in handler_precedence: - if key in conf: - mod = import_module('.source.' + key, __package__) - func = mod.get_version - get_cacheable_conf = getattr(mod, 'get_cacheable_conf', lambda name, conf: conf) - break - else: - logger.error('no idea to get version info.', name=name) - return - - cacheable_conf = get_cacheable_conf(name, conf) - cache_key = tuple(sorted(cacheable_conf.items())) - if cache_key in _cache: - version = _cache[cache_key] - logger.debug('cache hit', name=name, - cache_key=cache_key, cached=version) - return version - - version = await func(name, conf, **kwargs) - - if isinstance(version, list): - version = apply_list_options(version, conf) - - if version: - version = version.replace('\n', ' ') - try: - version = substitute_version(version, name, conf) - except (ValueError, re.error): - logger.exception('error occurred in version substitutions', name=name) - - if version is not None: - _cache[cache_key] = version - - return version diff --git a/nvchecker/source/__init__.py b/nvchecker/httpclient/__init__.py similarity index 72% rename from nvchecker/source/__init__.py rename to nvchecker/httpclient/__init__.py index ae79e12..024d9b0 100644 --- a/nvchecker/source/__init__.py +++ b/nvchecker/httpclient/__init__.py @@ -19,10 +19,3 @@ m = __import__('%s_httpclient' % which, globals(), locals(), level=1) __all__ = m.__all__ for x in __all__: globals()[x] = getattr(m, x) - -def conf_cacheable_with_name(key): - def get_cacheable_conf(name, conf): - conf = dict(conf) - conf[key] = conf.get(key) or name - return conf - return get_cacheable_conf diff --git a/nvchecker/source/aiohttp_httpclient.py b/nvchecker/httpclient/aiohttp_httpclient.py similarity index 100% rename from nvchecker/source/aiohttp_httpclient.py rename to nvchecker/httpclient/aiohttp_httpclient.py diff --git a/nvchecker/source/httpclient.py b/nvchecker/httpclient/httpclient.py similarity index 100% rename from nvchecker/source/httpclient.py rename to nvchecker/httpclient/httpclient.py diff --git a/nvchecker/source/tornado_httpclient.py b/nvchecker/httpclient/tornado_httpclient.py similarity index 94% rename from nvchecker/source/tornado_httpclient.py rename to nvchecker/httpclient/tornado_httpclient.py index c8ce818..124a3d7 100644 --- a/nvchecker/source/tornado_httpclient.py +++ b/nvchecker/httpclient/tornado_httpclient.py @@ -11,7 +11,7 @@ try: import pycurl AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient", max_clients=20) except ImportError: - pycurl = None + pycurl = None # type: ignore from .httpclient import DEFAULT_USER_AGENT @@ -76,8 +76,8 @@ async def json_response(self, **kwargs): async def read(self): return self.body -HTTPResponse.json = json_response -HTTPResponse.read = read +HTTPResponse.json = json_response # type: ignore +HTTPResponse.read = read # type: ignore session = Session() NetworkErrors = () diff --git a/nvchecker/lib/README.md b/nvchecker/lib/README.md index 4bb0a99..0e4ffe4 100644 --- a/nvchecker/lib/README.md +++ b/nvchecker/lib/README.md @@ -1 +1 @@ -This directory belongs to modules from my [winterpy](https://github.com/lilydjwg/winterpy) and can be synced from there without care. +This directory contains modules from my [winterpy](https://github.com/lilydjwg/winterpy). diff --git a/nvchecker/lib/__init__.py b/nvchecker/lib/__init__.py index e69de29..6ccd251 100644 --- a/nvchecker/lib/__init__.py +++ b/nvchecker/lib/__init__.py @@ -0,0 +1,3 @@ +# MIT licensed +# Copyright (c) 2020 lilydjwg , et al. + diff --git a/nvchecker/lib/nicelogger.py b/nvchecker/lib/nicelogger.py index b4e627b..841add2 100644 --- a/nvchecker/lib/nicelogger.py +++ b/nvchecker/lib/nicelogger.py @@ -44,8 +44,7 @@ class TornadoLogFormatter(logging.Formatter): record.message = "Bad message (%r): %r" % (e, record.__dict__) record.asctime = time.strftime( "%m-%d %H:%M:%S", self.converter(record.created)) - record.asctime += '.%03d' % ((record.created % 1) * 1000) - prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ + prefix = '[%(levelname)1.1s %(asctime)s.%(msecs)03d %(module)s:%(lineno)d]' % \ record.__dict__ if self._color: prefix = (self._colors.get(record.levelno, self._normal) + diff --git a/nvchecker/lib/notify.py b/nvchecker/lib/notify.py deleted file mode 100644 index ce831bb..0000000 --- a/nvchecker/lib/notify.py +++ /dev/null @@ -1,102 +0,0 @@ -# MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. - -''' -调用 libnotify -''' - -__all__ = ["set", "show", "update", "set_timeout", "set_urgency"] - -from ctypes import * -from threading import Lock -import atexit - -NOTIFY_URGENCY_LOW = 0 -NOTIFY_URGENCY_NORMAL = 1 -NOTIFY_URGENCY_CRITICAL = 2 -UrgencyLevel = {NOTIFY_URGENCY_LOW, NOTIFY_URGENCY_NORMAL, NOTIFY_URGENCY_CRITICAL} - -libnotify = None -gobj = None -libnotify_lock = Lock() -libnotify_inited = False - -class obj: pass -notify_st = obj() - -def set(summary=None, body=None, icon_str=None): - with libnotify_lock: - init() - - if summary is not None: - notify_st.summary = summary.encode() - notify_st.body = notify_st.icon_str = None - if body is not None: - notify_st.body = body.encode() - if icon_str is not None: - notify_st.icon_str = icon_str.encode() - - libnotify.notify_notification_update( - notify_st.notify, - notify_st.summary, - notify_st.body, - notify_st.icon_str, - ) - -def show(): - libnotify.notify_notification_show(notify_st.notify, c_void_p()) - -def update(summary=None, body=None, icon_str=None): - if not any((summary, body)): - raise TypeError('at least one argument please') - - set(summary, body, icon_str) - show() - -def set_timeout(self, timeout): - '''set `timeout' in milliseconds''' - libnotify.notify_notification_set_timeout(notify_st.notify, int(timeout)) - -def set_urgency(self, urgency): - if urgency not in UrgencyLevel: - raise ValueError - libnotify.notify_notification_set_urgency(notify_st.notify, urgency) - -def init(): - global libnotify_inited, libnotify, gobj - if libnotify_inited: - return - - try: - libnotify = CDLL('libnotify.so') - except OSError: - libnotify = CDLL('libnotify.so.4') - gobj = CDLL('libgobject-2.0.so') - - libnotify.notify_init('pynotify') - libnotify_inited = True - - libnotify.notify_notification_new.restype = c_void_p - notify_st.notify = c_void_p(libnotify.notify_notification_new( - c_void_p(), c_void_p(), c_void_p(), - )) - atexit.register(uninit) - -def uninit(): - global libnotify_inited - try: - if libnotify_inited: - gobj.g_object_unref(notify_st.notify) - libnotify.notify_uninit() - libnotify_inited = False - except AttributeError: - # libnotify.so 已被卸载 - pass - -if __name__ == '__main__': - from time import sleep - notify = __import__('__main__') - notify.set('This is a test', '测试一下。') - notify.show() - sleep(1) - notify.update(body='再测试一下。') diff --git a/nvchecker/slogconf.py b/nvchecker/slogconf.py index 32fb5fd..73cd092 100644 --- a/nvchecker/slogconf.py +++ b/nvchecker/slogconf.py @@ -10,7 +10,7 @@ import sys import structlog -from .source import HTTPError, NetworkErrors +from .httpclient import HTTPError, NetworkErrors # type: ignore def _console_msg(event): evt = event['event'] diff --git a/nvchecker/source/aur.py b/nvchecker/source/aur.py deleted file mode 100644 index 8118a41..0000000 --- a/nvchecker/source/aur.py +++ /dev/null @@ -1,31 +0,0 @@ -# MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. - -import structlog -from datetime import datetime - -from . import session, conf_cacheable_with_name - -logger = structlog.get_logger(logger_name=__name__) - -AUR_URL = 'https://aur.archlinux.org/rpc/' - -get_cacheable_conf = conf_cacheable_with_name('aur') - -async def get_version(name, conf, **kwargs): - aurname = conf.get('aur') or name - use_last_modified = conf.getboolean('use_last_modified', False) - strip_release = conf.getboolean('strip-release', False) - async with session.get(AUR_URL, params={"v": 5, "type": "info", "arg[]": aurname}) as res: - data = await res.json() - - if not data['results']: - logger.error('AUR upstream not found', name=name) - return - - version = data['results'][0]['Version'] - if use_last_modified: - version += '-' + datetime.utcfromtimestamp(data['results'][0]['LastModified']).strftime('%Y%m%d%H%M%S') - if strip_release and '-' in version: - version = version.rsplit('-', 1)[0] - return version diff --git a/nvchecker/util.py b/nvchecker/util.py new file mode 100644 index 0000000..676727c --- /dev/null +++ b/nvchecker/util.py @@ -0,0 +1,72 @@ +# MIT licensed +# Copyright (c) 2020 lilydjwg , et al. + +from __future__ import annotations + +from asyncio import Queue +import contextlib +from typing import ( + Dict, Optional, List, AsyncGenerator, NamedTuple, Union, + Any, Tuple, +) +from pathlib import Path + +import toml + +Entry = Dict[str, Any] +Entries = Dict[str, Entry] +VersData = Dict[str, str] + +class KeyManager: + def __init__( + self, file: Optional[Path], + ) -> None: + if file is not None: + with file.open() as f: + keys = toml.load(f)['keys'] + else: + keys = {} + self.keys = keys + + def get_key(self, name: str) -> Optional[str]: + return self.keys.get(name) + +class BaseWorker: + def __init__( + self, + token_q: Queue[bool], + result_q: Queue[RawResult], + tasks: List[Tuple[str, Entry]], + tries: int, + keymanager: KeyManager, + ) -> None: + self.token_q = token_q + self.result_q = result_q + self.tries = tries + self.keymanager = keymanager + self.tasks = tasks + + @contextlib.asynccontextmanager + async def acquire_token(self) -> AsyncGenerator[None, None]: + token = await self.token_q.get() + try: + yield + finally: + await self.token_q.put(token) + +class RawResult(NamedTuple): + name: str + version: Union[Exception, List[str], str] + conf: Entry + +class Result(NamedTuple): + name: str + version: str + conf: Entry + +def conf_cacheable_with_name(key): + def get_cacheable_conf(name, conf): + conf = dict(conf) + conf[key] = conf.get(key) or name + return conf + return get_cacheable_conf diff --git a/nvchecker_source/aur.py b/nvchecker_source/aur.py new file mode 100644 index 0000000..cfc695e --- /dev/null +++ b/nvchecker_source/aur.py @@ -0,0 +1,84 @@ +# MIT licensed +# Copyright (c) 2013-2020 lilydjwg , et al. + +import structlog +from datetime import datetime +import asyncio +from typing import Iterable, Dict, List, Tuple, Any + +from nvchecker.util import ( + Entry, BaseWorker, RawResult, + conf_cacheable_with_name, +) +from nvchecker.httpclient import session # type: ignore + +get_cacheable_conf = conf_cacheable_with_name('aur') + +logger = structlog.get_logger(logger_name=__name__) + +AUR_URL = 'https://aur.archlinux.org/rpc/' + +class Worker(BaseWorker): + # https://wiki.archlinux.org/index.php/Aurweb_RPC_interface#Limitations + batch_size = 100 + + async def run(self) -> None: + tasks = self.tasks + n_batch, left = divmod(len(tasks), self.batch_size) + if left > 0: + n_batch += 1 + + ret = [] + for i in range(n_batch): + s = i * self.batch_size + batch = tasks[s : s+self.batch_size] + fu = self._run_batch(batch) + ret.append(fu) + + await asyncio.wait(ret) + + async def _run_batch(self, batch: List[Tuple[str, Entry]]) -> None: + task_by_name: Dict[str, Entry] = dict(self.tasks) + async with self.acquire_token(): + results = await _run_batch_impl(batch) + for name, version in results.items(): + r = RawResult(name, version, task_by_name[name]) + await self.result_q.put(r) + +async def _run_batch_impl(batch: List[Tuple[str, Entry]]) -> Dict[str, str]: + aurnames = {conf.get('aur', name) for name, conf in batch} + results = await _aur_get_multiple(aurnames) + + ret = {} + + for name, conf in batch: + aurname = conf.get('aur', name) + use_last_modified = conf.get('use_last_modified', False) + strip_release = conf.get('strip-release', False) + + result = results.get(aurname) + + if result is None: + logger.error('AUR upstream not found', name=name) + continue + + version = result['Version'] + if use_last_modified: + version += '-' + datetime.utcfromtimestamp(result['LastModified']).strftime('%Y%m%d%H%M%S') + if strip_release and '-' in version: + version = version.rsplit('-', 1)[0] + + ret[name] = version + + return ret + +async def _aur_get_multiple( + aurnames: Iterable[str], +) -> Dict[str, Dict[str, Any]]: + params = [('v', '5'), ('type', 'info')] + params.extend(('arg[]', name) for name in aurnames) + async with session.get(AUR_URL, params=params) as res: + data = await res.json() + results = {r['Name']: r for r in data['results']} + return results + diff --git a/nvchecker_source/none.py b/nvchecker_source/none.py new file mode 100644 index 0000000..6dd7cdc --- /dev/null +++ b/nvchecker_source/none.py @@ -0,0 +1,15 @@ +# MIT licensed +# Copyright (c) 2020 lilydjwg , et al. + +from __future__ import annotations + +import structlog +from nvchecker.util import BaseWorker + +logger = structlog.get_logger(logger_name=__name__) + +class Worker(BaseWorker): + async def run(self) -> None: + async with self.acquire_token(): + for name, _ in self.tasks: + logger.error('no source specified', name=name) diff --git a/setup.py b/setup.py index 613cc68..af2aa07 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( zip_safe = False, packages = find_packages(exclude=["tests"]), - install_requires = ['setuptools', 'structlog', 'tornado>=6', 'pycurl'], + install_requires = ['setuptools', 'toml', 'structlog', 'tornado>=6', 'pycurl'], extras_require = { 'vercmp': ['pyalpm'], }, @@ -50,8 +50,6 @@ setup( "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Topic :: Internet", diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..6ccd251 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +# MIT licensed +# Copyright (c) 2020 lilydjwg , et al. + diff --git a/tests/conftest.py b/tests/conftest.py index fe3faed..89f5381 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,6 @@ +# MIT licensed +# Copyright (c) 2020 lilydjwg , et al. + import configparser import pytest import asyncio