diff --git a/.travis.yml b/.travis.yml index 9bf3ca9..f11ac9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,8 @@ env: - DEPS=aiohttp - DEPS=tornado pycurl - DEPS=tornado +matrix: + exclude: + # doesn't work well, see https://travis-ci.org/lilydjwg/nvchecker/jobs/376326582 + python: pypy3 + env: DEPS=aiohttp diff --git a/README.rst b/README.rst index ead30df..bd717db 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,9 @@ proxy max_concurrent Max number of concurrent jobs. Default: 20. +keyfile + Specify an ini config file containing key (token) information. This file should contain a ``keys`` section, mapping key names to key values. See specific source for the key name(s) to use. + Global Options -------------- The following options apply to all checkers. @@ -228,8 +231,9 @@ sort_version_key proxy The HTTP proxy to use. The format is ``host:port``, e.g. ``localhost:8087``. -An environment variable ``NVCHECKER_GITHUB_TOKEN`` can be set to a GitHub OAuth -token in order to request more frequently than anonymously. +An environment variable ``NVCHECKER_GITHUB_TOKEN`` or a key named ``github`` +can be set to a GitHub OAuth token in order to request more frequently than +anonymously. Check BitBucket --------------- @@ -286,10 +290,20 @@ host Hostname for self-hosted GitLab instance. token - GitLab authorization token used to call the API. If not specified, an environment variable ``NVCHECKER_GITLAB_TOKEN_host`` must provide that token. The ``host`` part is the uppercased version of the ``host`` setting, with dots (``.``) and slashes (``/``) replaced by underscores (``_``), e.g. ``NVCHECKER_GITLAB_TOKEN_GITLAB_COM``. - + GitLab authorization token used to call the API. + Authenticated only. +To set a authorization token, you can set: + +- a key named ``gitlab_{host}`` in the keyfile (where ``host`` is formed the + same as the environment variable, but all lowercased). +- an environment variable ``NVCHECKER_GITLAB_TOKEN_{host}`` must provide that + token. The ``host`` part is the uppercased version of the ``host`` setting, + with dots (``.``) and slashes (``/``) replaced by underscores (``_``), e.g. + ``NVCHECKER_GITLAB_TOKEN_GITLAB_COM``. +- the token option + Check PyPI ---------- Check `PyPI `_ for updates. diff --git a/nvchecker/core.py b/nvchecker/core.py index 8e88f95..a63ef47 100644 --- a/nvchecker/core.py +++ b/nvchecker/core.py @@ -114,18 +114,25 @@ class Source: if '__config__' in config: c = config['__config__'] + d = os.path.dirname(file.name) if 'oldver' in c and 'newver' in c: - d = os.path.dirname(file.name) self.oldver = os.path.expandvars(os.path.expanduser( os.path.join(d, c.get('oldver')))) self.newver = os.path.expandvars(os.path.expanduser( os.path.join(d, c.get('newver')))) + keyfile = c.get('keyfile') + if keyfile: + keyfile = os.path.expandvars(os.path.expanduser( + os.path.join(d, c.get('keyfile')))) + self.max_concurrent = c.getint('max_concurrent', 20) + self.keymanager = KeyManager(keyfile) session.nv_config = config["__config__"] else: self.max_concurrent = 20 + self.keymanager = KeyManager(None) async def check(self): if self.oldver: @@ -139,7 +146,7 @@ class Source: async def worker(name, conf): await token_q.get() try: - ret = await get_version(name, conf) + ret = await get_version(name, conf, keyman=self.keymanager) return name, ret except Exception as e: return name, e @@ -164,9 +171,13 @@ class Source: for fu in asyncio.as_completed(futures): name, result = await fu if isinstance(result, Exception): - logger.error('unexpected error happened', name=name, exc_info=result) + logger.error('unexpected error happened', + name=name, exc_info=result) + self.on_exception(name, result) elif result is not None: self.print_version_update(name, result) + else: + self.on_no_result(name) await filler_fu @@ -185,5 +196,22 @@ class Source: def on_update(self, name, version, oldver): pass + def on_no_result(self, name, oldver): + pass + + def on_exception(self, name, exc): + pass + def __repr__(self): return '' % self.name + +class KeyManager: + def __init__(self, file): + self.config = config = configparser.ConfigParser(dict_type=dict) + if file is not None: + config.read([file]) + else: + config.add_section('keys') + + def get_key(self, name): + return self.config.get('keys', name, fallback=None) diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index e7a4810..96be522 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -38,11 +38,11 @@ def substitute_version(version, name, conf): # No substitution rules found. Just return the original version string. return version -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): for key in handler_precedence: if key in conf: func = import_module('.source.' + key, __package__).get_version - version = await func(name, conf) + version = await func(name, conf, **kwargs) if version: version = version.replace('\n', ' ') try: diff --git a/nvchecker/slogconf.py b/nvchecker/slogconf.py index aa0f7af..a0f34c4 100644 --- a/nvchecker/slogconf.py +++ b/nvchecker/slogconf.py @@ -40,6 +40,7 @@ def stdlib_renderer(logger, level, event): logger = logging.getLogger(std_event.get('logger_name')) msg = std_event.pop('msg', std_event['event']) exc_info = std_event.pop('exc_info', None) + # msg = f'{msg} {std_event!r}' getattr(logger, level)( msg, exc_info = exc_info, extra=std_event, ) diff --git a/nvchecker/source/aiohttp_httpclient.py b/nvchecker/source/aiohttp_httpclient.py index 9bd9e59..a950e17 100644 --- a/nvchecker/source/aiohttp_httpclient.py +++ b/nvchecker/source/aiohttp_httpclient.py @@ -5,14 +5,24 @@ import atexit import aiohttp connector = aiohttp.TCPConnector(limit=20) -__all__ = ['session'] +__all__ = ['session', 'HTTPError'] + +class HTTPError(Exception): + def __init__(self, code, message, response): + self.code = code + self.message = message + self.response = response class BetterClientSession(aiohttp.ClientSession): async def _request(self, *args, **kwargs): if hasattr(self, "nv_config") and self.nv_config.get("proxy"): kwargs.setdefault("proxy", self.nv_config.get("proxy")) - return await super(BetterClientSession, self)._request(*args, **kwargs) + res = await super(BetterClientSession, self)._request( + *args, **kwargs) + if res.status >= 400: + raise HTTPError(res.status, res.reason, res) + return res session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5) atexit.register(session.close) diff --git a/nvchecker/source/android_sdk.py b/nvchecker/source/android_sdk.py index 03f5a20..fc4a079 100644 --- a/nvchecker/source/android_sdk.py +++ b/nvchecker/source/android_sdk.py @@ -34,7 +34,7 @@ async def _get_repo_manifest(repo): return repo_manifest -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): repo = conf['repo'] pkg_path_prefix = conf['android_sdk'] diff --git a/nvchecker/source/anitya.py b/nvchecker/source/anitya.py index e988dd1..174fc64 100644 --- a/nvchecker/source/anitya.py +++ b/nvchecker/source/anitya.py @@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__) URL = 'https://release-monitoring.org/api/project/{pkg}' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): pkg = conf.get('anitya') url = URL.format(pkg = pkg) diff --git a/nvchecker/source/archpkg.py b/nvchecker/source/archpkg.py index c2f1037..b57bf16 100644 --- a/nvchecker/source/archpkg.py +++ b/nvchecker/source/archpkg.py @@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__) URL = 'https://www.archlinux.org/packages/search/json/' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): pkg = conf.get('archpkg') or name strip_release = conf.getboolean('strip-release', False) async with session.get(URL, params={"name": pkg}) as res: diff --git a/nvchecker/source/aur.py b/nvchecker/source/aur.py index 1d1da00..f33612d 100644 --- a/nvchecker/source/aur.py +++ b/nvchecker/source/aur.py @@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__) AUR_URL = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg[]=' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): aurname = conf.get('aur') or name strip_release = conf.getboolean('strip-release', False) async with session.get(AUR_URL, params={"v": 5, "type": "info", "arg[]": aurname}) as res: diff --git a/nvchecker/source/bitbucket.py b/nvchecker/source/bitbucket.py index 9e9706e..abd2128 100644 --- a/nvchecker/source/bitbucket.py +++ b/nvchecker/source/bitbucket.py @@ -8,7 +8,7 @@ from ..sortversion import sort_version_keys BITBUCKET_URL = 'https://bitbucket.org/api/2.0/repositories/%s/commits/%s' BITBUCKET_MAX_TAG = 'https://bitbucket.org/api/1.0/repositories/%s/tags' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): repo = conf.get('bitbucket') br = conf.get('branch', '') use_max_tag = conf.getboolean('use_max_tag', False) diff --git a/nvchecker/source/cmd.py b/nvchecker/source/cmd.py index 4af371c..e5a4933 100644 --- a/nvchecker/source/cmd.py +++ b/nvchecker/source/cmd.py @@ -7,7 +7,7 @@ import structlog logger = structlog.get_logger(logger_name=__name__) -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): cmd = conf['cmd'] p = await asyncio.create_subprocess_shell( cmd, diff --git a/nvchecker/source/cratesio.py b/nvchecker/source/cratesio.py index 5d5f9fa..5449665 100644 --- a/nvchecker/source/cratesio.py +++ b/nvchecker/source/cratesio.py @@ -6,7 +6,7 @@ from . import session API_URL = 'https://crates.io/api/v1/crates/%s' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): name = conf.get('cratesio') or name async with session.get(API_URL % name) as res: data = await res.json() diff --git a/nvchecker/source/debianpkg.py b/nvchecker/source/debianpkg.py index 0906a69..d5ff0da 100644 --- a/nvchecker/source/debianpkg.py +++ b/nvchecker/source/debianpkg.py @@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__) URL = 'https://sources.debian.org/api/src/%(pkgname)s/?suite=%(suite)s' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): pkg = conf.get('debianpkg') or name strip_release = conf.getboolean('strip-release', False) suite = conf.get('suite') or "sid" diff --git a/nvchecker/source/github.py b/nvchecker/source/github.py index 61c8d9c..c1c755c 100644 --- a/nvchecker/source/github.py +++ b/nvchecker/source/github.py @@ -3,11 +3,12 @@ import os import re +import time from functools import partial import structlog -from . import session +from . import session, HTTPError from ..sortversion import sort_version_keys logger = structlog.get_logger(logger_name=__name__) @@ -16,7 +17,13 @@ GITHUB_URL = 'https://api.github.com/repos/%s/commits' GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest' GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/tags' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): + try: + return await get_version_real(name, conf, **kwargs) + except HTTPError as e: + check_ratelimit(e, name) + +async def get_version_real(name, conf, **kwargs): repo = conf.get('github') br = conf.get('branch') use_latest_release = conf.getboolean('use_latest_release', False) @@ -38,6 +45,10 @@ async def get_version(name, conf): } if 'NVCHECKER_GITHUB_TOKEN' in os.environ: headers['Authorization'] = 'token %s' % os.environ['NVCHECKER_GITHUB_TOKEN'] + else: + key = kwargs['keyman'].get_key('github') + if key: + headers['Authorization'] = 'token %s' % key kwargs = {} if conf.get('proxy'): @@ -51,6 +62,8 @@ async def get_version(name, conf): ) async with session.get(url, headers=headers, **kwargs) as res: + logger.debug('X-RateLimit-Remaining', + n=res.headers.get('X-RateLimit-Remaining')) data = await res.json() if use_latest_release: @@ -75,6 +88,8 @@ async def max_tag( while True: async with getter(url) as res: + logger.debug('X-RateLimit-Remaining', + n=res.headers.get('X-RateLimit-Remaining')) links = res.headers.get('Link') data = await res.json() @@ -104,3 +119,16 @@ def get_next_page_url(links): return return next_link[0].split('>', 1)[0][1:] + +def check_ratelimit(exc, name): + res = exc.response + n = int(res.headers.get('X-RateLimit-Remaining')) + if n == 0: + reset = int(res.headers.get('X-RateLimit-Reset')) + logger.error('rate limited, resetting at %s. ' + 'Or get an API token to increase the allowance if not yet' + % time.ctime(reset), + name = name, + reset = reset) + else: + raise diff --git a/nvchecker/source/gitlab.py b/nvchecker/source/gitlab.py index 4a1cba1..889bb80 100644 --- a/nvchecker/source/gitlab.py +++ b/nvchecker/source/gitlab.py @@ -1,5 +1,5 @@ # MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. +# Copyright (c) 2013-2018 lilydjwg , et al. import os import urllib.parse @@ -14,7 +14,7 @@ logger = structlog.get_logger(logger_name=__name__) GITLAB_URL = 'https://%s/api/v3/projects/%s/repository/commits?ref_name=%s' GITLAB_MAX_TAG = 'https://%s/api/v3/projects/%s/repository/tags' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): repo = urllib.parse.quote_plus(conf.get('gitlab')) br = conf.get('branch', 'master') host = conf.get('host', "gitlab.com") @@ -22,8 +22,16 @@ async def get_version(name, conf): ignored_tags = conf.get("ignored_tags", "").split() sort_version_key = sort_version_keys[conf.get("sort_version_key", "parse_version")] - env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_") - token = conf.get('token', os.environ.get(env_name, None)) + token = conf.get('token') + + if token is None: + env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_") + global_key = os.environ.get(env_name) + if not global_key: + key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_") + global_key = kwargs['keyman'].get_key(key_name) + token = global_key + if token is None: logger.error('No gitlab token specified.', name=name) return diff --git a/nvchecker/source/manual.py b/nvchecker/source/manual.py index f042204..d7602f8 100644 --- a/nvchecker/source/manual.py +++ b/nvchecker/source/manual.py @@ -1,5 +1,5 @@ # MIT licensed # Copyright (c) 2013-2017 lilydjwg , et al. -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): return conf.get('manual').strip() or None diff --git a/nvchecker/source/pacman.py b/nvchecker/source/pacman.py index 3a03582..aa487a4 100644 --- a/nvchecker/source/pacman.py +++ b/nvchecker/source/pacman.py @@ -3,7 +3,7 @@ from . import cmd -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): referree = conf.get('pacman') or name c = "LANG=C pacman -Si %s | grep -F Version | awk '{print $3}'" % referree conf['cmd'] = c diff --git a/nvchecker/source/regex.py b/nvchecker/source/regex.py index da0c912..43d7825 100644 --- a/nvchecker/source/regex.py +++ b/nvchecker/source/regex.py @@ -11,7 +11,7 @@ from ..sortversion import sort_version_keys logger = structlog.get_logger(logger_name=__name__) -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): try: regex = re.compile(conf['regex']) except sre_constants.error: diff --git a/nvchecker/source/simple_json.py b/nvchecker/source/simple_json.py index e0dceb3..5561fa5 100644 --- a/nvchecker/source/simple_json.py +++ b/nvchecker/source/simple_json.py @@ -5,7 +5,7 @@ from . import session def simple_json(urlpat, confkey, version_from_json): - async def get_version(name, conf): + async def get_version(name, conf, **kwargs): repo = conf.get(confkey) or name url = urlpat % repo kwargs = {} diff --git a/nvchecker/source/tornado_httpclient.py b/nvchecker/source/tornado_httpclient.py index a87d098..53b46e4 100644 --- a/nvchecker/source/tornado_httpclient.py +++ b/nvchecker/source/tornado_httpclient.py @@ -5,6 +5,7 @@ import json from urllib.parse import urlencode from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPResponse +from tornado.httpclient import HTTPError from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future AsyncIOMainLoop().install() @@ -14,7 +15,7 @@ try: except ImportError: pycurl = None -__all__ = ['session'] +__all__ = ['session', 'HTTPError'] client = AsyncHTTPClient() HTTP2_AVAILABLE = None if pycurl else False diff --git a/nvchecker/source/ubuntupkg.py b/nvchecker/source/ubuntupkg.py index ea72106..98c7083 100644 --- a/nvchecker/source/ubuntupkg.py +++ b/nvchecker/source/ubuntupkg.py @@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__) URL = 'https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name=%s&exact_match=true' -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): pkg = conf.get('ubuntupkg') or name strip_release = conf.getboolean('strip-release', False) suite = conf.get('suite') diff --git a/nvchecker/source/vcs.py b/nvchecker/source/vcs.py index 62fdd4c..41b21aa 100644 --- a/nvchecker/source/vcs.py +++ b/nvchecker/source/vcs.py @@ -26,7 +26,7 @@ def _parse_oldver(oldver): return PROT_VER, 0, ver return PROT_VER, count, ver -async def get_version(name, conf): +async def get_version(name, conf, **kwargs): vcs = conf['vcs'] use_max_tag = conf.getboolean('use_max_tag', False) ignored_tags = conf.get("ignored_tags", "").split() diff --git a/setup.py b/setup.py index 7531fb5..24d0b73 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( url = 'https://github.com/lilydjwg/nvchecker', long_description = open('README.rst', encoding='utf-8').read(), platforms = 'any', - zip_safe = True, + zip_safe = False, packages = find_packages(exclude=["tests"]), install_requires = ['setuptools', 'structlog'], diff --git a/tests/conftest.py b/tests/conftest.py index 102dffa..6af862f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,37 @@ import configparser import pytest import asyncio +import io from nvchecker.get_version import get_version as _get_version +from nvchecker.core import Source + +class TestSource(Source): + def __init__(self, future, *args, **kwargs): + super().__init__(*args, **kwargs) + self._future = future + + def on_update(self, name, version, oldver): + self._future.set_result(version) + + def on_no_result(self, name): + self._future.set_result(None) + + def on_exception(self, name, exc): + self._future.set_exception(exc) + +@pytest.fixture(scope="module") +async def run_source(): + async def __call__(conf): + future = asyncio.Future() + file = io.StringIO(conf) + file.name = '' + + s = TestSource(future, file) + await s.check() + return await future + + return __call__ @pytest.fixture(scope="module") async def get_version(): diff --git a/tests/test_aur.py b/tests/test_aur.py index 441954b..2b67adb 100644 --- a/tests/test_aur.py +++ b/tests/test_aur.py @@ -5,10 +5,10 @@ from flaky import flaky import pytest pytestmark = pytest.mark.asyncio -@flaky(max_runs=5) +@flaky(max_runs=10) async def test_aur(get_version): assert await get_version("ssed", {"aur": None}) == "3.62-2" -@flaky(max_runs=5) +@flaky(max_runs=10) async def test_aur_strip_release(get_version): assert await get_version("ssed", {"aur": None, "strip-release": 1}) == "3.62" diff --git a/tests/test_keyfile.py b/tests/test_keyfile.py new file mode 100644 index 0000000..09346f4 --- /dev/null +++ b/tests/test_keyfile.py @@ -0,0 +1,77 @@ +# MIT licensed +# Copyright (c) 2018 lilydjwg , et al. + +import os +import tempfile +import contextlib + +from nvchecker.source import HTTPError + +import pytest +pytestmark = [pytest.mark.asyncio] + +@contextlib.contextmanager +def unset_github_token_env(): + token = os.environ.get('NVCHECKER_GITHUB_TOKEN') + try: + if token: + del os.environ['NVCHECKER_GITHUB_TOKEN'] + yield token + finally: + if token: + os.environ['NVCHECKER_GITHUB_TOKEN'] = token + +async def test_keyfile_missing(run_source): + test_conf = '''\ +[example] +github = harry-sanabria/ReleaseTestRepo +''' + + assert await run_source(test_conf) in ['20140122.012101', None] + +async def test_keyfile_invalid(run_source): + with tempfile.NamedTemporaryFile(mode='w') as f, \ + unset_github_token_env(): + f.write('''\ +[keys] +github = xxx + ''') + f.flush() + test_conf = '''\ +[example] +github = harry-sanabria/ReleaseTestRepo + +[__config__] +keyfile = {name} +'''.format(name=f.name) + + try: + version = await run_source(test_conf) + assert version is None # out of allowance + return + except HTTPError as e: + assert e.code == 401 + return + + raise Exception('expected 401 response') + +@pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ, + reason='no key given') +async def test_keyfile_valid(run_source): + with tempfile.NamedTemporaryFile(mode='w') as f, \ + unset_github_token_env() as token: + f.write('''\ +[keys] +github = {token} + '''.format(token=token)) + f.flush() + + test_conf = '''\ +[example] +github = harry-sanabria/ReleaseTestRepo + +[__config__] +keyfile = {name} + '''.format(name=f.name) + + assert await run_source(test_conf) == '20140122.012101' diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 5faa4c6..a8c808f 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -20,6 +20,8 @@ async def test_proxy(get_version, monkeypatch): async def fake_request(*args, proxy, **kwargs): class fake_response(): + status = 200 + async def read(): return proxy.encode("ascii") diff --git a/tests/test_regex.py b/tests/test_regex.py index 5f2b084..46f39ba 100644 --- a/tests/test_regex.py +++ b/tests/test_regex.py @@ -4,9 +4,17 @@ import pytest pytestmark = pytest.mark.asyncio -async def test_regex(get_version): +@pytest.mark.skipif(True, + reason='httpbin is overloaded?') +async def test_regex_httpbin(get_version): assert await get_version("example", { "url": "https://httpbin.org/get", "regex": '"User-Agent": "(\w+)"', "user_agent": "Meow", }) == "Meow" + +async def test_regex(get_version): + assert await get_version("example", { + "url": "https://example.net/", + "regex": 'for (\w+) examples', + }) == "illustrative"