Merge branch 'keyfile'

.travis.yml

@@ -17,3 +17,8 @@ env:
     - DEPS=aiohttp
     - DEPS=tornado pycurl
     - DEPS=tornado
+matrix:
+  exclude:
+    # doesn't work well, see https://travis-ci.org/lilydjwg/nvchecker/jobs/376326582
+    python: pypy3
+    env: DEPS=aiohttp

README.rst

@@ -127,6 +127,9 @@ proxy
 max_concurrent
   Max number of concurrent jobs. Default: 20.
 
+keyfile
+  Specify an ini config file containing key (token) information. This file should contain a ``keys`` section, mapping key names to key values. See specific source for the key name(s) to use.
+
 Global Options
 --------------
 The following options apply to all checkers.
@@ -228,8 +231,9 @@ sort_version_key
 proxy
   The HTTP proxy to use. The format is ``host:port``, e.g. ``localhost:8087``.
 
-An environment variable ``NVCHECKER_GITHUB_TOKEN`` can be set to a GitHub OAuth
-token in order to request more frequently than anonymously.
+An environment variable ``NVCHECKER_GITHUB_TOKEN`` or a key named ``github``
+can be set to a GitHub OAuth token in order to request more frequently than
+anonymously.
 
 Check BitBucket
 ---------------
@@ -286,10 +290,20 @@ host
   Hostname for self-hosted GitLab instance.
 
 token
-  GitLab authorization token used to call the API. If not specified, an environment variable ``NVCHECKER_GITLAB_TOKEN_host`` must provide that token. The ``host`` part is the uppercased version of the ``host`` setting, with dots (``.``) and slashes (``/``) replaced by underscores (``_``), e.g. ``NVCHECKER_GITLAB_TOKEN_GITLAB_COM``.
-
+  GitLab authorization token used to call the API.
+  
 Authenticated only.
 
+To set a authorization token, you can set:
+
+- a key named ``gitlab_{host}`` in the keyfile (where ``host`` is formed the
+  same as the environment variable, but all lowercased).
+- an environment variable ``NVCHECKER_GITLAB_TOKEN_{host}`` must provide that
+  token. The ``host`` part is the uppercased version of the ``host`` setting,
+  with dots (``.``) and slashes (``/``) replaced by underscores (``_``), e.g.
+  ``NVCHECKER_GITLAB_TOKEN_GITLAB_COM``.
+- the token option
+
 Check PyPI
 ----------
 Check `PyPI <https://pypi.python.org/>`_ for updates.

nvchecker/core.py

@@ -114,18 +114,25 @@ class Source:
     if '__config__' in config:
       c = config['__config__']
 
+      d = os.path.dirname(file.name)
       if 'oldver' in c and 'newver' in c:
-        d = os.path.dirname(file.name)
         self.oldver = os.path.expandvars(os.path.expanduser(
           os.path.join(d, c.get('oldver'))))
         self.newver = os.path.expandvars(os.path.expanduser(
           os.path.join(d, c.get('newver'))))
 
+      keyfile = c.get('keyfile')
+      if keyfile:
+        keyfile = os.path.expandvars(os.path.expanduser(
+          os.path.join(d, c.get('keyfile'))))
+
       self.max_concurrent = c.getint('max_concurrent', 20)
+      self.keymanager = KeyManager(keyfile)
       session.nv_config = config["__config__"]
 
     else:
       self.max_concurrent = 20
+      self.keymanager = KeyManager(None)
 
   async def check(self):
     if self.oldver:
@@ -139,7 +146,7 @@ class Source:
     async def worker(name, conf):
       await token_q.get()
       try:
-        ret = await get_version(name, conf)
+        ret = await get_version(name, conf, keyman=self.keymanager)
         return name, ret
       except Exception as e:
         return name, e
@@ -164,9 +171,13 @@ class Source:
     for fu in asyncio.as_completed(futures):
       name, result = await fu
       if isinstance(result, Exception):
-        logger.error('unexpected error happened', name=name, exc_info=result)
+        logger.error('unexpected error happened',
+                     name=name, exc_info=result)
+        self.on_exception(name, result)
       elif result is not None:
         self.print_version_update(name, result)
+      else:
+        self.on_no_result(name)
 
     await filler_fu
 
@@ -185,5 +196,22 @@ class Source:
   def on_update(self, name, version, oldver):
     pass
 
+  def on_no_result(self, name, oldver):
+    pass
+
+  def on_exception(self, name, exc):
+    pass
+
   def __repr__(self):
     return '<Source from %r>' % self.name
+
+class KeyManager:
+  def __init__(self, file):
+    self.config = config = configparser.ConfigParser(dict_type=dict)
+    if file is not None:
+      config.read([file])
+    else:
+      config.add_section('keys')
+
+  def get_key(self, name):
+    return self.config.get('keys', name, fallback=None)

nvchecker/get_version.py

@@ -38,11 +38,11 @@ def substitute_version(version, name, conf):
   # No substitution rules found. Just return the original version string.
   return version
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   for key in handler_precedence:
     if key in conf:
       func = import_module('.source.' + key, __package__).get_version
-      version = await func(name, conf)
+      version = await func(name, conf, **kwargs)
       if version:
         version = version.replace('\n', ' ')
         try:

nvchecker/slogconf.py

@@ -40,6 +40,7 @@ def stdlib_renderer(logger, level, event):
   logger = logging.getLogger(std_event.get('logger_name'))
   msg = std_event.pop('msg', std_event['event'])
   exc_info = std_event.pop('exc_info', None)
+  # msg = f'{msg} {std_event!r}'
   getattr(logger, level)(
     msg, exc_info = exc_info, extra=std_event,
   )

nvchecker/source/aiohttp_httpclient.py

@@ -5,14 +5,24 @@ import atexit
 import aiohttp
 connector = aiohttp.TCPConnector(limit=20)
 
-__all__ = ['session']
+__all__ = ['session', 'HTTPError']
+
+class HTTPError(Exception):
+    def __init__(self, code, message, response):
+        self.code = code
+        self.message = message
+        self.response = response
 
 class BetterClientSession(aiohttp.ClientSession):
     async def _request(self, *args, **kwargs):
         if hasattr(self, "nv_config") and self.nv_config.get("proxy"):
             kwargs.setdefault("proxy", self.nv_config.get("proxy"))
 
-        return await super(BetterClientSession, self)._request(*args, **kwargs)
+        res = await super(BetterClientSession, self)._request(
+            *args, **kwargs)
+        if res.status >= 400:
+            raise HTTPError(res.status, res.reason, res)
+        return res
 
 session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5)
 atexit.register(session.close)

nvchecker/source/android_sdk.py

@@ -34,7 +34,7 @@ async def _get_repo_manifest(repo):
 
     return repo_manifest
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   repo = conf['repo']
   pkg_path_prefix = conf['android_sdk']
 

nvchecker/source/anitya.py

@@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__)
 
 URL = 'https://release-monitoring.org/api/project/{pkg}'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   pkg = conf.get('anitya')
   url = URL.format(pkg = pkg)
 

nvchecker/source/archpkg.py

@@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__)
 
 URL = 'https://www.archlinux.org/packages/search/json/'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   pkg = conf.get('archpkg') or name
   strip_release = conf.getboolean('strip-release', False)
   async with session.get(URL, params={"name": pkg}) as res:

nvchecker/source/aur.py

@@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__)
 
 AUR_URL = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg[]='
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   aurname = conf.get('aur') or name
   strip_release = conf.getboolean('strip-release', False)
   async with session.get(AUR_URL, params={"v": 5, "type": "info", "arg[]": aurname}) as res:

nvchecker/source/bitbucket.py

@@ -8,7 +8,7 @@ from ..sortversion import sort_version_keys
 BITBUCKET_URL = 'https://bitbucket.org/api/2.0/repositories/%s/commits/%s'
 BITBUCKET_MAX_TAG = 'https://bitbucket.org/api/1.0/repositories/%s/tags'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   repo = conf.get('bitbucket')
   br = conf.get('branch', '')
   use_max_tag = conf.getboolean('use_max_tag', False)

nvchecker/source/cmd.py

@@ -7,7 +7,7 @@ import structlog
 
 logger = structlog.get_logger(logger_name=__name__)
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   cmd = conf['cmd']
   p = await asyncio.create_subprocess_shell(
     cmd,

nvchecker/source/cratesio.py

@@ -6,7 +6,7 @@ from . import session
 
 API_URL = 'https://crates.io/api/v1/crates/%s'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   name = conf.get('cratesio') or name
   async with session.get(API_URL % name) as res:
     data = await res.json()

nvchecker/source/debianpkg.py

@@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__)
 
 URL = 'https://sources.debian.org/api/src/%(pkgname)s/?suite=%(suite)s'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   pkg = conf.get('debianpkg') or name
   strip_release = conf.getboolean('strip-release', False)
   suite = conf.get('suite') or "sid"

nvchecker/source/github.py

@@ -3,11 +3,12 @@
 
 import os
 import re
+import time
 from functools import partial
 
 import structlog
 
-from . import session
+from . import session, HTTPError
 from ..sortversion import sort_version_keys
 
 logger = structlog.get_logger(logger_name=__name__)
@@ -16,7 +17,13 @@ GITHUB_URL = 'https://api.github.com/repos/%s/commits'
 GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest'
 GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/tags'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
+  try:
+    return await get_version_real(name, conf, **kwargs)
+  except HTTPError as e:
+    check_ratelimit(e, name)
+
+async def get_version_real(name, conf, **kwargs):
   repo = conf.get('github')
   br = conf.get('branch')
   use_latest_release = conf.getboolean('use_latest_release', False)
@@ -38,6 +45,10 @@ async def get_version(name, conf):
   }
   if 'NVCHECKER_GITHUB_TOKEN' in os.environ:
     headers['Authorization'] = 'token %s' % os.environ['NVCHECKER_GITHUB_TOKEN']
+  else:
+    key = kwargs['keyman'].get_key('github')
+    if key:
+      headers['Authorization'] = 'token %s' % key
 
   kwargs = {}
   if conf.get('proxy'):
@@ -51,6 +62,8 @@ async def get_version(name, conf):
     )
 
   async with session.get(url, headers=headers, **kwargs) as res:
+    logger.debug('X-RateLimit-Remaining',
+                  n=res.headers.get('X-RateLimit-Remaining'))
     data = await res.json()
 
   if use_latest_release:
@@ -75,6 +88,8 @@ async def max_tag(
 
   while True:
     async with getter(url) as res:
+      logger.debug('X-RateLimit-Remaining',
+                    n=res.headers.get('X-RateLimit-Remaining'))
       links = res.headers.get('Link')
       data = await res.json()
 
@@ -104,3 +119,16 @@ def get_next_page_url(links):
     return
 
   return next_link[0].split('>', 1)[0][1:]
+
+def check_ratelimit(exc, name):
+  res = exc.response
+  n = int(res.headers.get('X-RateLimit-Remaining'))
+  if n == 0:
+    reset = int(res.headers.get('X-RateLimit-Reset'))
+    logger.error('rate limited, resetting at %s. '
+                 'Or get an API token to increase the allowance if not yet'
+                 % time.ctime(reset),
+                 name = name,
+                 reset = reset)
+  else:
+    raise

nvchecker/source/gitlab.py

@@ -1,5 +1,5 @@
 # MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2018 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import os
 import urllib.parse
@@ -14,7 +14,7 @@ logger = structlog.get_logger(logger_name=__name__)
 GITLAB_URL = 'https://%s/api/v3/projects/%s/repository/commits?ref_name=%s'
 GITLAB_MAX_TAG = 'https://%s/api/v3/projects/%s/repository/tags'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   repo = urllib.parse.quote_plus(conf.get('gitlab'))
   br = conf.get('branch', 'master')
   host = conf.get('host', "gitlab.com")
@@ -22,8 +22,16 @@ async def get_version(name, conf):
   ignored_tags = conf.get("ignored_tags", "").split()
   sort_version_key = sort_version_keys[conf.get("sort_version_key", "parse_version")]
 
-  env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_")
-  token = conf.get('token', os.environ.get(env_name, None))
+  token = conf.get('token')
+
+  if token is None:
+    env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_")
+    global_key = os.environ.get(env_name)
+    if not global_key:
+      key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_")
+      global_key = kwargs['keyman'].get_key(key_name)
+    token = global_key
+
   if token is None:
     logger.error('No gitlab token specified.', name=name)
     return

nvchecker/source/manual.py

@@ -1,5 +1,5 @@
 # MIT licensed
 # Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   return conf.get('manual').strip() or None

nvchecker/source/pacman.py

@@ -3,7 +3,7 @@
 
 from . import cmd
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   referree = conf.get('pacman') or name
   c = "LANG=C pacman -Si %s | grep -F Version | awk '{print $3}'" % referree
   conf['cmd'] = c

nvchecker/source/regex.py

@@ -11,7 +11,7 @@ from ..sortversion import sort_version_keys
 
 logger = structlog.get_logger(logger_name=__name__)
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   try:
     regex = re.compile(conf['regex'])
   except sre_constants.error:

nvchecker/source/simple_json.py

@@ -5,7 +5,7 @@ from . import session
 
 def simple_json(urlpat, confkey, version_from_json):
 
-  async def get_version(name, conf):
+  async def get_version(name, conf, **kwargs):
     repo = conf.get(confkey) or name
     url = urlpat % repo
     kwargs = {}

nvchecker/source/tornado_httpclient.py

@@ -5,6 +5,7 @@ import json
 from urllib.parse import urlencode
 
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPResponse
+from tornado.httpclient import HTTPError
 from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future
 AsyncIOMainLoop().install()
 
@@ -14,7 +15,7 @@ try:
 except ImportError:
   pycurl = None
 
-__all__ = ['session']
+__all__ = ['session', 'HTTPError']
 
 client = AsyncHTTPClient()
 HTTP2_AVAILABLE = None if pycurl else False

nvchecker/source/ubuntupkg.py

@@ -9,7 +9,7 @@ logger = structlog.get_logger(logger_name=__name__)
 
 URL = 'https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name=%s&exact_match=true'
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   pkg = conf.get('ubuntupkg') or name
   strip_release = conf.getboolean('strip-release', False)
   suite = conf.get('suite')

nvchecker/source/vcs.py

@@ -26,7 +26,7 @@ def _parse_oldver(oldver):
         return PROT_VER, 0, ver
     return PROT_VER, count, ver
 
-async def get_version(name, conf):
+async def get_version(name, conf, **kwargs):
   vcs = conf['vcs']
   use_max_tag = conf.getboolean('use_max_tag', False)
   ignored_tags = conf.get("ignored_tags", "").split()

setup.py

@@ -15,7 +15,7 @@ setup(
   url = 'https://github.com/lilydjwg/nvchecker',
   long_description = open('README.rst', encoding='utf-8').read(),
   platforms = 'any',
-  zip_safe = True,
+  zip_safe = False,
 
   packages = find_packages(exclude=["tests"]),
   install_requires = ['setuptools', 'structlog'],

tests/conftest.py

@@ -1,8 +1,37 @@
 import configparser
 import pytest
 import asyncio
+import io
 
 from nvchecker.get_version import get_version as _get_version
+from nvchecker.core import Source
+
+class TestSource(Source):
+    def __init__(self, future, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._future = future
+
+    def on_update(self, name, version, oldver):
+        self._future.set_result(version)
+
+    def on_no_result(self, name):
+        self._future.set_result(None)
+
+    def on_exception(self, name, exc):
+        self._future.set_exception(exc)
+
+@pytest.fixture(scope="module")
+async def run_source():
+    async def __call__(conf):
+        future = asyncio.Future()
+        file = io.StringIO(conf)
+        file.name = '<StringIO>'
+
+        s = TestSource(future, file)
+        await s.check()
+        return await future
+
+    return __call__
 
 @pytest.fixture(scope="module")
 async def get_version():

tests/test_aur.py

@@ -5,10 +5,10 @@ from flaky import flaky
 import pytest
 pytestmark = pytest.mark.asyncio
 
-@flaky(max_runs=5)
+@flaky(max_runs=10)
 async def test_aur(get_version):
     assert await get_version("ssed", {"aur": None}) == "3.62-2"
 
-@flaky(max_runs=5)
+@flaky(max_runs=10)
 async def test_aur_strip_release(get_version):
     assert await get_version("ssed", {"aur": None, "strip-release": 1}) == "3.62"

tests/test_keyfile.py

@@ -0,0 +1,77 @@
+# MIT licensed
+# Copyright (c) 2018 lilydjwg <lilydjwg@gmail.com>, et al.
+
+import os
+import tempfile
+import contextlib
+
+from nvchecker.source import HTTPError
+
+import pytest
+pytestmark = [pytest.mark.asyncio]
+
+@contextlib.contextmanager
+def unset_github_token_env():
+  token = os.environ.get('NVCHECKER_GITHUB_TOKEN')
+  try:
+    if token:
+      del os.environ['NVCHECKER_GITHUB_TOKEN']
+    yield token
+  finally:
+    if token:
+      os.environ['NVCHECKER_GITHUB_TOKEN'] = token
+
+async def test_keyfile_missing(run_source):
+  test_conf = '''\
+[example]
+github = harry-sanabria/ReleaseTestRepo
+'''
+
+  assert await run_source(test_conf) in ['20140122.012101', None]
+
+async def test_keyfile_invalid(run_source):
+  with tempfile.NamedTemporaryFile(mode='w') as f, \
+    unset_github_token_env():
+    f.write('''\
+[keys]
+github = xxx
+            ''')
+    f.flush()
+    test_conf = '''\
+[example]
+github = harry-sanabria/ReleaseTestRepo
+
+[__config__]
+keyfile = {name}
+'''.format(name=f.name)
+
+    try:
+      version = await run_source(test_conf)
+      assert version is None # out of allowance
+      return
+    except HTTPError as e:
+      assert e.code == 401
+      return
+
+    raise Exception('expected 401 response')
+
+@pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ,
+                    reason='no key given')
+async def test_keyfile_valid(run_source):
+  with tempfile.NamedTemporaryFile(mode='w') as f, \
+    unset_github_token_env() as token:
+    f.write('''\
+[keys]
+github = {token}
+            '''.format(token=token))
+    f.flush()
+
+    test_conf = '''\
+[example]
+github = harry-sanabria/ReleaseTestRepo
+
+[__config__]
+keyfile = {name}
+      '''.format(name=f.name)
+
+    assert await run_source(test_conf) == '20140122.012101'

tests/test_proxy.py

@@ -20,6 +20,8 @@ async def test_proxy(get_version, monkeypatch):
 
   async def fake_request(*args, proxy, **kwargs):
     class fake_response():
+      status = 200
+
       async def read():
         return proxy.encode("ascii")
 

tests/test_regex.py

@@ -4,9 +4,17 @@
 import pytest
 pytestmark = pytest.mark.asyncio
 
-async def test_regex(get_version):
+@pytest.mark.skipif(True,
+                    reason='httpbin is overloaded?')
+async def test_regex_httpbin(get_version):
     assert await get_version("example", {
         "url": "https://httpbin.org/get",
         "regex": '"User-Agent": "(\w+)"',
         "user_agent": "Meow",
     }) == "Meow"
+
+async def test_regex(get_version):
+    assert await get_version("example", {
+        "url": "https://example.net/",
+        "regex": 'for (\w+) examples',
+    }) == "illustrative"