cache result during a run

so that different entries can refer to a same software without much overhead.

This is needed because entries may be generated, and only nvchecker can
tell if two entries are expecting the same result because the name may
or may not be relevant.

Closes #81.

nvchecker/get_version.py

@@ -38,15 +38,27 @@ def substitute_version(version, name, conf):
   # No substitution rules found. Just return the original version string.
   return version
 
+_cache = {}
+
 async def get_version(name, conf, **kwargs):
   for key in handler_precedence:
     if key in conf:
-      func = import_module('.source.' + key, __package__).get_version
+      mod = import_module('.source.' + key, __package__)
+      func = mod.get_version
+      get_cacheable_conf = getattr(mod, 'get_cacheable_conf', lambda name, conf: conf)
       break
   else:
     logger.error('no idea to get version info.', name=name)
     return
 
+  cacheable_conf = get_cacheable_conf(name, conf)
+  cache_key = tuple(sorted(cacheable_conf.items()))
+  if cache_key in _cache:
+    version = _cache[cache_key]
+    logger.debug('cache hit', name=name,
+                 cache_key=cache_key, cached=version)
+    return version
+
   version = await func(name, conf, **kwargs)
   if version:
     version = version.replace('\n', ' ')
@@ -54,4 +66,7 @@ async def get_version(name, conf, **kwargs):
       version = substitute_version(version, name, conf)
     except (ValueError, re.error):
       logger.exception('error occurred in version substitutions', name=name)
+
+  if version is not None:
+    _cache[cache_key] = version
   return version

nvchecker/source/__init__.py

@@ -19,3 +19,10 @@ m = __import__('%s_httpclient' % which, globals(), locals(), level=1)
 __all__ = m.__all__
 for x in __all__:
   globals()[x] = getattr(m, x)
+
+def conf_cacheable_with_name(key):
+  def get_cacheable_conf(name, conf):
+    conf = dict(conf)
+    conf[key] = conf.get(key) or name
+    return conf
+  return get_cacheable_conf

nvchecker/source/archpkg.py

@@ -3,12 +3,14 @@
 
 import structlog
 
-from . import session
+from . import session, conf_cacheable_with_name
 
 logger = structlog.get_logger(logger_name=__name__)
 
 URL = 'https://www.archlinux.org/packages/search/json/'
 
+get_cacheable_conf = conf_cacheable_with_name('archpkg')
+
 async def get_version(name, conf, **kwargs):
   pkg = conf.get('archpkg') or name
   strip_release = conf.getboolean('strip-release', False)

nvchecker/source/aur.py

@@ -4,12 +4,14 @@
 import structlog
 from datetime import datetime
 
-from . import session
+from . import session, conf_cacheable_with_name
 
 logger = structlog.get_logger(logger_name=__name__)
 
 AUR_URL = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg[]='
 
+get_cacheable_conf = conf_cacheable_with_name('aur')
+
 async def get_version(name, conf, **kwargs):
   aurname = conf.get('aur') or name
   use_last_modified = conf.getboolean('use_last_modified', False)

nvchecker/source/cpan.py

@@ -9,7 +9,7 @@ CPAN_URL = 'https://fastapi.metacpan.org/release/%s'
 def _version_from_json(data):
   return str(data['version'])
 
-get_version = simple_json(
+get_version, get_cacheable_conf = simple_json(
   CPAN_URL,
   'cpan',
   _version_from_json,

nvchecker/source/cratesio.py

@@ -1,11 +1,12 @@
 # MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2018 lilydjwg <lilydjwg@gmail.com>, et al.
 
-import os
+from . import session, conf_cacheable_with_name
-from . import session
 
 API_URL = 'https://crates.io/api/v1/crates/%s'
 
+get_cacheable_conf = conf_cacheable_with_name('cratesio')
+
 async def get_version(name, conf, **kwargs):
   name = conf.get('cratesio') or name
   async with session.get(API_URL % name) as res:

nvchecker/source/debianpkg.py

@@ -3,12 +3,14 @@
 
 import structlog
 
-from . import session
+from . import session, conf_cacheable_with_name
 
 logger = structlog.get_logger(logger_name=__name__)
 
 URL = 'https://sources.debian.org/api/src/%(pkgname)s/?suite=%(suite)s'
 
+get_cacheable_conf = conf_cacheable_with_name('debianpkg')
+
 async def get_version(name, conf, **kwargs):
   pkg = conf.get('debianpkg') or name
   strip_release = conf.getboolean('strip-release', False)

nvchecker/source/gems.py

@@ -8,7 +8,7 @@ GEMS_URL = 'https://rubygems.org/api/v1/versions/%s.json'
 def _version_from_json(data):
   return data[0]['number']
 
-get_version = simple_json(
+get_version, get_cacheable_conf = simple_json(
   GEMS_URL,
   'gems',
   _version_from_json,

nvchecker/source/hackage.py

@@ -8,7 +8,7 @@ HACKAGE_URL = 'https://hackage.haskell.org/package/%s/preferred.json'
 def _version_from_json(data):
   return data['normal-version'][0]
 
-get_version = simple_json(
+get_version, get_cacheable_conf = simple_json(
   HACKAGE_URL,
   'hackage',
   _version_from_json,

nvchecker/source/npm.py

@@ -8,7 +8,7 @@ NPM_URL = 'https://registry.npmjs.org/%s'
 def _version_from_json(data):
   return data['dist-tags']['latest']
 
-get_version = simple_json(
+get_version, get_cacheable_conf = simple_json(
   NPM_URL,
   'npm',
   _version_from_json,

nvchecker/source/packagist.py

@@ -11,7 +11,7 @@ def _version_from_json(data):
   if len(data):
     return max(data, key=lambda version: data[version]["time"])
 
-get_version = simple_json(
+get_version, get_cacheable_conf = simple_json(
   PACKAGIST_URL,
   'packagist',
   _version_from_json,

nvchecker/source/pacman.py

@@ -1,7 +1,9 @@
 # MIT licensed
 # Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from . import cmd
+from . import cmd, conf_cacheable_with_name
+
+get_cacheable_conf = conf_cacheable_with_name('debianpkg')
 
 async def get_version(name, conf, **kwargs):
   referree = conf.get('pacman') or name

nvchecker/source/pypi.py

@@ -8,7 +8,7 @@ PYPI_URL = 'https://pypi.python.org/pypi/%s/json'
 def _version_from_json(data):
   return data['info']['version']
 
-get_version = simple_json(
+get_version, get_cacheable_conf = simple_json(
   PYPI_URL,
   'pypi',
   _version_from_json,

nvchecker/source/simple_json.py

@@ -1,7 +1,7 @@
 # MIT licensed
 # Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from . import session
+from . import session, conf_cacheable_with_name
 
 def simple_json(urlpat, confkey, version_from_json):
 
@@ -17,4 +17,6 @@ def simple_json(urlpat, confkey, version_from_json):
     version = version_from_json(data)
     return version
 
-  return get_version
+  get_cacheable_conf = conf_cacheable_with_name(confkey)
+
+  return get_version, get_cacheable_conf

nvchecker/source/ubuntupkg.py

@@ -3,12 +3,14 @@
 
 import structlog
 
-from . import session
+from . import session, conf_cacheable_with_name
 
 logger = structlog.get_logger(logger_name=__name__)
 
 URL = 'https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name=%s&exact_match=true'
 
+get_cacheable_conf = conf_cacheable_with_name('ubuntupkg')
+
 async def get_version(name, conf, **kwargs):
   pkg = conf.get('ubuntupkg') or name
   strip_release = conf.getboolean('strip-release', False)

tests/test_anitya.py

@@ -5,4 +5,4 @@ import pytest
 pytestmark = pytest.mark.asyncio
 
 async def test_anitya(get_version):
-    assert await get_version("shutter", {"anitya": "fedora/shutter"}) == "0.94"
+  assert await get_version("shutter", {"anitya": "fedora/shutter"}) == "0.94.2"