diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index b8a75b5..cd79670 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -38,15 +38,27 @@ def substitute_version(version, name, conf): # No substitution rules found. Just return the original version string. return version +_cache = {} + async def get_version(name, conf, **kwargs): for key in handler_precedence: if key in conf: - func = import_module('.source.' + key, __package__).get_version + mod = import_module('.source.' + key, __package__) + func = mod.get_version + get_cacheable_conf = getattr(mod, 'get_cacheable_conf', lambda name, conf: conf) break else: logger.error('no idea to get version info.', name=name) return + cacheable_conf = get_cacheable_conf(name, conf) + cache_key = tuple(sorted(cacheable_conf.items())) + if cache_key in _cache: + version = _cache[cache_key] + logger.debug('cache hit', name=name, + cache_key=cache_key, cached=version) + return version + version = await func(name, conf, **kwargs) if version: version = version.replace('\n', ' ') @@ -54,4 +66,7 @@ async def get_version(name, conf, **kwargs): version = substitute_version(version, name, conf) except (ValueError, re.error): logger.exception('error occurred in version substitutions', name=name) + + if version is not None: + _cache[cache_key] = version return version diff --git a/nvchecker/source/__init__.py b/nvchecker/source/__init__.py index 024d9b0..ae79e12 100644 --- a/nvchecker/source/__init__.py +++ b/nvchecker/source/__init__.py @@ -19,3 +19,10 @@ m = __import__('%s_httpclient' % which, globals(), locals(), level=1) __all__ = m.__all__ for x in __all__: globals()[x] = getattr(m, x) + +def conf_cacheable_with_name(key): + def get_cacheable_conf(name, conf): + conf = dict(conf) + conf[key] = conf.get(key) or name + return conf + return get_cacheable_conf diff --git a/nvchecker/source/archpkg.py b/nvchecker/source/archpkg.py index b57bf16..cb6a192 100644 --- a/nvchecker/source/archpkg.py +++ b/nvchecker/source/archpkg.py @@ -3,12 +3,14 @@ import structlog -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) URL = 'https://www.archlinux.org/packages/search/json/' +get_cacheable_conf = conf_cacheable_with_name('archpkg') + async def get_version(name, conf, **kwargs): pkg = conf.get('archpkg') or name strip_release = conf.getboolean('strip-release', False) diff --git a/nvchecker/source/aur.py b/nvchecker/source/aur.py index fc4b0eb..1ec378e 100644 --- a/nvchecker/source/aur.py +++ b/nvchecker/source/aur.py @@ -4,12 +4,14 @@ import structlog from datetime import datetime -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) AUR_URL = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg[]=' +get_cacheable_conf = conf_cacheable_with_name('aur') + async def get_version(name, conf, **kwargs): aurname = conf.get('aur') or name use_last_modified = conf.getboolean('use_last_modified', False) diff --git a/nvchecker/source/cpan.py b/nvchecker/source/cpan.py index 22c3888..9fbfbe7 100644 --- a/nvchecker/source/cpan.py +++ b/nvchecker/source/cpan.py @@ -9,7 +9,7 @@ CPAN_URL = 'https://fastapi.metacpan.org/release/%s' def _version_from_json(data): return str(data['version']) -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( CPAN_URL, 'cpan', _version_from_json, diff --git a/nvchecker/source/cratesio.py b/nvchecker/source/cratesio.py index 5449665..9aea99c 100644 --- a/nvchecker/source/cratesio.py +++ b/nvchecker/source/cratesio.py @@ -1,11 +1,12 @@ # MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. +# Copyright (c) 2013-2018 lilydjwg , et al. -import os -from . import session +from . import session, conf_cacheable_with_name API_URL = 'https://crates.io/api/v1/crates/%s' +get_cacheable_conf = conf_cacheable_with_name('cratesio') + async def get_version(name, conf, **kwargs): name = conf.get('cratesio') or name async with session.get(API_URL % name) as res: diff --git a/nvchecker/source/debianpkg.py b/nvchecker/source/debianpkg.py index d5ff0da..336315a 100644 --- a/nvchecker/source/debianpkg.py +++ b/nvchecker/source/debianpkg.py @@ -3,12 +3,14 @@ import structlog -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) URL = 'https://sources.debian.org/api/src/%(pkgname)s/?suite=%(suite)s' +get_cacheable_conf = conf_cacheable_with_name('debianpkg') + async def get_version(name, conf, **kwargs): pkg = conf.get('debianpkg') or name strip_release = conf.getboolean('strip-release', False) diff --git a/nvchecker/source/gems.py b/nvchecker/source/gems.py index 2d79bce..308e56d 100644 --- a/nvchecker/source/gems.py +++ b/nvchecker/source/gems.py @@ -8,7 +8,7 @@ GEMS_URL = 'https://rubygems.org/api/v1/versions/%s.json' def _version_from_json(data): return data[0]['number'] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( GEMS_URL, 'gems', _version_from_json, diff --git a/nvchecker/source/hackage.py b/nvchecker/source/hackage.py index 178c94d..79ced66 100644 --- a/nvchecker/source/hackage.py +++ b/nvchecker/source/hackage.py @@ -8,7 +8,7 @@ HACKAGE_URL = 'https://hackage.haskell.org/package/%s/preferred.json' def _version_from_json(data): return data['normal-version'][0] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( HACKAGE_URL, 'hackage', _version_from_json, diff --git a/nvchecker/source/npm.py b/nvchecker/source/npm.py index e046197..256c4b4 100644 --- a/nvchecker/source/npm.py +++ b/nvchecker/source/npm.py @@ -8,7 +8,7 @@ NPM_URL = 'https://registry.npmjs.org/%s' def _version_from_json(data): return data['dist-tags']['latest'] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( NPM_URL, 'npm', _version_from_json, diff --git a/nvchecker/source/packagist.py b/nvchecker/source/packagist.py index 7e2397a..dbb4d6d 100644 --- a/nvchecker/source/packagist.py +++ b/nvchecker/source/packagist.py @@ -11,7 +11,7 @@ def _version_from_json(data): if len(data): return max(data, key=lambda version: data[version]["time"]) -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( PACKAGIST_URL, 'packagist', _version_from_json, diff --git a/nvchecker/source/pacman.py b/nvchecker/source/pacman.py index aa487a4..65de504 100644 --- a/nvchecker/source/pacman.py +++ b/nvchecker/source/pacman.py @@ -1,7 +1,9 @@ # MIT licensed # Copyright (c) 2013-2017 lilydjwg , et al. -from . import cmd +from . import cmd, conf_cacheable_with_name + +get_cacheable_conf = conf_cacheable_with_name('debianpkg') async def get_version(name, conf, **kwargs): referree = conf.get('pacman') or name diff --git a/nvchecker/source/pypi.py b/nvchecker/source/pypi.py index 6915e15..4b2ff69 100644 --- a/nvchecker/source/pypi.py +++ b/nvchecker/source/pypi.py @@ -8,7 +8,7 @@ PYPI_URL = 'https://pypi.python.org/pypi/%s/json' def _version_from_json(data): return data['info']['version'] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( PYPI_URL, 'pypi', _version_from_json, diff --git a/nvchecker/source/simple_json.py b/nvchecker/source/simple_json.py index 5561fa5..0f7f879 100644 --- a/nvchecker/source/simple_json.py +++ b/nvchecker/source/simple_json.py @@ -1,7 +1,7 @@ # MIT licensed # Copyright (c) 2013-2017 lilydjwg , et al. -from . import session +from . import session, conf_cacheable_with_name def simple_json(urlpat, confkey, version_from_json): @@ -17,4 +17,6 @@ def simple_json(urlpat, confkey, version_from_json): version = version_from_json(data) return version - return get_version + get_cacheable_conf = conf_cacheable_with_name(confkey) + + return get_version, get_cacheable_conf diff --git a/nvchecker/source/ubuntupkg.py b/nvchecker/source/ubuntupkg.py index 98c7083..a72be41 100644 --- a/nvchecker/source/ubuntupkg.py +++ b/nvchecker/source/ubuntupkg.py @@ -3,12 +3,14 @@ import structlog -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) URL = 'https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name=%s&exact_match=true' +get_cacheable_conf = conf_cacheable_with_name('ubuntupkg') + async def get_version(name, conf, **kwargs): pkg = conf.get('ubuntupkg') or name strip_release = conf.getboolean('strip-release', False) diff --git a/tests/test_anitya.py b/tests/test_anitya.py index 8ec7943..cf0a818 100644 --- a/tests/test_anitya.py +++ b/tests/test_anitya.py @@ -5,4 +5,4 @@ import pytest pytestmark = pytest.mark.asyncio async def test_anitya(get_version): - assert await get_version("shutter", {"anitya": "fedora/shutter"}) == "0.94" + assert await get_version("shutter", {"anitya": "fedora/shutter"}) == "0.94.2"