From 789731729472824e79e519c033a432a35cf402b2 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 6 Mar 2019 21:51:04 +0800 Subject: [PATCH] add the "list options" feature closes #83 old options will be removed in #99 --- README.rst | 14 ++++++++------ nvchecker/get_version.py | 27 +++++++++++++++++++++++++++ nvchecker/source/github.py | 9 ++------- nvchecker/source/gitlab.py | 6 +----- nvchecker/source/regex.py | 4 +--- tests/test_github.py | 13 ++++++++++++- tests/test_gitlab.py | 13 +++++++++++++ 7 files changed, 64 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index c5044b1..5764606 100644 --- a/README.rst +++ b/README.rst @@ -197,16 +197,18 @@ The following options apply to sources that return a list. See individual source sections to determine whether they are supported. include_regex - Only consider version strings that match the given regex. + Only consider version strings that match the given regex. The whole string + should match the regex. sort_version_key - Sort the version string using this key function. Choose between ``parse_version`` and - ``vercmp``. Default value is ``parse_version``. ``parse_version`` use - ``pkg_resources.parse_version``. ``vercmp`` use ``pyalpm.vercmp``. + Sort the version string using this key function. Choose between + ``parse_version`` and ``vercmp``. Default value is ``parse_version``. + ``parse_version`` use ``pkg_resources.parse_version``. ``vercmp`` use + ``pyalpm.vercmp``. ignored - Version strings that are explicitly ignored, separated by whitespace. This can - be useful to avoid some known mis-named versions, so newer ones won't be + Version strings that are explicitly ignored, separated by whitespace. This + can be useful to avoid some known mis-named versions, so newer ones won't be "overridden" by the old broken ones. Search in a Webpage diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index cd79670..c7b28e5 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -6,6 +6,8 @@ from importlib import import_module import structlog +from .sortversion import sort_version_keys + logger = structlog.get_logger(logger_name=__name__) handler_precedence = ( @@ -38,6 +40,26 @@ def substitute_version(version, name, conf): # No substitution rules found. Just return the original version string. return version +def apply_list_options(versions, conf): + pattern = conf.get('include_regex') + if pattern: + pattern = re.compile(pattern) + versions = [x for x in versions + if pattern.fullmatch(x)] + + ignored = set(conf.get('ignored', '').split()) + if ignored: + versions = [x for x in versions if x not in ignored] + + if not versions: + return + + sort_version_key = sort_version_keys[ + conf.get("sort_version_key", "parse_version")] + versions.sort(key=sort_version_key) + + return versions[-1] + _cache = {} async def get_version(name, conf, **kwargs): @@ -60,6 +82,10 @@ async def get_version(name, conf, **kwargs): return version version = await func(name, conf, **kwargs) + + if isinstance(version, list): + version = apply_list_options(version, conf) + if version: version = version.replace('\n', ' ') try: @@ -69,4 +95,5 @@ async def get_version(name, conf, **kwargs): if version is not None: _cache[cache_key] = version + return version diff --git a/nvchecker/source/github.py b/nvchecker/source/github.py index cc0b5be..5ac5985 100644 --- a/nvchecker/source/github.py +++ b/nvchecker/source/github.py @@ -10,7 +10,6 @@ from functools import partial import structlog from . import session, HTTPError -from ..sortversion import sort_version_keys logger = structlog.get_logger(logger_name=__name__) @@ -32,7 +31,6 @@ async def get_version_real(name, conf, **kwargs): use_max_tag = conf.getboolean('use_max_tag', False) include_tags_pattern = conf.get("include_tags_pattern", "") ignored_tags = conf.get("ignored_tags", "").split() - sort_version_key = sort_version_keys[conf.get("sort_version_key", "parse_version")] if use_latest_release: url = GITHUB_LATEST_RELEASE % repo elif use_max_tag: @@ -64,7 +62,6 @@ async def get_version_real(name, conf, **kwargs): return await max_tag(partial( session.get, headers=headers, **kwargs), url, name, ignored_tags, include_tags_pattern, - sort_version_key, ) async with session.get(url, headers=headers, **kwargs) as res: @@ -87,8 +84,7 @@ async def get_version_real(name, conf, **kwargs): return version async def max_tag( - getter, url, name, - ignored_tags, include_tags_pattern, sort_version_key, + getter, url, name, ignored_tags, include_tags_pattern, ): # paging is needed @@ -104,8 +100,7 @@ async def max_tag( data = [x for x in data if re.search(include_tags_pattern, x)] if data: - data.sort(key=sort_version_key) - return data[-1] + return data else: next_page_url = get_next_page_url(links) if not next_page_url: diff --git a/nvchecker/source/gitlab.py b/nvchecker/source/gitlab.py index 779334d..a0bbc9e 100644 --- a/nvchecker/source/gitlab.py +++ b/nvchecker/source/gitlab.py @@ -7,7 +7,6 @@ import urllib.parse import structlog from . import session, HTTPError -from ..sortversion import sort_version_keys logger = structlog.get_logger(logger_name=__name__) @@ -26,7 +25,6 @@ async def get_version_real(name, conf, **kwargs): host = conf.get('host', "gitlab.com") use_max_tag = conf.getboolean('use_max_tag', False) ignored_tags = conf.get("ignored_tags", "").split() - sort_version_key = sort_version_keys[conf.get("sort_version_key", "parse_version")] if use_max_tag: url = GITLAB_MAX_TAG % (host, repo) @@ -52,9 +50,7 @@ async def get_version_real(name, conf, **kwargs): async with session.get(url, headers=headers) as res: data = await res.json() if use_max_tag: - data = [tag["name"] for tag in data if tag["name"] not in ignored_tags] - data.sort(key=sort_version_key) - version = data[-1] + version = [tag["name"] for tag in data if tag["name"] not in ignored_tags] else: version = data[0]['created_at'].split('T', 1)[0].replace('-', '') return version diff --git a/nvchecker/source/regex.py b/nvchecker/source/regex.py index 38450f5..ac61fd5 100644 --- a/nvchecker/source/regex.py +++ b/nvchecker/source/regex.py @@ -7,7 +7,6 @@ import sre_constants import structlog from . import session -from ..sortversion import sort_version_keys logger = structlog.get_logger(logger_name=__name__) @@ -26,12 +25,11 @@ async def get_version(name, conf, **kwargs): kwargs["proxy"] = conf.get("proxy") if conf.get('user_agent'): headers['User-Agent'] = conf['user_agent'] - sort_version_key = sort_version_keys[conf.get("sort_version_key", "parse_version")] async with session.get(conf['url'], headers=headers, **kwargs) as res: body = (await res.read()).decode(encoding) try: - version = max(regex.findall(body), key=sort_version_key) + version = regex.findall(body) except ValueError: version = None if not conf.getboolean('missing_ok', False): diff --git a/tests/test_github.py b/tests/test_github.py index ff7aae2..0a076e2 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -23,16 +23,27 @@ async def test_github_max_tag(get_version): async def test_github_max_tag_with_ignored_tags(get_version): assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1, "ignored_tags": "second_release release3"}) == "first_release" +async def test_github_max_tag_with_ignored(get_version): + assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1, "ignored": "second_release release3"}) == "first_release" + async def test_github_with_path(get_version): assert await get_version("example", {"github": "petronny/ReleaseTestRepo", "path": "test_directory"}) == "20140122.012101" async def test_github_with_path_and_branch(get_version): assert await get_version("example", {"github": "petronny/ReleaseTestRepo", "branch": "test", "path": "test_directory/test_directory"}) == "20190128.113201" -async def test_github_max_tag_with_include(get_version): +async def test_github_max_tag_with_include_old(get_version): version = await get_version("example", { "github": "EFForg/https-everywhere", "use_max_tag": 1, "include_tags_pattern": r"^\d", }) assert re.match(r'[\d.]+', version) + +async def test_github_max_tag_with_include(get_version): + version = await get_version("example", { + "github": "EFForg/https-everywhere", + "use_max_tag": 1, + "include_regex": r"^\d", + }) + assert re.match(r'[\d.]+', version) diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py index 3e7c381..24ecea3 100644 --- a/tests/test_gitlab.py +++ b/tests/test_gitlab.py @@ -18,3 +18,16 @@ async def test_gitlab_max_tag(get_version): async def test_gitlab_max_tag_with_ignored_tags(get_version): assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, "ignored_tags": "v1.1.0"}) == "v1.0.0" + +async def test_gitlab_max_tag_with_include(get_version): + assert await get_version("example", { + "gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, + "include": r'v1\.0.*', + }) == "v1.0.0" + +async def test_gitlab_max_tag_with_ignored(get_version): + assert await get_version("example", { + "gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, + "ignored": "v1.1.0", + }) == "v1.0.0" +