From 3628505ea74960de1b05f1b0ac6c28794c3c73fa Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 10 Oct 2017 00:39:24 +0800 Subject: [PATCH 1/6] Support substitution of version strings --- nvchecker/get_version.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index 8a94d23..d0d8489 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -2,6 +2,7 @@ # Copyright (c) 2013-2017 lilydjwg , et al. import logging +import re from importlib import import_module logger = logging.getLogger(__name__) @@ -13,13 +14,45 @@ handler_precedence = ( 'anitya', ) +def substitute_version(version, name, conf): + ''' + Substitute the version string via defined rules in the configuration file. + Accepted rules are: + + * from_pattern, to_pattern: Both should be Python regular expressions. + `from_pattern` found in the version string will be substituted to + `to_pattern`. + * prefix: If the version string starts with `prefix`, the prefix is removed. + Otherwise the version string is returned as-is. + + If both prefix and from_pattern/to_pattern are used, from_pattern/to_pattern + are ignored. + ''' + prefix = conf.get('prefix') + if prefix: + if version.startswith(prefix): + version = version[len(prefix):] + return version + + from_pattern = conf.get('from_pattern') + if from_pattern: + to_pattern = conf.get('to_pattern') + if not to_pattern: + raise ValueError('%s: from_pattern exists but to_pattern doesn\'t', name) + + return re.sub(from_pattern, to_pattern, version) + + # No substitution rules found. Just return the original version string. + return version + async def get_version(name, conf): for key in handler_precedence: if key in conf: func = import_module('.source.' + key, __package__).get_version version = await func(name, conf) if version: - version.replace('\n', ' ') + version = version.replace('\n', ' ') + version = substitute_version(version, name, conf) return version else: logger.error('%s: no idea to get version info.', name) From ffa6eb7b8f6f38b3a36246b7b0bc556f86e70665 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 11 Oct 2017 01:50:44 +0800 Subject: [PATCH 2/6] Add documents and tests for substitution commands --- README.rst | 11 +++++++++++ tests/test_substitute.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/test_substitute.py diff --git a/README.rst b/README.rst index 80b922e..47c6bbc 100644 --- a/README.rst +++ b/README.rst @@ -124,6 +124,17 @@ proxy max_concurrent Max number of concurrent jobs. Default: 20. +Global Options +-------------- +The following options applies to all checkers. + +prefix + Strip the prefix string if the version string starts with it. + +from_pattern, to_pattern + Both are Python-compatible regular expressions. If ``from_pattern`` is found + in the version string, it will be replaced with ``to_pattern`` + Search in a Webpage ------------------- Search through a specific webpage for the version string. This type of version finding has these fields: diff --git a/tests/test_substitute.py b/tests/test_substitute.py new file mode 100644 index 0000000..d1f8af7 --- /dev/null +++ b/tests/test_substitute.py @@ -0,0 +1,17 @@ +# MIT licensed +# Copyright (c) 2013-2017 lilydjwg , et al. + +import pytest +pytestmark = pytest.mark.asyncio + +async def test_substitute_prefix(get_version): + assert await get_version("example", {"manual": "v1.0", "prefix": "v"}) == "1.0" + +async def test_substitute_prefix_missing_ok(get_version): + assert await get_version("example", {"manual": "1.0", "prefix": "v"}) == "1.0" + +async def test_substitute_regex(get_version): + assert await get_version("example", {"manual": "r15c", "from_pattern": r"r(\d+)([a-z])", "to_pattern": r"r\1.\2"}) == "r15.c" + +async def test_substitute_regex_missing_ok(get_version): + assert await get_version("example", {"manual": "r15", "from_pattern": r"r(\d+)([a-z])", "to_pattern": r"r\1.\2"}) == "r15" From b834f9a81c8ba110997f10c6e3e6bcb7b14ca9f3 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 11 Oct 2017 01:57:35 +0800 Subject: [PATCH 3/6] Add a link for the new section --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 47c6bbc..5b8b022 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,7 @@ Contents * `Version Source Files <#version-source-files>`_ * `Configuration Section <#configuration-section>`_ + * `Global Optons <#global-options>`_ * `Search in a Webpage <#search-in-a-webpage>`_ * `Find with a Command <#find-with-a-command>`_ * `Check AUR <#check-aur>`_ From 78fef9d34d021336820554e4427fde624ba17022 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 11 Oct 2017 01:58:21 +0800 Subject: [PATCH 4/6] Fix README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5b8b022..7e95101 100644 --- a/README.rst +++ b/README.rst @@ -134,7 +134,7 @@ prefix from_pattern, to_pattern Both are Python-compatible regular expressions. If ``from_pattern`` is found - in the version string, it will be replaced with ``to_pattern`` + in the version string, it will be replaced with ``to_pattern``. Search in a Webpage ------------------- From 05b604530c656cc2bf7ed80d47d31f0e66ec0378 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 12 Oct 2017 22:46:30 +0800 Subject: [PATCH 5/6] Refine docs of substitution options and add a test for option priority --- README.rst | 11 +++++++++-- nvchecker/get_version.py | 11 +---------- tests/test_substitute.py | 3 +++ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 7e95101..e0925d4 100644 --- a/README.rst +++ b/README.rst @@ -127,15 +127,22 @@ max_concurrent Global Options -------------- -The following options applies to all checkers. +The following options apply to all checkers. prefix - Strip the prefix string if the version string starts with it. + Strip the prefix string if the version string starts with it. Otherwise the + version string is returned as-is. from_pattern, to_pattern Both are Python-compatible regular expressions. If ``from_pattern`` is found in the version string, it will be replaced with ``to_pattern``. +If both ``prefix`` and ``from_pattern``/``to_pattern`` are used, +``from_pattern``/``to_pattern`` are ignored. If you want to strip the prefix +and then do something special, just use ``from_pattern```/``to_pattern``. For +example, the transformation of ``v1_1_0`` => ``1.1.0`` can be achieved with +``from_pattern = v(\d+)_(\d+)_(\d+)`` and ``to_pattern = \1.\2.\3``. + Search in a Webpage ------------------- Search through a specific webpage for the version string. This type of version finding has these fields: diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index d0d8489..b4668a8 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -17,16 +17,7 @@ handler_precedence = ( def substitute_version(version, name, conf): ''' Substitute the version string via defined rules in the configuration file. - Accepted rules are: - - * from_pattern, to_pattern: Both should be Python regular expressions. - `from_pattern` found in the version string will be substituted to - `to_pattern`. - * prefix: If the version string starts with `prefix`, the prefix is removed. - Otherwise the version string is returned as-is. - - If both prefix and from_pattern/to_pattern are used, from_pattern/to_pattern - are ignored. + See README.rst#global-options for details. ''' prefix = conf.get('prefix') if prefix: diff --git a/tests/test_substitute.py b/tests/test_substitute.py index d1f8af7..1b033ec 100644 --- a/tests/test_substitute.py +++ b/tests/test_substitute.py @@ -15,3 +15,6 @@ async def test_substitute_regex(get_version): async def test_substitute_regex_missing_ok(get_version): assert await get_version("example", {"manual": "r15", "from_pattern": r"r(\d+)([a-z])", "to_pattern": r"r\1.\2"}) == "r15" + +async def test_substitute_prefix_has_higher_priority(get_version): + assert await get_version("example", {"manual": "r15", "prefix": "r", "from_pattern": "r(\d+)", "to_pattern": "R\1"}) == "15" From a1222a0820fad4964d9272d79f9f49cc192f4932 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 12 Oct 2017 22:56:24 +0800 Subject: [PATCH 6/6] Catch exceptions with version substitutions and log them --- nvchecker/get_version.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index b4668a8..1c8e6eb 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -43,7 +43,10 @@ async def get_version(name, conf): version = await func(name, conf) if version: version = version.replace('\n', ' ') - version = substitute_version(version, name, conf) + try: + version = substitute_version(version, name, conf) + except (ValueError, re.error): + logger.exception('error occured in version substitutions for %s', name) return version else: logger.error('%s: no idea to get version info.', name)