diff --git a/NEW b/NEW index e2e87d1..c71b351 100644 --- a/NEW +++ b/NEW @@ -1,3 +1,4 @@ TODO: -* update README * create source plugin documentation +* nvtake and nvcmp +* setup.py nvchecker_source diff --git a/README.rst b/README.rst index 230fb94..320f1ad 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,10 @@ **nvchecker** (short for *new version checker*) is for checking if a new version of some software has been released. +This is the 2.0 version branch that is still in development. Things may change with backward-compatibility. + .. image:: https://travis-ci.org/lilydjwg/nvchecker.svg :alt: Build Status - :target: https://travis-ci.org/lilydjwg/nvchecker + :target: https://travis-ci.org/lilydjwg/nvchecker?branch=nvchecker2 .. image:: https://badge.fury.io/py/nvchecker.svg :alt: PyPI version :target: https://badge.fury.io/py/nvchecker @@ -20,12 +22,13 @@ Contents * `Install and Run <#install-and-run>`_ * `JSON logging <#json-logging>`_ + * `Upgrade from 1.x version <#upgrade-from-1-x-version>`_ * `Version Record Files <#version-record-files>`_ * `The nvtake Command <#the-nvtake-command>`_ -* `Version Source Files <#version-source-files>`_ +* `Configuration Files <#configuration-files>`_ * `Configuration Section <#configuration-section>`_ * `Global Optons <#global-options>`_ @@ -53,7 +56,7 @@ Contents * `Check Sparkle framework <#check-sparkle-framework>`_ * `Manually updating <#manually-updating>`_ * `Version Control System (VCS) (git, hg, svn, bzr) <#version-control-system-vcs-git-hg-svn-bzr>`_ - * `Other <#other>`_ + * `Extending <#extending>`_ * `Bugs <#bugs>`_ @@ -114,6 +117,16 @@ level=error There is an error. Fields ``name`` and ``exc_info`` may be available to give further information. +Upgrade from 1.x version +------------------------ + +There are several backward-incompatible changes from the previous 1.x version. + +1. The command syntax changes a bit. You need to use a ``-c`` switch to specify your software version configuration file. +2. The configuration file format has been changed from ini to `toml`_. You can use the ``scripts/ini2toml`` script in this repo to convert your old configuration files. However, comments and formatting will be lost. +3. Several options have been renamed. ``max_concurrent`` to ``max_concurrency``, and all option names have their ``-`` be replaced with ``_``. +4. All software configuration sections need a ``source`` option to specify which source is to be used rather than being figured out from option names in use. This enables additional source plugins to be discovered. + Version Record Files ==================== Version record files record which version of the software you know or is available. They are simple key-value pairs of ``(name, version)`` separated by a space:: @@ -124,11 +137,11 @@ Version record files record which version of the software you know or is availab Say you've got a version record file called ``old_ver.txt`` which records all your watched software and their versions, as well as some configuration entries. To update it using ``nvchecker``:: - nvchecker source.ini + nvchecker -c source.toml See what are updated with ``nvcmp``:: - nvcmp source.ini + nvcmp -c source.toml Manually compare the two files for updates (assuming they are sorted alphabetically; files generated by ``nvchecker`` are already sorted):: @@ -146,11 +159,11 @@ This helps when you have known (and processed) some of the updated software, but This command will help most if you specify where you version record files are in your config file. See below for how to use a config file. -Version Source Files -==================== -The software version source files are in ini format. *Section names* is the name of the software. Following fields are used to tell nvchecker how to determine the current version of that software. +Configuration Files +=================== +The software version source files are in `toml`_ format. The *key name* is the name of the software. Following fields are used to tell nvchecker how to determine the current version of that software. -See ``sample_source.ini`` for an example. +See ``sample_source.toml`` for an example. Configuration Section --------------------- @@ -167,13 +180,15 @@ newver Specify a version record file to store the new version info. proxy - The HTTP proxy to use. The format is ``proto://host:port``, e.g. ``http://localhost:8087``. + The HTTP proxy to use. The format is ``proto://host:port``, e.g. ``http://localhost:8087``. Different backends have different level support for this, e.g. with ``pycurl`` you can use ``socks5h://host:port`` proxies. max_concurrency Max number of concurrent jobs. Default: 20. keyfile - Specify an ini config file containing key (token) information. This file should contain a ``keys`` section, mapping key names to key values. See specific source for the key name(s) to use. + Specify an ini config file containing key (token) information. This file + should contain a ``keys`` section, mapping key names to key values. See + specific source for the key name(s) to use. Global Options -------------- @@ -191,6 +206,22 @@ missing_ok Suppress warnings and errors if a version checking module finds nothing. Currently only ``regex`` supports it. +proxy + The HTTP proxy to use. The format is ``proto://host:port``, e.g. + ``http://localhost:8087``. Different backends have different level support + for this, e.g. with ``pycurl`` you can use ``socks5h://host:port`` proxies. + + Set it to ``""`` (empty string) to override the global setting. + + This only works when the source implementation uses the builtin HTTP client, + and doesn't work with the ``aur`` source because it's batched. However the + global proxy config applies. + +tries + Try specified times when a network error occurs. Default is ``1``. + + This only works when the source implementation uses the builtin HTTP client. + If both ``prefix`` and ``from_pattern``/``to_pattern`` are used, ``from_pattern``/``to_pattern`` are ignored. If you want to strip the prefix and then do something special, just use ``from_pattern```/``to_pattern``. For @@ -253,6 +284,8 @@ cmd Check AUR --------- Check `Arch User Repository `_ for updates. +Per-item proxy setting doesn't work for this because several items will be +batched into one request. aur The package name in AUR. If empty, use the name of software (the *section name*). @@ -280,13 +313,13 @@ path use_latest_release Set this to ``true`` to check for the latest release on GitHub. - - GitHub releases are not the same with git tags. You'll see big version names + + GitHub releases are not the same with git tags. You'll see big version names and descriptions in the release page for such releases, e.g. `zfsonlinux/zfs's `_, and those small ones like `nvchecker's `_ are only git tags that should use ``use_max_tag`` below. - + Will return the release name instead of date. use_latest_tag @@ -297,15 +330,23 @@ use_latest_tag query When ``use_latest_tag`` is ``true``, this sets a query for the tag. The exact matching method is not documented by GitHub. - + use_max_tag Set this to ``true`` to check for the max tag on GitHub. Unlike ``use_latest_release``, this option includes both annotated tags and lightweight ones, and return the largest one sorted by the ``sort_version_key`` option. Will return the tag name instead of date. -A key named ``github`` can be set to a GitHub OAuth token in order to request -more frequently than anonymously. +token + A personal authorization token used to call the API. + +An authorization token may be needed in order to use ``use_latest_tag`` or to +request more frequently than anonymously. + +To set an authorization token, you can set: + +- a key named ``github`` in the keyfile +- the token option This source supports `list options`_ when ``use_max_tag`` is set. @@ -329,15 +370,10 @@ host token Gitea authorization token used to call the API. - + To set an authorization token, you can set: -- a key named ``gitea_{host}`` in the keyfile (where ``host`` is formed the - same as the environment variable, but all lowercased). -- an environment variable ``NVCHECKER_GITEA_TOKEN_{host}`` must provide that - token. The ``host`` part is the uppercased version of the ``host`` setting, - with dots (``.``) and slashes (``/``) replaced by underscores (``_``), e.g. - ``NVCHECKER_GITEA_TOKEN_GITEA_COM``. +- a key named ``gitea_{host}`` in the keyfile, where ``host`` is all-lowercased host name - the token option This source supports `list options`_ when ``use_max_tag`` is set. @@ -383,15 +419,10 @@ host token GitLab authorization token used to call the API. - + To set an authorization token, you can set: -- a key named ``gitlab_{host}`` in the keyfile (where ``host`` is formed the - same as the environment variable, but all lowercased). -- an environment variable ``NVCHECKER_GITLAB_TOKEN_{host}`` must provide that - token. The ``host`` part is the uppercased version of the ``host`` setting, - with dots (``.``) and slashes (``/``) replaced by underscores (``_``), e.g. - ``NVCHECKER_GITLAB_TOKEN_GITLAB_COM``. +- a key named ``gitlab_{host}`` in the keyfile, where ``host`` is all-lowercased host name - the token option This source supports `list options`_ when ``use_max_tag`` is set. @@ -544,9 +575,10 @@ use_max_tag This source supports `list options`_ when ``use_max_tag`` is set. -Other ------ -More to come. Send me a patch or pull request if you can't wait and have written one yourself :-) +Extending +--------- +It's possible to extend the supported sources by writing plugins. See +``plugins.rst`` for documentation. Bugs ==== @@ -554,3 +586,4 @@ Bugs .. _Pacman: https://wiki.archlinux.org/index.php/Pacman .. _list options: #list-options +.. _toml: https://toml.io/ diff --git a/nvchecker/__main__.py b/nvchecker/__main__.py index d8fc160..73d4df1 100755 --- a/nvchecker/__main__.py +++ b/nvchecker/__main__.py @@ -14,6 +14,7 @@ import structlog from . import core from .util import VersData, RawResult, KeyManager +from .ctxvars import proxy as ctx_proxy logger = structlog.get_logger(logger_name=__name__) @@ -45,6 +46,9 @@ def main() -> None: else: keymanager = options.keymanager + if options.proxy is not None: + ctx_proxy.set(options.proxy) + token_q = core.token_queue(options.max_concurrency) result_q: asyncio.Queue[RawResult] = asyncio.Queue() try: diff --git a/nvchecker/core.py b/nvchecker/core.py index 4be2fc0..53eefcd 100644 --- a/nvchecker/core.py +++ b/nvchecker/core.py @@ -130,6 +130,7 @@ def write_verfile(file: Path, versions: VersData) -> None: class Options(NamedTuple): ver_files: Optional[Tuple[Path, Path]] max_concurrency: int + proxy: Optional[str] keymanager: KeyManager def load_file( @@ -161,13 +162,14 @@ def load_file( keyfile = d / keyfile_s keymanager = KeyManager(keyfile) - max_concurrency = c.get( - 'max_concurrency', 20) + max_concurrency = c.get('max_concurrency', 20) + proxy = c.get('proxy') else: max_concurrency = 20 + proxy = None return cast(Entries, config), Options( - ver_files, max_concurrency, keymanager) + ver_files, max_concurrency, proxy, keymanager) def token_queue(maxsize: int) -> Queue[bool]: token_q: Queue[bool] = Queue(maxsize=maxsize) diff --git a/nvchecker/httpclient/base.py b/nvchecker/httpclient/base.py index a56c12c..6b3b141 100644 --- a/nvchecker/httpclient/base.py +++ b/nvchecker/httpclient/base.py @@ -36,7 +36,7 @@ class BaseSession: for i in range(1, t+1): try: return await self.request_impl( - proxy = p, + proxy = p or None, *args, **kwargs, ) except TemporaryError as e: