tests: update

feat: add snapcraft source

.github/workflows/codeql-analysis.yml

@@ -1,71 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-name: "CodeQL"
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master]
-  schedule:
-    - cron: '0 18 * * 5'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
-        language: ['python']
-        # Learn more...
-        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file. 
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/codeql.yml

@@ -0,0 +1,84 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: '34 14 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
+        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/mypy.yaml

@@ -3,12 +3,12 @@ on: [push, pull_request]
 jobs:
   build:
     name: run mypy
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout Code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       env:
         cache-name: cache-pipmypy
       with:
@@ -18,11 +18,11 @@ jobs:
           ${{ runner.os }}-${{ env.cache-name }}-
           ${{ runner.os }}-cache-pip-
     - name: Install deps
-      run: pip3 install -U tornado pytest pytest-asyncio pytest-httpbin flaky structlog tomli aiohttp httpx mypy awesomeversion
+      run: pip3 install -U tornado pytest pytest-asyncio pytest-httpbin pytest-rerunfailures structlog tomli platformdirs aiohttp httpx mypy awesomeversion
     - name: Run mypy for --install-types
-      run: PATH=$HOME/.local/bin:$PATH mypy nvchecker nvchecker_source tests
+      run: PATH=$HOME/.local/bin:$PATH mypy --namespace-packages --explicit-package-bases nvchecker nvchecker_source tests
       continue-on-error: true
     - name: Install types
       run: PATH=$HOME/.local/bin:$PATH yes | mypy --install-types
     - name: Run mypy
-      run: PATH=$HOME/.local/bin:$PATH mypy nvchecker nvchecker_source tests
+      run: PATH=$HOME/.local/bin:$PATH mypy --namespace-packages --explicit-package-bases nvchecker nvchecker_source tests

.github/workflows/tests.yaml

@@ -7,10 +7,12 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-        - "3.7"
         - "3.8"
         - "3.9"
         - "3.10"
+        - "3.11"
+        - "3.12"
+        - "3.13"
         # pypy fails in some cases but we don't care much about that
         # with github actions we can't mark some jobs to not affect the overall
         # conclusion so we have to omit "allow-failure" tests.
@@ -18,26 +20,29 @@ jobs:
         # - pypy-3.7
         deps:
         - tornado pycurl
-        - aiohttp
+        # timer runs when loop is closed, see https://github.com/lilydjwg/nvchecker/actions/runs/11650699759/job/32439742210
+        # - aiohttp
         - tornado
         - httpx[http2]>=0.14.0
+        exclude: []
     steps:
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Setup Cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       env:
         cache-name: cache-pip
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.deps }}-${{ hashFiles('setup.py') }}
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.deps }}-${{ hashFiles('pyproject.toml', 'setup.cfg') }}
         restore-keys: |
           ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.deps }}-
           ${{ runner.os }}-${{ env.cache-name }}-
+
     - name: Install pycurl deps
       if: ${{ contains(matrix.deps, 'pycurl') }}
       run: |
@@ -45,10 +50,44 @@ jobs:
         sudo apt install -y libcurl4-openssl-dev
     # werkzeug is pinned for httpbin compatibility https://github.com/postmanlabs/httpbin/issues/673
     - name: Install Python deps
-      run: pip install -U ${{ matrix.deps }} pytest pytest-asyncio pytest-httpbin flaky structlog tomli appdirs lxml 'werkzeug<2.1' awesomeversion
+      env:
+        # use env to avoid `>` being redirection
+        deps: ${{ matrix.deps }}
+      run: pip install -U $deps pytest 'pytest-asyncio>=0.24' pytest-httpbin pytest-rerunfailures structlog tomli platformdirs lxml jq 'werkzeug<2.1' awesomeversion
+    # don't use binary distribution because:
+    #   hardcoded cacert path doesn't work on Ubuntu (should have been resolved?)
+    #   limited compression support (only deflate & gzip)
+    - name: Install pycurl
+      if: ${{ contains(matrix.deps, 'pycurl') }}
+      run: |
+        pip uninstall -y pycurl
+        pip install -U pycurl --no-binary :all:
     - name: Decrypt keys
       env:
         KEY: ${{ secrets.KEY }}
       run: if [[ -n $KEY ]]; then openssl enc -d -aes-256-ctr -pbkdf2 -k $KEY -in keyfile.toml.enc -out keyfile.toml; fi
+
+    - name: Setup mitmproxy cache
+      uses: actions/cache@v4
+      env:
+        cache-name: cache-mitm
+      with:
+        path: ~/.mitmproxy
+        key: ${{ env.cache-name }}
+        restore-keys: |
+          ${{ env.cache-name }}-
+    - name: Install mitmproxy
+      run: |
+        /usr/bin/python -m venv --system-site-packages ~/.mitmproxy/venv
+        . ~/.mitmproxy/venv/bin/activate
+        pip install -U mitmproxy
+        # https://github.com/DevToys-app/DevToys/issues/1373#issuecomment-2599820594
+        sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
+        sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+
+    # - name: Setup upterm session
+    #   uses: lhotari/action-upterm@v1
     - name: Run pytest
-      run: if [[ -f keyfile.toml ]]; then KEYFILE=keyfile.toml pytest; else pytest; fi
+      env:
+        mitmdump: /home/runner/.mitmproxy/venv/bin/mitmdump
+      run: scripts/run_cached_tests

.gitignore

@@ -1,10 +1,12 @@
 *.egg-info/
 __pycache__/
 /build/
+/dist/
 .cache/
 .eggs/
 *.pyc
 *.pyo
 .travis.pub
 .pytest_cache/
+.tox/
 keyfile.toml

.readthedocs.yaml

@@ -0,0 +1,10 @@
+version: 2
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+sphinx:
+  configuration: docs/conf.py
+python:
+  install:
+  - requirements: docs/requirements.txt

README.rst

@@ -23,8 +23,8 @@ This is the version 2.0 branch. For the old version 1.x, please switch to the ``
 
 Dependency
 ----------
-- Python 3.7+
-- Python library: structlog, tomli, appdirs
+- Python 3.8+
+- Python library: structlog, platformdirs, tomli (on Python < 3.11)
 - One of these Python library combinations (ordered by preference):
 
   * tornado + pycurl
@@ -42,7 +42,7 @@ To install::
 
 To use the latest code, you can also clone this repository and run::
 
-  python3 setup.py install
+  pip install .
 
 To see available options::
 

docs/conf.py

@@ -30,8 +30,9 @@ autodoc_inherit_docstrings = False
 # https://bitbucket.org/birkenfeld/sphinx/issue/1337/autoclass_content-both-uses-object__init__
 autodoc_docstring_signature = False
 
-intersphinx_mapping = {"python": ("https://docs.python.org/3.8/", None)}
+intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
 
+html_theme = "sphinx_rtd_theme"
 on_rtd = os.environ.get("READTHEDOCS", None) == "True"
 
 # On RTD we can't import sphinx_rtd_theme, but it will be applied by
@@ -39,8 +40,6 @@ on_rtd = os.environ.get("READTHEDOCS", None) == "True"
 # as on RTD.
 if not on_rtd:
     import sphinx_rtd_theme
-
-    html_theme = "sphinx_rtd_theme"
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 html_theme_options = {

docs/requirements.txt

@@ -1,6 +1,5 @@
-tomli
 structlog
-appdirs
+platformdirs
 tornado>=6
 sphinx>=3.2
 # <5 has strange bottom margins for p, and no list indicators

docs/usage.rst

@@ -17,8 +17,8 @@ This is the version 2.0 branch. For the old version 1.x, please switch to the ``
 
 Dependency
 ----------
-- Python 3.7+
-- Python library: structlog, tomli, appdirs
+- Python 3.8+
+- Python library: structlog, platformdirs, tomli (on Python < 3.11)
 - One of these Python library combinations (ordered by preference):
 
   * tornado + pycurl
@@ -71,8 +71,8 @@ The JSON log is one JSON string per line. The following documented events and
 fields are stable, undocumented ones may change without notice.
 
 event=updated
-  An update is detected. Fields ``name``, ``old_version`` and ``version`` are
-  available. ``old_version`` maybe ``null``.
+  An update is detected. Fields ``name``, ``revision``, ``old_version`` and ``version`` are
+  available. ``old_version`` may be ``null`` and ``revision`` may be absent.
 
 event=up-to-date
   There is no update. Fields ``name`` and ``version`` are available.
@@ -170,6 +170,9 @@ prefix
   Strip the prefix string if the version string starts with it. Otherwise the
   version string is returned as-is.
 
+  If both ``prefix`` and ``from_pattern``/``to_pattern`` are used, ``prefix``
+  is applied first.
+
 from_pattern, to_pattern
   Both are Python-compatible regular expressions. If ``from_pattern`` is found
   in the version string, it will be replaced with ``to_pattern``.
@@ -179,7 +182,7 @@ from_pattern, to_pattern
 
 missing_ok
   Suppress warnings and errors if a version checking module finds nothing.
-  Currently only ``regex`` supports it.
+  Not all sources support it.
 
 proxy
   The HTTP proxy to use. The format is ``proto://host:port``, e.g.
@@ -204,21 +207,14 @@ httptoken
   A personal authorization token used to fetch the url with the ``Authorization`` header.
   The type of token depends on the authorization required.
 
-  - For Bearer token set : ``Bearer <Your_bearer_token>``
-  - For Basic token set : ``Basic <Your_base64_encoded_token>``
+  - For Bearer token set \: ``Bearer <Your_bearer_token>``
+  - For Basic token set \: ``Basic <Your_base64_encoded_token>``
 
   In the keyfile add ``httptoken_{name}`` token.
 
 verify_cert
   Whether to verify the HTTPS certificate or not. Default is ``true``.
 
-If both ``prefix`` and ``from_pattern``/``to_pattern`` are used,
-``from_pattern``/``to_pattern`` are ignored. If you want to strip the prefix
-and then do something special, just use ``from_pattern``/``to_pattern``. For
-example, the transformation of ``v1_1_0`` => ``1.1.0`` can be achieved with
-``from_pattern = 'v(\d+)_(\d+)_(\d+)'`` and ``to_pattern = '\1.\2.\3'``.
-(Note that in TOML it's easiler to write regexes in single quotes so you don't need to escape ``\``.)
-
 .. _list options:
 
 List Options
@@ -326,8 +322,45 @@ post_data
 post_data_type
   (*Optional*) Specifies the ``Content-Type`` of the request body (``post_data``). By default, this is ``application/x-www-form-urlencoded``.
 
+This source can also work with XML to some extent, e.g. it can parse an RSS feed like this:
+
+.. code-block:: toml
+
+   [ProxmoxVE]
+   source = "htmlparser"
+   url = "https://my.proxmox.com/en/announcements/tag/proxmox-ve/rss"
+   xpath = "//item/title"
+   from_pattern = 'Proxmox VE ([\d.]+) released!'
+   to_pattern = '\1'
+
 .. note::
-  An additional dependency "lxml" is required.
+   An additional dependency "lxml" is required.
+   You can use ``pip install 'nvchecker[htmlparser]'``.
+
+Search with an JSON Parser (jq)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "jq"
+
+Send an HTTP request and search through the body with a specific ``jq`` filter.
+
+url
+  The URL of the HTTP request.
+
+filter
+  An ``jq`` filter used to find the version string.
+
+post_data
+  (*Optional*) When present, a ``POST`` request (instead of a ``GET``) will be used. The value should be a string containing the full body of the request. The encoding of the string can be specified using the ``post_data_type`` option.
+
+post_data_type
+  (*Optional*) Specifies the ``Content-Type`` of the request body (``post_data``). By default, this is ``application/json``.
+
+This source supports :ref:`list options`.
+
+.. note::
+   An additional dependency "jq" is required.
 
 Find with a Command
 ~~~~~~~~~~~~~~~~~~~
@@ -366,18 +399,21 @@ Check GitHub
   source = "github"
 
 Check `GitHub <https://github.com/>`_ for updates. The version returned is in
-date format ``%Y%m%d.%H%M%S``, e.g. ``20130701.012212``, unless ``use_latest_release``
-or ``use_max_tag`` is used. See below.
+date format ``%Y%m%d.%H%M%S``, e.g. ``20130701.012212``, unless ``use_latest_release``,
+``use_max_tag`` or ``use_max_release`` is used. See below.
 
 github
   The github repository, with author, e.g. ``lilydjwg/nvchecker``.
 
 branch
-  Which branch to track? Default: ``master``.
+  Which branch to track? Default: the repository's default.
 
 path
   Only commits containing this file path will be returned.
 
+host
+  Hostname for self-hosted GitHub instance.
+
 use_latest_release
   Set this to ``true`` to check for the latest release on GitHub.
 
@@ -387,7 +423,26 @@ use_latest_release
   small ones like `nvchecker's <https://github.com/lilydjwg/nvchecker/releases>`_
   are only git tags that should use ``use_max_tag`` below.
 
-  Will return the release name instead of date.
+  Will return the release's tag name instead of date. (For historical reasons
+  it doesn't return the release name. See below to change.)
+
+use_max_release
+  Set this to ``true`` to check for the max release on GitHub.
+  This option returns the largest one sorted by the
+  ``sort_version_key`` option. Will return the tag name instead of date.
+
+use_release_name
+  When ``use_latest_release`` or ``use_max_release`` is ``true``,
+  setting this to ``true`` will cause nvchecker to return the release name
+  instead of the tag name.
+
+include_prereleases
+  When ``use_latest_release`` or ``use_max_release`` is ``true``,
+  set this to ``true`` to take prereleases into account.
+
+  This returns the release names (not the tag names).
+
+  This requires a token because it's using the v4 GraphQL API.
 
 use_latest_tag
   Set this to ``true`` to check for the latest tag on GitHub.
@@ -400,22 +455,24 @@ query
 
 use_max_tag
   Set this to ``true`` to check for the max tag on GitHub. Unlike
-  ``use_latest_release``, this option includes both annotated tags and
+  ``use_max_release``, this option includes both annotated tags and
   lightweight ones, and return the largest one sorted by the
   ``sort_version_key`` option. Will return the tag name instead of date.
 
 token
   A personal authorization token used to call the API.
 
-An authorization token may be needed in order to use ``use_latest_tag`` or to
-request more frequently than anonymously.
+An authorization token may be needed in order to use ``use_latest_tag``,
+``include_prereleases`` or to request more frequently than anonymously.
 
 To set an authorization token, you can set:
 
-- a key named ``github`` in the keyfile
 - the token option
+- an entry in the keyfile for the host (e.g. ``github.com``)
+- an entry in your ``netrc`` file for the host
 
-This source supports :ref:`list options` when ``use_max_tag`` is set.
+This source supports :ref:`list options` when ``use_max_tag`` or
+``use_max_release`` is set.
 
 Check Gitea
 ~~~~~~~~~~~
@@ -430,7 +487,7 @@ gitea
   The gitea repository, with author, e.g. ``gitea/tea``.
 
 branch
-  Which branch to track? Default: ``master``.
+  Which branch to track? Default: the repository's default.
 
 use_max_tag
   Set this to ``true`` to check for the max tag on Gitea. Will return the biggest one
@@ -444,11 +501,21 @@ token
 
 To set an authorization token, you can set:
 
-- a key named ``gitea_{host}`` in the keyfile, where ``host`` is all-lowercased host name
 - the token option
+- an entry in the keyfile for the host (e.g. ``gitea.com``)
+- an entry in your ``netrc`` file for the host
 
 This source supports :ref:`list options` when ``use_max_tag`` is set.
 
+Check Gogs / Forgejo / Codeberg
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Please use the above "gitea" source. Gitea is a fork of `Gogs <https://gogs.io/>`_.
+`Forgejo <https://forgejo.org/>`_ is a fork of Gitea. Codeberg is a code
+hosting provider that uses Forgejo. They share the same API endpoints nvchecker uses.
+
+Alternatively, you can try the generic "git" source.
+
 Check BitBucket
 ~~~~~~~~~~~~~~~
 ::
@@ -462,7 +529,7 @@ bitbucket
   The bitbucket repository, with author, e.g. ``lilydjwg/dotvim``.
 
 branch
-  Which branch to track? Default is the repository's default.
+  Which branch to track? Default: the repository's default.
 
 use_max_tag
   Set this to ``true`` to check for the max tag on BitBucket. Will return the biggest one
@@ -517,8 +584,9 @@ token
 
 To set an authorization token, you can set:
 
-- a key named ``gitlab_{host}`` in the keyfile, where ``host`` is all-lowercased host name
 - the token option
+- an entry in the keyfile for the host (e.g. ``gitlab.com``)
+- an entry in your ``netrc`` file for the host
 
 This source supports :ref:`list options` when ``use_max_tag`` is set.
 
@@ -528,7 +596,7 @@ Check PyPI
 
   source = "pypi"
 
-Check `PyPI <https://pypi.python.org/>`_ for updates.
+Check `PyPI <https://pypi.python.org/>`_ for updates. Yanked releases are ignored.
 
 pypi
   The name used on PyPI, e.g. ``PySide``.
@@ -536,6 +604,12 @@ pypi
 use_pre_release
   Whether to accept pre release. Default is false.
 
+This source supports :ref:`list options`.
+
+.. note::
+   An additional dependency "packaging" is required.
+   You can use ``pip install 'nvchecker[pypi]'``.
+
 Check RubyGems
 ~~~~~~~~~~~~~~
 ::
@@ -616,11 +690,16 @@ Check crates.io
 
   source = "cratesio"
 
-Check `crates.io <https://crates.io/>`_ for updates.
+Check `crates.io <https://crates.io/>`_ for updates. Yanked releases are ignored.
 
 cratesio
   The crate name on crates.io, e.g. ``tokio``.
 
+use_pre_release
+  Whether to accept pre release. Default is false.
+
+This source supports :ref:`list options`.
+
 Check Local Pacman Database
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ::
@@ -717,6 +796,11 @@ This enables you to track updates from `Anitya <https://release-monitoring.org/>
 anitya
   ``distro/package``, where ``distro`` can be a lot of things like "fedora", "arch linux", "gentoo", etc. ``package`` is the package name of the chosen distribution.
 
+anitya_id
+  The identifier of the project/package in anitya.
+
+Note that either anitya or anitya_id needs to be specified, anitya_id is preferred when both specified.
+
 Check Android SDK
 ~~~~~~~~~~~~~~~~~
 ::
@@ -750,6 +834,9 @@ This enables you to track updates of macOS applications which using `Sparkle fra
 sparkle
   The url of the sparkle appcast.
 
+release_notes_language
+  The language of release notes to return when localized release notes are available (defaults to ``en`` for English, the unlocalized release notes are used as a fallback)
+
 Check Pagure
 ~~~~~~~~~~~~
 ::
@@ -797,6 +884,29 @@ strip_release
 
 Note that either pkg or srcpkg needs to be specified (but not both) or the item name will be used as pkg.
 
+Check RPM repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "rpmrepo"
+
+This enables you to check latest package versions in an arbitrary RPM repository in `repomd` format used by package managers such as ``dnf`` (Fedora, RHEL, AlmaLinux etc.) or ``zypper`` (openSUSE) without the need for native RPM tools.
+
+pkg
+  Name of the RPM package (you can also use ``rpmrepo`` as with other sources, but ``pkg`` is preferred for clarity)
+
+repo
+  URL of the repository (required, ``repodata/repomd.xml`` should be there)
+
+arch
+  Architecture of the RPM package (``binary``, ``src``, ``any``, ``x86_64``, ``aarch64``, etc, defaults to ``binary``)
+
+This source supports :ref:`list options`.
+
+.. note::
+   An additional dependency "lxml" is required.
+   You can use ``pip install 'nvchecker[rpmrepo]'``.
+
 Check Git repository
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ::
@@ -816,6 +926,19 @@ branch
 
 When this source returns tags (``use_commit`` is not true) it supports :ref:`list options`.
 
+Check Mercurial repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "mercurial"
+
+This enables you to check tags of an arbitrary mercurial (hg) repository.
+
+mercurial
+  URL of the Mercurial repository.
+
+This source returns tags and supports :ref:`list options`.
+
 Check container registry
 ~~~~~~~~~~~~~~~~~~~~~~~~
 ::
@@ -825,7 +948,9 @@ Check container registry
 This enables you to check tags of images on a container registry like Docker.
 
 container
-  The path for the container image. For official Docker images, use namespace ``library/`` (e.g. ``library/python``).
+  The path (and tag) for the container image. For official Docker images, use namespace ``library/`` (e.g. ``library/python``).
+
+  If no tag is given, it checks latest available tag (sort by tag name), otherwise, it checks the tag's update time.
 
 registry
   The container registry host. Default: ``docker.io``
@@ -836,17 +961,23 @@ container name while this plugin requires the full name. If the host part is
 omitted, use ``docker.io``, and if there is no slash in the path, prepend
 ``library/`` to the path. Here are some examples:
 
-+----------------------------------------------+-----------+--------------------------+
-| Pull command                                 | registry  | container                |
-+==============================================+===========+==========================+
-| docker pull quay.io/prometheus/node-exporter | quay.io   | prometheus/node-exporter |
-+----------------------------------------------+-----------+--------------------------+
-| docker pull nvidia/cuda                      | docker.io | nvidia/cuda              |
-+----------------------------------------------+-----------+--------------------------+
-| docker pull python                           | docker.io | library/python           |
-+----------------------------------------------+-----------+--------------------------+
++-----------------------------------------------------+-----------+---------------------------------+
+| Pull        command                                 | registry  | container                       |
++=====================================================+===========+=================================+
+| docker pull quay.io/prometheus/node-exporter        | quay.io   | prometheus/node-exporter        |
++-----------------------------------------------------+-----------+---------------------------------+
+| docker pull quay.io/prometheus/node-exporter:master | quay.io   | prometheus/node-exporter:master |
++-----------------------------------------------------+-----------+---------------------------------+
+| docker pull openeuler/openeuler                     | docker.io | openeuler/openeuler             |
++-----------------------------------------------------+-----------+---------------------------------+
+| docker pull openeuler/openeuler:20.03-lts           | docker.io | openeuler/openeuler:20.03-lts   |
++-----------------------------------------------------+-----------+---------------------------------+
+| docker pull python                                  | docker.io | library/python                  |
++-----------------------------------------------------+-----------+---------------------------------+
+| docker pull python:3.11                             | docker.io | library/python:3.11             |
++-----------------------------------------------------+-----------+---------------------------------+
 
-This source returns tags and supports :ref:`list options`.
+If no tag is given, this source returns tags and supports :ref:`list options`.
 
 Check ALPM database
 ~~~~~~~~~~~~~~~~~~~
@@ -863,7 +994,7 @@ repo
   Name of the package repository in which the package resides. If not provided, nvchecker will use ``repos`` value, see below.
 
 repos
-  An array of possible repositories in which the package may reside in, nvchecker will use the first repository which contains the package. If not provided, ``core``, ``extra``, ``community`` and ``multilib`` will be used, in that order.
+  An array of possible repositories in which the package may reside in, nvchecker will use the first repository which contains the package. If not provided, ``core``, ``extra`` and ``multilib`` will be used, in that order.
 
 dbpath
   Path to the ALPM database directory. Default: ``/var/lib/pacman``. You need to update the database yourself.
@@ -874,6 +1005,32 @@ strip_release
 provided
   Instead of the package version, return the version this package provides. Its value is what the package provides, and ``strip_release`` takes effect too. This is best used with libraries.
 
+.. note::
+   An additional dependency "pyalpm" is required.
+
+Check ALPM files database
+~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "alpmfiles"
+
+Search package files in a local ALPM files database. The package does not need to be installed. This can be useful for checking shared library versions if a package does not list them in its ``provides``.
+
+pkgname
+  Name of the package.
+
+filename
+  Regular expression for the file path. If it contains one matching group, that group is returned. Otherwise return the whole file path. Paths do not have an initial slash. For example, ``usr/lib/libuv\\.so\\.([^.]+)`` matches the major shared library version of libuv.
+
+repo
+  Name of the package repository in which the package resides. If not provided, search all repositories.
+
+strip_dir
+  Strip directory from the path before matching. Defaults to ``false``.
+
+dbpath
+  Path to the ALPM database directory. Default: ``/var/lib/pacman``. You need to update the database yourself with ``pacman -Fy``.
+
 Check Open Vsx
 ~~~~~~~~~~~~~~~
 ::
@@ -896,6 +1053,47 @@ Check `Visual Studio Code Marketplace <https://marketplace.visualstudio.com/vsco
 vsmarketplace
   The extension's Unique Identifier on marketplace.visualstudio.com/vscode, e.g. ``ritwickdey.LiveServer``.
 
+Check Go packages and modules
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "go"
+
+Check `Go packages and modules <https://pkg.go.dev/>`_ for updates.
+
+go
+  The name of Go package or module, e.g. ``github.com/caddyserver/caddy/v2/cmd``.
+
+Check opam repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "opam"
+
+This enables you to check latest package versions in an arbitrary `opam repository <https://opam.ocaml.org/doc/Manual.html#Repositories>` without the need for the opam command line tool.
+
+pkg
+  Name of the opam package
+
+repo
+  URL of the repository (optional, the default ``https://opam.ocaml.org`` repository is used if not specified)
+
+This source supports :ref:`list options`.
+
+Check Snapcraft
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "snapcraft"
+
+This source allows you to check the latest package versions in the `Snapcraft <https://snapcraft.io>`_.
+
+snap
+  Name of the snap package.
+
+channel
+  Name of the channel.
+
 Combine others' results
 ~~~~~~~~~~~~~~~~~~~~~~~
 ::

mypy.ini

@@ -18,8 +18,14 @@ ignore_missing_imports = True
 [mypy-pytest_httpbin]
 ignore_missing_imports = True
 
-[mypy-appdirs]
-ignore_missing_imports = True
-
 [mypy-lxml]
 ignore_missing_imports = True
+
+[mypy-tomllib]
+ignore_missing_imports = True
+
+[mypy-jq]
+ignore_missing_imports = True
+
+[mypy-tomli]
+ignore_missing_imports = True

nvchecker/__init__.py

@@ -1,4 +1,4 @@
 # MIT licensed
-# Copyright (c) 2013-2022 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
-__version__ = '2.10'
+__version__ = '2.17dev'

nvchecker/__main__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # MIT licensed
-# Copyright (c) 2013-2022 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 from __future__ import annotations
 
@@ -13,7 +13,7 @@ from pathlib import Path
 import structlog
 
 from . import core
-from .util import VersData, RawResult, KeyManager, EntryWaiter
+from .util import ResultData, RawResult, KeyManager, EntryWaiter
 from .ctxvars import proxy as ctx_proxy
 
 logger = structlog.get_logger(logger_name=__name__)
@@ -75,26 +75,33 @@ def main() -> None:
     oldvers = core.read_verfile(options.ver_files[0])
   else:
     oldvers = {}
-  result_coro = core.process_result(oldvers, result_q, entry_waiter)
+  result_coro = core.process_result(oldvers, result_q, entry_waiter, verbose=bool(args.entry))
   runner_coro = core.run_tasks(futures)
 
   if sys.version_info >= (3, 10):
     # Python 3.10 has deprecated asyncio.get_event_loop
-    newvers, has_failures = asyncio.run(run(result_coro, runner_coro))
+    results, has_failures = asyncio.run(run(result_coro, runner_coro))
   else:
     # Python < 3.10 will create an eventloop when asyncio.Queue is initialized
-    newvers, has_failures = asyncio.get_event_loop().run_until_complete(run(result_coro, runner_coro))
+    results, has_failures = asyncio.get_event_loop().run_until_complete(run(result_coro, runner_coro))
 
   if options.ver_files is not None:
-    core.write_verfile(options.ver_files[1], newvers)
+    newverf = options.ver_files[1]
+    if args.entry:
+      # don't remove other entries when only one entry is specified on cmdline
+      vers = core.read_verfile(newverf)
+    else:
+      vers = {}
+    vers.update(results)
+    core.write_verfile(newverf, vers)
 
   if args.failures and has_failures:
     sys.exit(3)
 
 async def run(
-  result_coro: Coroutine[None, None, Tuple[VersData, bool]],
+  result_coro: Coroutine[None, None, Tuple[ResultData, bool]],
   runner_coro: Coroutine[None, None, None],
-) -> Tuple[VersData, bool]:
+) -> Tuple[ResultData, bool]:
   result_fu = asyncio.create_task(result_coro)
   runner_fu = asyncio.create_task(runner_coro)
   await runner_fu

nvchecker/api.py

@@ -3,7 +3,7 @@
 
 from .httpclient import session, TemporaryError, HTTPError
 from .util import (
-  Entry, BaseWorker, RawResult, VersionResult,
+  Entry, BaseWorker, RawResult, VersionResult, RichResult,
   AsyncCache, KeyManager, GetVersionError, EntryWaiter,
 )
 from .sortversion import sort_version_keys

nvchecker/core.py

@@ -1,5 +1,5 @@
 # MIT licensed
-# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020, 2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ import argparse
 from typing import (
   Tuple, NamedTuple, Optional, List, Union,
   cast, Dict, Awaitable, Sequence, Any,
+  TYPE_CHECKING,
 )
 import types
 from pathlib import Path
@@ -19,15 +20,24 @@ from importlib import import_module
 import re
 import contextvars
 import json
+import dataclasses
 
 import structlog
-import tomli
-import appdirs
+
+if TYPE_CHECKING:
+  import tomli as tomllib
+else:
+  try:
+    import tomllib
+  except ModuleNotFoundError:
+    import tomli as tomllib
+
+import platformdirs
 
 from .lib import nicelogger
 from . import slogconf
 from .util import (
-  Entry, Entries, KeyManager, RawResult, Result, VersData,
+  Entry, Entries, KeyManager, RawResult, RichResult, ResultData,
   FunctionWorker, GetVersionError,
   FileLoadError, EntryWaiter,
 )
@@ -40,7 +50,7 @@ from . import httpclient
 logger = structlog.get_logger(logger_name=__name__)
 
 def get_default_config() -> str:
-  confdir = appdirs.user_config_dir(appname='nvchecker')
+  confdir = platformdirs.user_config_dir(appname='nvchecker')
   file = os.path.join(confdir, 'nvchecker.toml')
   return file
 
@@ -67,6 +77,8 @@ def process_common_arguments(args: argparse.Namespace) -> bool:
   processors = [
     slogconf.exc_info,
     slogconf.filter_exc,
+    slogconf.filter_nones,
+    slogconf.filter_taskname,
   ]
   logger_factory = None
 
@@ -101,11 +113,12 @@ def process_common_arguments(args: argparse.Namespace) -> bool:
     return True
   return False
 
-def safe_overwrite(fname: str, data: Union[bytes, str], *,
+def safe_overwrite(file: Path, data: Union[bytes, str], *,
                    method: str = 'write', mode: str = 'w', encoding: Optional[str] = None) -> None:
   # FIXME: directory has no read perm
-  # FIXME: symlinks and hard links
-  tmpname = fname + '.tmp'
+  # FIXME: hard links
+  resolved_path = file.resolve()
+  tmpname = str(resolved_path) + '.tmp'
   # if not using "with", write can fail without exception
   with open(tmpname, mode, encoding=encoding) as f:
     getattr(f, method)(data)
@@ -113,9 +126,9 @@ def safe_overwrite(fname: str, data: Union[bytes, str], *,
     f.flush()
     os.fsync(f.fileno())
   # if the above write failed (because disk is full etc), the old data should be kept
-  os.rename(tmpname, fname)
+  os.rename(tmpname, resolved_path)
 
-def read_verfile(file: Path) -> VersData:
+def read_verfile(file: Path) -> ResultData:
   try:
     with open(file) as f:
       data = f.read()
@@ -131,16 +144,34 @@ def read_verfile(file: Path) -> VersData:
       name, ver = l.rstrip().split(None, 1)
       v[name] = ver
 
+  if v.get('version') is None:
+    v = {k: RichResult(version=a) for k, a in v.items()}
+  elif v['version'] == 2:
+    v = {k: RichResult(**a) for k, a in v['data'].items()}
+  else:
+    raise Exception('unknown verfile version', v['version'])
+
   return v
 
-def write_verfile(file: Path, versions: VersData) -> None:
-  # sort and indent to make it friendly to human and git
+def write_verfile(file: Path, versions: ResultData) -> None:
+  d = {
+    'version': 2,
+    # sort and indent to make it friendly to human and git
+    'data': dict(sorted(versions.items())),
+  }
   data = json.dumps(
-    dict(sorted(versions.items())),
-    indent=2,
-    ensure_ascii=False,
+    d,
+    indent = 2,
+    ensure_ascii = False,
+    default = json_encode,
   ) + '\n'
-  safe_overwrite(str(file), data)
+  safe_overwrite(file, data)
+
+def json_encode(obj):
+  if isinstance(obj, RichResult):
+    d = {k: v for k, v in dataclasses.asdict(obj).items() if v is not None}
+    return d
+  raise TypeError(obj)
 
 class Options(NamedTuple):
   ver_files: Optional[Tuple[Path, Path]]
@@ -157,8 +188,8 @@ def load_file(
 ) -> Tuple[Entries, Options]:
   try:
     with open(file, 'rb') as f:
-      config = tomli.load(f)
-  except (OSError, tomli.TOMLDecodeError) as e:
+      config = tomllib.load(f)
+  except (OSError, tomllib.TOMLDecodeError) as e:
     raise FileLoadError('version configuration file', file, e)
 
   ver_files: Optional[Tuple[Path, Path]] = None
@@ -268,13 +299,12 @@ def substitute_version(
 ) -> str:
   '''
   Substitute the version string via defined rules in the configuration file.
-  See README.rst#global-options for details.
+  See usage.rst#global-options for details.
   '''
   prefix = conf.get('prefix')
   if prefix:
     if version.startswith(prefix):
       version = version[len(prefix):]
-    return version
 
   from_pattern = conf.get('from_pattern')
   if from_pattern:
@@ -282,44 +312,54 @@ def substitute_version(
     if to_pattern is None:
       raise ValueError("from_pattern exists but to_pattern doesn't")
 
-    return re.sub(from_pattern, to_pattern, version)
+    version = re.sub(from_pattern, to_pattern, version)
 
-  # No substitution rules found. Just return the original version string.
   return version
 
 def apply_list_options(
-  versions: List[str], conf: Entry,
-) -> Optional[str]:
+  versions: List[Union[str, RichResult]],
+  conf: Entry,
+  name: str,
+) -> Optional[Union[str, RichResult]]:
   pattern = conf.get('include_regex')
-  if pattern:
+  if versions and pattern:
     re_pat = re.compile(pattern)
-    versions = [x for x in versions
-                if re_pat.fullmatch(x)]
+    versions2 = [x for x in versions
+                if re_pat.fullmatch(str(x))]
+    if not versions2:
+      logger.warning('include_regex matched no versions',
+                     name=name, versions=versions, regex=pattern)
+      return None
+    versions = versions2
 
   pattern = conf.get('exclude_regex')
   if pattern:
     re_pat = re.compile(pattern)
     versions = [x for x in versions
-                if not re_pat.fullmatch(x)]
+                if not re_pat.fullmatch(str(x))]
 
   ignored = set(conf.get('ignored', '').split())
   if ignored:
-    versions = [x for x in versions if x not in ignored]
+    versions = [x for x in versions
+                if str(x) not in ignored]
 
   if not versions:
     return None
 
   sort_version_key = sort_version_keys[
     conf.get("sort_version_key", "parse_version")]
-  versions.sort(key=sort_version_key) # type: ignore
+  versions.sort(key=lambda version: sort_version_key(str(version))) # type: ignore
 
   return versions[-1]
 
-def _process_result(r: RawResult) -> Union[Result, Exception]:
+def _process_result(r: RawResult) -> Union[RichResult, Exception]:
   version = r.version
   conf = r.conf
   name = r.name
 
+  url = None
+  revision = None
+  gitref = None
   if isinstance(version, GetVersionError):
     kw = version.kwargs
     kw['name'] = name
@@ -330,7 +370,17 @@ def _process_result(r: RawResult) -> Union[Result, Exception]:
                   name=r.name, exc_info=r.version)
     return version
   elif isinstance(version, list):
-    version_str = apply_list_options(version, conf)
+    version_str = apply_list_options(version, conf, name)
+    if isinstance(version_str, RichResult):
+      url = version_str.url
+      gitref = version_str.gitref
+      revision = version_str.revision
+      version_str = version_str.version
+  elif isinstance(version, RichResult):
+    version_str = version.version
+    url = version.url
+    gitref = version.gitref
+    revision = version.revision
   else:
     version_str = version
 
@@ -339,7 +389,12 @@ def _process_result(r: RawResult) -> Union[Result, Exception]:
 
     try:
       version_str = substitute_version(version_str, conf)
-      return Result(name, version_str, conf)
+      return RichResult(
+        version = version_str,
+        url = url,
+        gitref = gitref,
+        revision = revision,
+      )
     except (ValueError, re.error) as e:
       logger.exception('error occurred in version substitutions', name=name)
       return e
@@ -348,19 +403,35 @@ def _process_result(r: RawResult) -> Union[Result, Exception]:
     return ValueError('no version returned')
 
 def check_version_update(
-  oldvers: VersData, name: str, version: str,
+  oldvers: ResultData,
+  name: str,
+  r: RichResult,
+  verbose: bool,
 ) -> None:
-  oldver = oldvers.get(name, None)
-  if not oldver or oldver != version:
-    logger.info('updated', name=name, version=version, old_version=oldver)
+  if old_result := oldvers.get(name):
+    oldver = old_result.version
   else:
-    logger.debug('up-to-date', name=name, version=version)
+    oldver = None
+  if not oldver or oldver != r.version:
+    logger.info(
+      'updated',
+      name = name,
+      version = r.version,
+      revision = r.revision,
+      old_version = oldver,
+      url = r.url,
+    )
+  else:
+    # provide visible user feedback if it was the only entry
+    level = logging.INFO if verbose else logging.DEBUG
+    logger.log(level, 'up-to-date', name=name, version=r.version, url=r.url)
 
 async def process_result(
-  oldvers: VersData,
+  oldvers: ResultData,
   result_q: Queue[RawResult],
   entry_waiter: EntryWaiter,
-) -> Tuple[VersData, bool]:
+  verbose: bool = False,
+) -> Tuple[ResultData, bool]:
   ret = {}
   has_failures = False
   try:
@@ -373,11 +444,13 @@ async def process_result(
         r1 = e
       if isinstance(r1, Exception):
         entry_waiter.set_exception(r.name, r1)
+        # no versions are returned from "apply_list_options"?
+        logger.error('no-result', name=r.name, error=repr(r1))
         has_failures = True
         continue
-      check_version_update(oldvers, r1.name, r1.version)
-      entry_waiter.set_result(r1.name, r1.version)
-      ret[r1.name] = r1.version
+      check_version_update(oldvers, r.name, r1, verbose)
+      entry_waiter.set_result(r.name, r1.version)
+      ret[r.name] = r1
   except asyncio.CancelledError:
     return ret, has_failures
 

nvchecker/httpclient/aiohttp_httpclient.py

@@ -12,7 +12,6 @@ from .base import BaseSession, TemporaryError, Response, HTTPError
 __all__ = ['session']
 
 logger = structlog.get_logger(logger_name=__name__)
-connector = aiohttp.TCPConnector(limit=20)
 
 class AiohttpSession(BaseSession):
   session = None

nvchecker/httpclient/base.py

@@ -121,4 +121,4 @@ class TemporaryError(BaseHTTPError):
   '''A temporary error (e.g. network error) happens.'''
 
 class HTTPError(BaseHTTPError):
-  ''' An HTTP 4xx error happens '''
+  '''An HTTP 4xx error happens'''

nvchecker/httpclient/httpx_httpclient.py

@@ -1,5 +1,5 @@
 # MIT licensed
-# Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2020-2022,2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 from typing import Dict, Optional, Tuple
 
@@ -34,7 +34,7 @@ class HttpxSession(BaseSession):
       client = httpx.AsyncClient(
         timeout = httpx.Timeout(self.timeout, pool=None),
         http2 = True,
-        proxies = {'all://': proxy},
+        proxy = proxy,
         verify = verify_cert,
       )
       self.clients[(proxy, verify_cert)] = client
@@ -49,7 +49,8 @@ class HttpxSession(BaseSession):
         method, url, json = json, content = body,
         headers = headers,
         follow_redirects = follow_redirects,
-        params = params,
+        # httpx checks for None but not ()
+        params = params or None,
       )
       err_cls: Optional[type] = None
       if r.status_code >= 500:

nvchecker/httpclient/tornado_httpclient.py

@@ -4,6 +4,7 @@
 import json as _json
 from urllib.parse import urlencode
 from typing import Optional, Dict, Any
+import os
 
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 
@@ -17,8 +18,9 @@ from .base import BaseSession, TemporaryError, Response, HTTPError
 __all__ = ['session']
 
 HTTP2_AVAILABLE = None if pycurl else False
+SSL_CERT_FILE = os.environ.get('SSL_CERT_FILE')
 
-def try_use_http2(curl):
+def setup_curl(curl):
   global HTTP2_AVAILABLE
   if HTTP2_AVAILABLE is None:
     try:
@@ -29,6 +31,10 @@ def try_use_http2(curl):
   elif HTTP2_AVAILABLE:
     curl.setopt(pycurl.HTTP_VERSION, 4)
 
+  if SSL_CERT_FILE:
+    curl.setopt_string(pycurl.CAINFO, SSL_CERT_FILE)
+  curl.setopt_string(pycurl.ACCEPT_ENCODING, "")
+
 class TornadoSession(BaseSession):
   def setup(
     self,
@@ -68,7 +74,7 @@ class TornadoSession(BaseSession):
       kwargs['body'] = body
     elif json:
       kwargs['body'] = _json.dumps(json)
-    kwargs['prepare_curl_callback'] = try_use_http2
+    kwargs['prepare_curl_callback'] = setup_curl
 
     if proxy:
       host, port = proxy.rsplit(':', 1)

nvchecker/lib/nicelogger.py

@@ -71,6 +71,7 @@ class TornadoLogFormatter(logging.Formatter):
         'filename', 'exc_info', 'exc_text', 'created', 'funcName',
         'processName', 'process', 'msecs', 'relativeCreated', 'thread',
         'threadName', 'name', 'levelno', 'msg', 'pathname', 'stack_info',
+        'taskName',
       })
 
     if record.exc_info:

nvchecker/slogconf.py

@@ -1,6 +1,6 @@
 # vim: se sw=2:
 # MIT licensed
-# Copyright (c) 2018 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2018-2020,2023-2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import logging
 import os
@@ -26,6 +26,9 @@ def _console_msg(event):
   else:
     msg = evt
 
+  if 'revision' in event and not event['revision']:
+      del event['revision']
+
   if 'name' in event:
     msg = f"{event['name']}: {msg}"
     del event['name']
@@ -39,6 +42,17 @@ def exc_info(logger, level, event):
     event['exc_info'] = True
   return event
 
+def filter_nones(logger, level, event):
+  if 'url' in event and event['url'] is None:
+    del event['url']
+  return event
+
+def filter_taskname(logger, level, event):
+  # added in Python 3.12, not useful to us, but appears as a normal KV.
+  if 'taskName' in event:
+    del event['taskName']
+  return event
+
 def filter_exc(logger, level, event):
   exc_info = event.get('exc_info')
   if not exc_info:

nvchecker/tools.py

@@ -1,13 +1,16 @@
 # vim: se sw=2:
 # MIT licensed
-# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import sys
 import argparse
+import shutil
 import structlog
 import json
+import os.path
 
 from . import core
+from .util import RichResult
 
 logger = structlog.get_logger(logger_name=__name__)
 
@@ -42,10 +45,11 @@ def take() -> None:
   if args.all:
     oldvers.update(newvers)
   else:
+    name: str
     for name in args.names:
       if "=" in name:
         name, newver = name.split("=")
-        oldvers[name] = newver
+        oldvers[name] = RichResult(version=newver)
       else:
         try:
           oldvers[name] = newvers[name]
@@ -60,9 +64,12 @@ def take() -> None:
           sys.exit(2)
 
   try:
-    oldverf.rename(
-      oldverf.with_name(oldverf.name + '~'),
-    )
+    if os.path.islink(oldverf):
+      shutil.copy(oldverf, oldverf.with_name(oldverf.name + '~'))
+    else:
+      oldverf.rename(
+        oldverf.with_name(oldverf.name + '~'),
+      )
   except FileNotFoundError:
     pass
   core.write_verfile(oldverf, oldvers)
@@ -101,8 +108,8 @@ def cmp() -> None:
     oldverf = opt.ver_files[0]
     newverf = opt.ver_files[1]
 
-  oldvers = core.read_verfile(oldverf)
-  newvers = core.read_verfile(newverf)
+  oldvers = {k: v.version for k, v in core.read_verfile(oldverf).items()}
+  newvers = {k: v.version for k, v in core.read_verfile(newverf).items()}
 
   differences = []
 

nvchecker/util.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import sys
 import asyncio
 from asyncio import Queue
 from typing import (
@@ -13,8 +14,17 @@ from typing import (
 from pathlib import Path
 import contextvars
 import abc
+import netrc
+from dataclasses import dataclass
+
+if TYPE_CHECKING:
+  import tomli as tomllib
+else:
+  try:
+    import tomllib
+  except ModuleNotFoundError:
+    import tomli as tomllib
 
-import tomli
 import structlog
 
 from .httpclient import session
@@ -29,13 +39,40 @@ logger = structlog.get_logger(logger_name=__name__)
 Entry = Dict[str, Any]
 Entry.__doc__ = '''The configuration `dict` for an entry.'''
 Entries = Dict[str, Entry]
-VersData = Dict[str, str]
-VersionResult = Union[None, str, List[str], Exception]
+
+if sys.version_info[:2] >= (3, 11):
+  from typing import LiteralString
+else:
+  LiteralString = str
+
+if sys.version_info[:2] >= (3, 10):
+  @dataclass(kw_only=True)
+  class RichResult:
+    version: str
+    gitref: Optional[str] = None
+    revision: Optional[str] = None
+    url: Optional[str] = None
+
+    def __str__(self):
+      return self.version
+else:
+  @dataclass
+  class RichResult:
+    version: str
+    gitref: Optional[str] = None
+    revision: Optional[str] = None
+    url: Optional[str] = None
+
+    def __str__(self):
+      return self.version
+
+VersionResult = Union[None, str, RichResult, List[Union[str, RichResult]], Exception]
 VersionResult.__doc__ = '''The result of a `get_version` check.
 
 * `None` - No version found.
 * `str` - A single version string is found.
-* `List[str]` - Multiple version strings are found. :ref:`list options` will be applied.
+* `RichResult` - A version string with additional information.
+* `List[Union[str, RichResult]]` - Multiple version strings with or without additional information are found. :ref:`list options` will be applied.
 * `Exception` - An error occurred.
 '''
 
@@ -56,16 +93,24 @@ class KeyManager:
     if file is not None:
       try:
         with file.open('rb') as f:
-          keys = tomli.load(f)['keys']
-      except (OSError, tomli.TOMLDecodeError) as e:
+          keys = tomllib.load(f)['keys']
+      except (OSError, tomllib.TOMLDecodeError) as e:
         raise FileLoadError('keyfile', str(file), e)
     else:
       keys = {}
     self.keys = keys
+    try:
+      netrc_file = netrc.netrc()
+      netrc_hosts = netrc_file.hosts
+    except (FileNotFoundError, netrc.NetrcParseError):
+      netrc_hosts = {}
+    self.netrc = netrc_hosts
 
-  def get_key(self, name: str) -> Optional[str]:
+  def get_key(self, name: str, legacy_name: Optional[str] = None) -> Optional[str]:
     '''Get the named key (token) in the keyfile.'''
-    return self.keys.get(name)
+    keyfile_token = self.keys.get(name) or self.keys.get(legacy_name)
+    netrc_passwd = (e := self.netrc.get(name)) and e[2]
+    return keyfile_token or netrc_passwd
 
 class EntryWaiter:
   def __init__(self) -> None:
@@ -99,10 +144,7 @@ RawResult.name.__doc__ = 'The name (table name) of the entry.'
 RawResult.version.__doc__ = 'The result from the check.'
 RawResult.conf.__doc__ = 'The entry configuration (table content) of the entry.'
 
-class Result(NamedTuple):
-  name: str
-  version: str
-  conf: Entry
+ResultData = Dict[str, RichResult]
 
 class BaseWorker:
   '''The base class for defining `Worker` classes for source plugins.
@@ -283,6 +325,6 @@ class GetVersionError(Exception):
   :param msg: The error message.
   :param kwargs: Arbitrary additional context for the error.
   '''
-  def __init__(self, msg: str, **kwargs: Any) -> None:
+  def __init__(self, msg: LiteralString, **kwargs: Any) -> None:
     self.msg = msg
     self.kwargs = kwargs

nvchecker_source/alpm.py

@@ -20,7 +20,7 @@ async def get_version(name, conf, *, cache, **kwargs):
 
   repo = conf.get('repo')
   if repo is None:
-    repos = conf.get('repos') or ['core', 'extra', 'community', 'multilib']
+    repos = conf.get('repos') or ['core', 'extra', 'multilib']
   else:
     repos = [repo]
 

nvchecker_source/alpmfiles.py

@@ -0,0 +1,51 @@
+# MIT licensed
+# Copyright (c) 2023 Pekka Ristola <pekkarr [at] protonmail [dot] com>, et al.
+
+from asyncio import create_subprocess_exec
+from asyncio.subprocess import PIPE
+import re
+from typing import Tuple, List
+
+from nvchecker.api import GetVersionError
+
+async def get_files(info: Tuple[str, str]) -> List[str]:
+  dbpath, pkg = info
+  # there's no pyalpm bindings for the file databases
+  cmd = ['pacman', '-Flq', '--dbpath', dbpath, pkg]
+
+  p = await create_subprocess_exec(*cmd, stdout = PIPE, stderr = PIPE)
+  stdout, stderr = await p.communicate()
+
+  if p.returncode == 0:
+    return stdout.decode().splitlines()
+  else:
+    raise GetVersionError(
+      'pacman failed to get file list',
+      pkg = pkg,
+      cmd = cmd,
+      stdout = stdout.decode(errors='replace'),
+      stderr = stderr.decode(errors='replace'),
+      returncode = p.returncode,
+    )
+
+async def get_version(name, conf, *, cache, **kwargs):
+  pkg = conf['pkgname']
+  repo = conf.get('repo')
+  if repo is not None:
+    pkg = f'{repo}/{pkg}'
+  dbpath = conf.get('dbpath', '/var/lib/pacman')
+  regex = re.compile(conf['filename'])
+  if regex.groups > 1:
+    raise GetVersionError('multi-group regex')
+  strip_dir = conf.get('strip_dir', False)
+
+  files = await cache.get((dbpath, pkg), get_files)
+
+  for f in files:
+    fn = f.rsplit('/', 1)[-1] if strip_dir else f
+    match = regex.fullmatch(fn)
+    if match:
+      groups = match.groups()
+      return groups[0] if len(groups) > 0 else fn
+
+  raise GetVersionError('no file matches specified regex')

nvchecker_source/anitya.py

@@ -1,10 +1,17 @@
 # MIT licensed
 # Copyright (c) 2017-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+from nvchecker.api import RichResult
+
 URL = 'https://release-monitoring.org/api/project/{pkg}'
 
 async def get_version(name, conf, *, cache, **kwargs):
-  pkg = conf.get('anitya')
+  pkg = conf.get('anitya_id')
+  if pkg is None:
+    pkg = conf.get('anitya')
   url = URL.format(pkg = pkg)
   data = await cache.get_json(url)
-  return data['version']
+  return RichResult(
+    version = data['version'],
+    url = f'https://release-monitoring.org/project/{data["id"]}/',
+  )

nvchecker_source/apt.py

@@ -11,8 +11,8 @@ import functools
 from collections import defaultdict
 
 from nvchecker.api import (
-  session, GetVersionError,
-  VersionResult, Entry, AsyncCache, KeyManager,
+  session, GetVersionError, VersionResult,
+  RichResult, Entry, AsyncCache, KeyManager,
 )
 
 APT_RELEASE_URL = "%s/dists/%s/Release"
@@ -92,12 +92,13 @@ async def get_url(url: str) -> str:
     None, _decompress_data,
     url, data)
 
-async def parse_packages(key: Tuple[AsyncCache, str]) -> Tuple[Dict[str, str], Dict[str, str]]:
+async def parse_packages(key: Tuple[AsyncCache, str]) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
   cache, url = key
   apt_packages = await cache.get(url, get_url) # type: ignore
 
   pkg_map = defaultdict(list)
   srcpkg_map = defaultdict(list)
+  pkg_to_src_map = defaultdict(list)
 
   pkg = None
   srcpkg = None
@@ -110,6 +111,7 @@ async def parse_packages(key: Tuple[AsyncCache, str]) -> Tuple[Dict[str, str], D
       version = line[9:]
       if pkg is not None:
         pkg_map[pkg].append(version)
+        pkg_to_src_map["%s/%s" % (pkg, version)] = srcpkg if srcpkg is not None else pkg
       if srcpkg is not None:
         srcpkg_map[srcpkg].append(version)
       pkg = srcpkg = None
@@ -118,8 +120,10 @@ async def parse_packages(key: Tuple[AsyncCache, str]) -> Tuple[Dict[str, str], D
                  for pkg, vs in pkg_map.items()}
   srcpkg_map_max = {pkg: max(vs, key=functools.cmp_to_key(compare_version))
                  for pkg, vs in srcpkg_map.items()}
+  pkg_to_src_map_max = {pkg: pkg_to_src_map["%s/%s" % (pkg, vs)]
+                 for pkg, vs in pkg_map_max.items()}
 
-  return pkg_map_max, srcpkg_map_max
+  return pkg_map_max, srcpkg_map_max, pkg_to_src_map_max
 
 async def get_version(
   name: str, conf: Entry, *,
@@ -148,16 +152,38 @@ async def get_version(
   else:
     raise GetVersionError('Packages file not found in APT repository')
 
-  pkg_map, srcpkg_map = await cache.get(
+  pkg_map, srcpkg_map, pkg_to_src_map = await cache.get(
     (cache, APT_PACKAGES_URL % (mirror, suite, packages_path)), parse_packages) # type: ignore
 
   if pkg and pkg in pkg_map:
     version = pkg_map[pkg]
+    changelog_name = pkg_to_src_map[pkg]
   elif srcpkg and srcpkg in srcpkg_map:
     version = srcpkg_map[srcpkg]
+    changelog_name = srcpkg
   else:
     raise GetVersionError('package not found in APT repository')
 
+  # Get Changelogs field from the Release file
+  changelogs_url = None
+  for line in apt_release.split('\n'):
+    if line.startswith('Changelogs: '):
+      changelogs_url = line[12:]
+      break
+
+  # Build the changelog URL (see https://wiki.debian.org/DebianRepository/Format#Changelogs for spec)
+  changelog = None
+  if changelogs_url is not None and changelogs_url != 'no':
+    changelog_section = changelog_name[:4] if changelog_name.startswith('lib') else changelog_name[:1]
+    changelog = changelogs_url.replace('@CHANGEPATH@', f'{repo}/{changelog_section}/{changelog_name}/{changelog_name}_{version}')
+
   if strip_release:
     version = version.split("-")[0]
-  return version
+
+  if changelog is not None:
+    return RichResult(
+      version = version,
+      url = changelog,
+    )
+  else:
+    return version

nvchecker_source/archpkg.py

@@ -1,9 +1,9 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from nvchecker.api import session, GetVersionError
+from nvchecker.api import session, RichResult, GetVersionError
 
-URL = 'https://www.archlinux.org/packages/search/json/'
+URL = 'https://archlinux.org/packages/search/json/'
 
 async def request(pkg):
   res = await session.get(URL, params={"name": pkg})
@@ -31,4 +31,7 @@ async def get_version(name, conf, *, cache, **kwargs):
   else:
     version = r['pkgver'] + '-' + r['pkgrel']
 
-  return version
+  return RichResult(
+    version = version,
+    url = f'https://archlinux.org/packages/{r["repo"]}/{r["arch"]}/{r["pkgname"]}/',
+  )

nvchecker_source/aur.py

@@ -1,12 +1,12 @@
 # MIT licensed
-# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020,2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from datetime import datetime
+from datetime import datetime, timezone
 import asyncio
 from typing import Iterable, Dict, List, Tuple, Any, Optional
 
 from nvchecker.api import (
-  session, GetVersionError, VersionResult,
+  session, GetVersionError, VersionResult, RichResult,
   Entry, BaseWorker, RawResult,
 )
 
@@ -95,11 +95,15 @@ async def _run_batch_impl(
 
     version = result['Version']
     if use_last_modified:
-      version += '-' + datetime.utcfromtimestamp(result['LastModified']).strftime('%Y%m%d%H%M%S')
+      dt = datetime.fromtimestamp(result['LastModified'], timezone.utc)
+      version += '-' + dt.strftime('%Y%m%d%H%M%S')
     if strip_release and '-' in version:
       version = version.rsplit('-', 1)[0]
 
-    ret[name] = version
+    ret[name] = RichResult(
+      version = version,
+      url = f'https://aur.archlinux.org/packages/{name}',
+    )
 
   return ret
 

nvchecker_source/bitbucket.py

@@ -1,10 +1,10 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from typing import Any, List
+from typing import Any, List, Union
 from urllib.parse import urlencode
 
-from nvchecker.api import VersionResult, Entry, AsyncCache
+from nvchecker.api import VersionResult, RichResult, Entry, AsyncCache
 
 # doc: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-commits/#api-repositories-workspace-repo-slug-commits-get
 BITBUCKET_URL = 'https://bitbucket.org/api/2.0/repositories/%s/commits/%s'
@@ -22,7 +22,7 @@ async def get_version(
   use_sorted_tags = conf.get('use_sorted_tags', False)
 
   if use_sorted_tags or use_max_tag:
-    parameters = {'fields': 'values.name,next'}
+    parameters = {'fields': 'values.name,values.links.html.href,next'}
 
     if use_sorted_tags:
       parameters['sort'] = conf.get('sort', '-target.date')
@@ -33,37 +33,41 @@ async def get_version(
     url = BITBUCKET_MAX_TAG % repo
     url += '?' + urlencode(parameters)
 
-    version = await _get_tags(url, max_page=1, cache=cache)
+    return await _get_tags(url, max_page=1, cache=cache)
 
   elif use_max_tag:
     url = BITBUCKET_MAX_TAG % repo
     url += '?' + urlencode(parameters)
 
     max_page = conf.get('max_page', 3)
-    version = await _get_tags(url, max_page=max_page, cache=cache)
+    return await _get_tags(url, max_page=max_page, cache=cache)
 
   else:
     url = BITBUCKET_URL % (repo, br)
     data = await cache.get_json(url)
-
-    version = data['values'][0]['date'].split('T', 1)[0].replace('-', '')
-
-  return version
+    return RichResult(
+      version = data['values'][0]['date'].split('T', 1)[0].replace('-', ''),
+      url = data['values'][0]['links']['html']['href'],
+    )
 
 async def _get_tags(
   url: str, *,
   max_page: int,
   cache: AsyncCache,
-) -> List[str]:
-  ret: List[str] = []
+) -> VersionResult:
+  ret: List[Union[str, RichResult]] = []
 
   for _ in range(max_page):
     data = await cache.get_json(url)
-    ret.extend(x['name'] for x in data['values'])
+    ret.extend([
+      RichResult(
+        version = tag['name'],
+        url = tag['links']['html']['href'],
+      ) for tag in data['values']
+    ])
     if 'next' in data:
       url = data['next']
     else:
       break
 
   return ret
-

nvchecker_source/container.py

@@ -4,6 +4,7 @@
 from typing import Dict, List, NamedTuple, Optional, Tuple
 from urllib.request import parse_http_list
 from urllib.parse import urljoin
+import json
 
 from nvchecker.api import session, HTTPError
 
@@ -57,15 +58,7 @@ async def get_registry_auth_info(registry_host: str) -> AuthInfo:
 
 async def get_container_tags(info: Tuple[str, str, AuthInfo]) -> List[str]:
   image_path, registry_host, auth_info = info
-
-  auth_params = {
-    'scope': f'repository:{image_path}:pull',
-  }
-  if auth_info.service:
-    auth_params['service'] = auth_info.service
-  res = await session.get(auth_info.realm, params=auth_params)
-  token = res.json()['token']
-
+  token = await get_auth_token(auth_info, image_path)
   tags = []
   url = f'https://{registry_host}/v2/{image_path}/tags/list'
 
@@ -83,6 +76,18 @@ async def get_container_tags(info: Tuple[str, str, AuthInfo]) -> List[str]:
 
   return tags
 
+
+async def get_auth_token(auth_info, image_path):
+  auth_params = {
+    'scope': f'repository:{image_path}:pull',
+  }
+  if auth_info.service:
+    auth_params['service'] = auth_info.service
+  res = await session.get(auth_info.realm, params=auth_params)
+  token = res.json()['token']
+  return token
+
+
 def parse_next_link(value: str) -> str:
   ending = '>; rel="next"'
   if value.endswith(ending):
@@ -90,13 +95,70 @@ def parse_next_link(value: str) -> str:
   else:
     raise ValueError(value)
 
+
+async def get_container_tag_update_time(info: Tuple[str, str, str, AuthInfo]):
+  '''
+  Find the update time of a container tag.
+
+  In fact, it's the creation time of the image ID referred by the tag. Tag itself does not have any update time.
+  '''
+  image_path, image_tag, registry_host, auth_info = info
+  token = await get_auth_token(auth_info, image_path)
+
+  # HTTP headers
+  headers = {
+    'Authorization': f'Bearer {token}',
+    # Prefer Image Manifest Version 2, Schema 2: https://distribution.github.io/distribution/spec/manifest-v2-2/
+    'Accept': ', '.join([
+      'application/vnd.oci.image.manifest.v1+json',
+      'application/vnd.oci.image.index.v1+json',
+      'application/vnd.docker.distribution.manifest.v2+json',
+      'application/vnd.docker.distribution.manifest.list.v2+json',
+      'application/json',
+    ]),
+  }
+
+  # Get tag manifest
+  url = f'https://{registry_host}/v2/{image_path}/manifests/{image_tag}'
+  res = await session.get(url, headers=headers)
+  data = res.json()
+  # Schema 1 returns the creation time in the response
+  if data['schemaVersion'] == 1:
+    return json.loads(data['history'][0]['v1Compatibility'])['created']
+
+  # For schema 2, we have to fetch the config's blob
+  # For multi-arch images, multiple manifests are bounded with the same tag. We should choose one and then request
+  # the manifest's detail
+  if data.get('manifests'):
+    # It's quite hard to find the manifest matching with current CPU architecture and system.
+    # For now we just choose the first and it should probably work for most cases
+    image_digest = data['manifests'][0]['digest']
+    url = f'https://{registry_host}/v2/{image_path}/manifests/{image_digest}'
+    res = await session.get(url, headers=headers)
+    data = res.json()
+
+  digest = data['config']['digest']
+  url = f'https://{registry_host}/v2/{image_path}/blobs/{digest}'
+  res = await session.get(url, headers=headers)
+  data = res.json()
+  return data['created']
+
+
 async def get_version(name, conf, *, cache, **kwargs):
   image_path = conf.get('container', name)
+  image_tag = None
+  # image tag is optional
+  if ':' in image_path:
+    image_path, image_tag = image_path.split(':', 1)
   registry_host = conf.get('registry', 'docker.io')
   if registry_host == 'docker.io':
     registry_host = 'registry-1.docker.io'
 
   auth_info = await cache.get(registry_host, get_registry_auth_info)
 
+  # if a tag is given, return the tag's update time, otherwise return the image's tag list
+  if image_tag:
+    key = image_path, image_tag, registry_host, auth_info
+    return await cache.get(key, get_container_tag_update_time)
   key = image_path, registry_host, auth_info
   return await cache.get(key, get_container_tags)

nvchecker_source/cpan.py

@@ -1,11 +1,15 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+from nvchecker.api import RichResult
+
 # Using metacpan
 CPAN_URL = 'https://fastapi.metacpan.org/release/%s'
 
 async def get_version(name, conf, *, cache, **kwargs):
   key = conf.get('cpan', name)
   data = await cache.get_json(CPAN_URL % key)
-  return str(data['version'])
-
+  return RichResult(
+    version = str(data['version']),
+    url = f'https://metacpan.org/release/{data["author"]}/{data["name"]}',
+  )

nvchecker_source/cran.py

@@ -1,7 +1,7 @@
 # MIT licensed
 # Copyright (c) 2022 Pekka Ristola <pekkarr [at] protonmail [dot] com>, et al.
 
-from nvchecker.api import session, GetVersionError
+from nvchecker.api import session, RichResult, GetVersionError
 
 CRAN_URL = 'https://cran.r-project.org/package=%s/DESCRIPTION'
 VERSION_FIELD = 'Version: '
@@ -23,4 +23,7 @@ async def get_version(name, conf, *, cache, **kwargs):
   else:
     raise GetVersionError('Invalid DESCRIPTION file')
 
-  return version
+  return RichResult(
+    version = version,
+    url = f'https://cran.r-project.org/web/packages/{package}/',
+  )

nvchecker_source/cratesio.py

@@ -1,10 +1,40 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+import re
+
+import structlog
+
+from nvchecker.api import RichResult
+
+logger = structlog.get_logger(logger_name=__name__)
+
+
 API_URL = 'https://crates.io/api/v1/crates/%s'
+# https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+VERSION_PATTERN = r'^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$'
+
 
 async def get_version(name, conf, *, cache, **kwargs):
   name = conf.get('cratesio') or name
+  use_pre_release = conf.get('use_pre_release', False)
   data = await cache.get_json(API_URL % name)
-  version = [v['num'] for v in data['versions'] if not v['yanked']][0]
-  return version
+  results = []
+  for v in data['versions']:
+    if v['yanked']:
+      continue
+    version = v['num']
+    match = re.fullmatch(VERSION_PATTERN, version)
+    if match is None:
+      logger.warning('ignoring invalid version', version=version)
+      continue
+    if not use_pre_release and match.group('prerelease'):
+      continue
+    results.append(
+      RichResult(
+        version=version,
+        url=f'https://crates.io/crates/{name}/{version}',
+      )
+    )
+
+  return results

nvchecker_source/debianpkg.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 # Copyright (c) 2017 Felix Yan <felixonmars@archlinux.org>, et al.
 
-from nvchecker.api import GetVersionError
+from nvchecker.api import RichResult, GetVersionError
 
 URL = 'https://sources.debian.org/api/src/%(pkgname)s/?suite=%(suite)s'
 
@@ -22,4 +22,7 @@ async def get_version(name, conf, *, cache, **kwargs):
   else:
     version = r['version']
 
-  return version
+  return RichResult(
+    version = version,
+    url = f'https://sources.debian.org/src/{data["package"]}/{r["version"]}/',
+  )

nvchecker_source/gems.py

@@ -1,9 +1,16 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+from nvchecker.api import RichResult
+
 GEMS_URL = 'https://rubygems.org/api/v1/versions/%s.json'
 
 async def get_version(name, conf, *, cache, **kwargs):
   key = conf.get('gems', name)
   data = await cache.get_json(GEMS_URL % key)
-  return [item['number'] for item in data]
+  return [
+    RichResult(
+      version = item['number'],
+      url = f'https://rubygems.org/gems/{key}/versions/{item["number"]}',
+    ) for item in data
+  ]

nvchecker_source/git.py

@@ -1,7 +1,9 @@
 # MIT licensed
 # Copyright (c) 2020 Felix Yan <felixonmars@archlinux.org>, et al.
 
-from .cmd import run_cmd # type: ignore
+from .cmd import run_cmd
+
+from nvchecker.api import RichResult
 
 async def get_version(
   name, conf, *, cache, keymanager=None
@@ -13,13 +15,27 @@ async def get_version(
     ref = conf.get('branch')
     if ref is None:
       ref = 'HEAD'
+      gitref = None
     else:
       ref = 'refs/heads/' + ref
+      gitref = ref
     cmd = f"git ls-remote {git} {ref}"
     data = await cache.get(cmd, run_cmd)
-    return data.split(None, 1)[0]
+    version = data.split(None, 1)[0]
+    return RichResult(
+      version = version,
+      revision = version,
+      gitref = gitref,
+    )
   else:
     cmd = f"git ls-remote --tags --refs {git}"
     data = await cache.get(cmd, run_cmd)
-    versions = [line.split("refs/tags/")[1] for line in data.splitlines()]
+    versions = []
+    for line in data.splitlines():
+      revision, version = line.split("\trefs/tags/", 1)
+      versions.append(RichResult(
+        version = version,
+        revision = revision,
+        gitref = f"refs/tags/{version}",
+      ))
     return versions

nvchecker_source/gitea.py

@@ -9,7 +9,8 @@ GITEA_URL = 'https://%s/api/v1/repos/%s/commits'
 GITEA_MAX_TAG = 'https://%s/api/v1/repos/%s/tags'
 
 from nvchecker.api import (
-  VersionResult, Entry, AsyncCache, KeyManager,
+  VersionResult, RichResult, Entry,
+  AsyncCache, KeyManager,
 )
 
 async def get_version(
@@ -32,8 +33,7 @@ async def get_version(
   token = conf.get('token')
   # Load token from keyman
   if token is None:
-    key_name = 'gitea_' + host.lower()
-    token = keymanager.get_key(key_name)
+    token = keymanager.get_key(host.lower(), 'gitea_' + host.lower())
 
   # Set private token if token exists.
   headers = {}
@@ -42,7 +42,16 @@ async def get_version(
 
   data = await cache.get_json(url, headers = headers)
   if use_max_tag:
-    version = [tag["name"] for tag in data]
+    return [
+      RichResult(
+        version = tag['name'],
+        revision = tag['id'],
+        url = f'https://{host}/{conf["gitea"]}/releases/tag/{tag["name"]}',
+      ) for tag in data
+    ]
   else:
-    version = data[0]['commit']['committer']['date'].split('T', 1)[0].replace('-', '')
-  return version
+    return RichResult(
+      version = data[0]['commit']['committer']['date'],
+      revision = data[0]['sha'],
+      url = data[0]['html_url'],
+    )

nvchecker_source/github.py

@@ -1,30 +1,52 @@
 # MIT licensed
-# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020, 2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import time
 from urllib.parse import urlencode
-from typing import Tuple
+from typing import List, Tuple, Union, Optional
+import asyncio
 
 import structlog
 
 from nvchecker.api import (
   VersionResult, Entry, AsyncCache, KeyManager,
-  TemporaryError, session, GetVersionError,
+  HTTPError, session, RichResult, GetVersionError,
 )
 
 logger = structlog.get_logger(logger_name=__name__)
+ALLOW_REQUEST = None
+RATE_LIMITED_ERROR = False
 
-GITHUB_URL = 'https://api.github.com/repos/%s/commits'
-GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest'
+GITHUB_URL = 'https://api.%s/repos/%s/commits'
+GITHUB_LATEST_RELEASE = 'https://api.%s/repos/%s/releases/latest'
 # https://developer.github.com/v3/git/refs/#get-all-references
-GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/git/refs/tags'
-GITHUB_GRAPHQL_URL = 'https://api.github.com/graphql'
+GITHUB_MAX_TAG = 'https://api.%s/repos/%s/git/refs/tags'
+GITHUB_MAX_RELEASE = 'https://api.%s/repos/%s/releases'
+GITHUB_GRAPHQL_URL = 'https://api.%s/graphql'
 
 async def get_version(name, conf, **kwargs):
-  try:
-    return await get_version_real(name, conf, **kwargs)
-  except TemporaryError as e:
-    check_ratelimit(e, name)
+  global RATE_LIMITED_ERROR, ALLOW_REQUEST
+
+  if RATE_LIMITED_ERROR:
+    raise RuntimeError('rate limited')
+
+  if ALLOW_REQUEST is None:
+    ALLOW_REQUEST = asyncio.Event()
+    ALLOW_REQUEST.set()
+
+  for _ in range(2): # retry once
+    try:
+      await ALLOW_REQUEST.wait()
+      return await get_version_real(name, conf, **kwargs)
+    except HTTPError as e:
+      if e.code in [403, 429]:
+        if n := check_ratelimit(e, name):
+          ALLOW_REQUEST.clear()
+          await asyncio.sleep(n+1)
+          ALLOW_REQUEST.set()
+          continue
+        RATE_LIMITED_ERROR = True
+      raise
 
 QUERY_LATEST_TAG = '''
 {{
@@ -35,6 +57,9 @@ QUERY_LATEST_TAG = '''
       edges {{
         node {{
           name
+          target {{
+            oid
+          }}
         }}
       }}
     }}
@@ -42,8 +67,29 @@ QUERY_LATEST_TAG = '''
 }}
 '''
 
-async def get_latest_tag(key: Tuple[str, str, str]) -> str:
-  repo, query, token = key
+QUERY_LATEST_RELEASE_WITH_PRERELEASES = '''
+{{
+  repository(name: "{name}", owner: "{owner}") {{
+    releases(first: 1, orderBy: {{field: CREATED_AT, direction: DESC}}) {{
+      edges {{
+        node {{
+          name
+          url
+          tag {{
+            name
+          }}
+          tagCommit {{
+            oid
+          }}
+        }}
+      }}
+    }}
+  }}
+}}
+'''
+
+async def get_latest_tag(key: Tuple[str, str, str, str]) -> RichResult:
+  host, repo, query, token = key
   owner, reponame = repo.split('/')
   headers = {
     'Authorization': f'bearer {token}',
@@ -56,7 +102,7 @@ async def get_latest_tag(key: Tuple[str, str, str]) -> str:
   )
 
   res = await session.post(
-    GITHUB_GRAPHQL_URL,
+    GITHUB_GRAPHQL_URL % host,
     headers = headers,
     json = {'query': q},
   )
@@ -66,7 +112,50 @@ async def get_latest_tag(key: Tuple[str, str, str]) -> str:
   if not refs:
     raise GetVersionError('no tag found')
 
-  return refs[0]['node']['name']
+  version = refs[0]['node']['name']
+  revision = refs[0]['node']['target']['oid']
+  return RichResult(
+    version = version,
+    gitref = f"refs/tags/{version}",
+    revision = revision,
+    url = f'https://github.com/{repo}/releases/tag/{version}',
+  )
+
+async def get_latest_release_with_prereleases(key: Tuple[str, str, str, str]) -> RichResult:
+  host, repo, token, use_release_name = key
+  owner, reponame = repo.split('/')
+  headers = {
+    'Authorization': f'bearer {token}',
+    'Content-Type': 'application/json',
+  }
+  q = QUERY_LATEST_RELEASE_WITH_PRERELEASES.format(
+    owner = owner,
+    name = reponame,
+  )
+
+  res = await session.post(
+    GITHUB_GRAPHQL_URL % host,
+    headers = headers,
+    json = {'query': q},
+  )
+  j = res.json()
+
+  refs = j['data']['repository']['releases']['edges']
+  if not refs:
+    raise GetVersionError('no release found')
+
+  tag_name = refs[0]['node']['tag']['name']
+  if use_release_name:
+    version = refs[0]['node']['name']
+  else:
+    version = tag_name
+
+  return RichResult(
+    version = version,
+    gitref = f"refs/tags/{tag_name}",
+    revision = refs[0]['node']['tagCommit']['oid'],
+    url = refs[0]['node']['url'],
+  )
 
 async def get_version_real(
   name: str, conf: Entry, *,
@@ -74,12 +163,13 @@ async def get_version_real(
   **kwargs,
 ) -> VersionResult:
   repo = conf['github']
+  host = conf.get('host', "github.com")
 
   # Load token from config
   token = conf.get('token')
   # Load token from keyman
   if token is None:
-    token = keymanager.get_key('github')
+    token = keymanager.get_key(host.lower(), 'github')
 
   use_latest_tag = conf.get('use_latest_tag', False)
   if use_latest_tag:
@@ -87,18 +177,31 @@ async def get_version_real(
       raise GetVersionError('token not given but it is required')
 
     query = conf.get('query', '')
-    return await cache.get((repo, query, token), get_latest_tag) # type: ignore
+    return await cache.get((host, repo, query, token), get_latest_tag) # type: ignore
+
+  use_latest_release = conf.get('use_latest_release', False)
+  include_prereleases = conf.get('include_prereleases', False)
+  use_release_name = conf.get('use_release_name', False)
+  if use_latest_release and include_prereleases:
+    if not token:
+      raise GetVersionError('token not given but it is required')
+
+    return await cache.get(
+      (host, repo, token, use_release_name),
+      get_latest_release_with_prereleases) # type: ignore
 
   br = conf.get('branch')
   path = conf.get('path')
-  use_latest_release = conf.get('use_latest_release', False)
   use_max_tag = conf.get('use_max_tag', False)
+  use_max_release = conf.get('use_max_release', False)
   if use_latest_release:
-    url = GITHUB_LATEST_RELEASE % repo
+    url = GITHUB_LATEST_RELEASE % (host, repo)
   elif use_max_tag:
-    url = GITHUB_MAX_TAG % repo
+    url = GITHUB_MAX_TAG % (host, repo)
+  elif use_max_release:
+    url = GITHUB_MAX_RELEASE % (host, repo)
   else:
-    url = GITHUB_URL % repo
+    url = GITHUB_URL % (host, repo)
     parameters = {}
     if br:
       parameters['sha'] = br
@@ -114,27 +217,62 @@ async def get_version_real(
   data = await cache.get_json(url, headers = headers)
 
   if use_max_tag:
-    tags = [ref['ref'].split('/', 2)[-1] for ref in data]
+    tags: List[Union[str, RichResult]] = [
+      RichResult(
+        version = ref['ref'].split('/', 2)[-1],
+        gitref = ref['ref'],
+        revision = ref['object']['sha'],
+        url = f'https://github.com/{repo}/releases/tag/{ref["ref"].split("/", 2)[-1]}',
+      ) for ref in data
+    ]
     if not tags:
       raise GetVersionError('No tag found in upstream repository.')
     return tags
 
+  if use_max_release:
+    releases: List[Union[str, RichResult]] = [
+      RichResult(
+        version = ref['name'] if use_release_name else ref['tag_name'],
+        gitref = f"refs/tags/{ref['tag_name']}",
+        url = ref['html_url'],
+      ) for ref in data if include_prereleases or not ref['prerelease']
+    ]
+    if not releases:
+      raise GetVersionError('No release found in upstream repository.')
+    return releases
+
   if use_latest_release:
     if 'tag_name' not in data:
       raise GetVersionError('No release found in upstream repository.')
-    version = data['tag_name']
+
+    if use_release_name:
+      version = data['name']
+    else:
+      version = data['tag_name']
+
+    return RichResult(
+      version = version,
+      gitref = f"refs/tags/{data['tag_name']}",
+      url = data['html_url'],
+    )
 
   else:
-    # YYYYMMDD.HHMMSS
-    version = data[0]['commit']['committer']['date'] \
-        .rstrip('Z').replace('-', '').replace(':', '').replace('T', '.')
+    return RichResult(
+      # YYYYMMDD.HHMMSS
+      version = data[0]['commit']['committer']['date'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.'),
+      revision = data[0]['sha'],
+      url = data[0]['html_url'],
+    )
 
-  return version
-
-def check_ratelimit(exc, name):
+def check_ratelimit(exc: HTTPError, name: str) -> Optional[int]:
   res = exc.response
   if not res:
-    raise
+    raise exc
+
+  if v := res.headers.get('retry-after'):
+    n = int(v)
+    logger.warning('retry-after', n=n)
+    return n
 
   # default -1 is used to re-raise the exception
   n = int(res.headers.get('X-RateLimit-Remaining', -1))
@@ -144,5 +282,6 @@ def check_ratelimit(exc, name):
                   'Or get an API token to increase the allowance if not yet',
                  name = name,
                  reset = reset)
-  else:
-    raise
+    return None
+
+  raise exc

nvchecker_source/gitlab.py

@@ -6,8 +6,8 @@ import urllib.parse
 import structlog
 
 from nvchecker.api import (
-  VersionResult, Entry, AsyncCache, KeyManager,
-  TemporaryError,
+  VersionResult, RichResult, Entry,
+  AsyncCache, KeyManager, TemporaryError,
 )
 
 GITLAB_URL = 'https://%s/api/v4/projects/%s/repository/commits'
@@ -42,8 +42,7 @@ async def get_version_real(
   token = conf.get('token')
   # Load token from keyman
   if token is None:
-    key_name = 'gitlab_' + host.lower()
-    token = keymanager.get_key(key_name)
+    token = keymanager.get_key(host.lower(), 'gitlab_' + host.lower())
 
   # Set private token if token exists.
   headers = {}
@@ -52,10 +51,19 @@ async def get_version_real(
 
   data = await cache.get_json(url, headers = headers)
   if use_max_tag:
-    version = [tag["name"] for tag in data]
+    return [
+      RichResult(
+        version = tag['name'],
+        revision = tag['commit']['id'],
+        url = f'https://{host}/{conf["gitlab"]}/-/tags/{tag["name"]}',
+      ) for tag in data
+    ]
   else:
-    version = data[0]['created_at'].split('T', 1)[0].replace('-', '')
-  return version
+    return RichResult(
+      version = data[0]['created_at'].split('T', 1)[0].replace('-', ''),
+      revision = data[0]['id'],
+      url = data[0]['web_url'],
+    )
 
 def check_ratelimit(exc, name):
   res = exc.response

nvchecker_source/go.py

@@ -0,0 +1,40 @@
+# MIT licensed
+# Copyright (c) 2024 bgme <i@bgme.me>.
+
+from lxml import html
+
+from nvchecker.api import (
+  RichResult, Entry, AsyncCache, KeyManager,
+  session, GetVersionError,
+)
+
+GO_PKG_URL = 'https://pkg.go.dev/{pkg}?tab=versions'
+GO_PKG_VERSION_URL = 'https://pkg.go.dev/{pkg}@{version}'
+
+
+async def get_version(
+    name: str, conf: Entry, *,
+    cache: AsyncCache, keymanager: KeyManager,
+    **kwargs,
+) -> RichResult:
+  key = tuple(sorted(conf.items()))
+  return await cache.get(key, get_version_impl)
+
+
+async def get_version_impl(info) -> RichResult:
+  conf = dict(info)
+  pkg_name = conf.get('go')
+
+  url = GO_PKG_URL.format(pkg=pkg_name)
+  res = await session.get(url)
+  doc = html.fromstring(res.body.decode())
+
+  elements = doc.xpath("//div[@class='Version-tag']/a/text()")
+  try:
+    version = elements[0] # type: ignore
+    return RichResult(
+      version = version, # type: ignore
+      url = GO_PKG_VERSION_URL.format(pkg=pkg_name, version=version),
+    )
+  except IndexError:
+    raise GetVersionError("parse error", pkg_name=pkg_name)

nvchecker_source/hackage.py

@@ -1,10 +1,15 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+from nvchecker.api import RichResult
+
 HACKAGE_URL = 'https://hackage.haskell.org/package/%s/preferred.json'
 
 async def get_version(name, conf, *, cache, **kwargs):
   key = conf.get('hackage', name)
   data = await cache.get_json(HACKAGE_URL % key)
-  return data['normal-version'][0]
-
+  version = data['normal-version'][0]
+  return RichResult(
+    version = version,
+    url = f'https://hackage.haskell.org/package/{key}-{version}',
+  )

nvchecker_source/httpheader.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2021 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import re
-import sre_constants
 
 from nvchecker.api import session, GetVersionError
 
@@ -19,7 +18,7 @@ async def get_version_impl(info):
 
   try:
     regex = re.compile(conf['regex'])
-  except sre_constants.error as e:
+  except re.error as e:
     raise GetVersionError('bad regex', exc_info=e)
 
   res = await session.request(
@@ -30,7 +29,11 @@ async def get_version_impl(info):
 
   header_value = res.headers.get(header)
   if not header_value:
-    raise GetVersionError('header %s not found or is empty' % header)
+    raise GetVersionError(
+      'header not found or is empty',
+      header = header,
+      value = header_value,
+    )
 
   try:
     version = regex.findall(header_value)

nvchecker_source/jq.py

@@ -0,0 +1,42 @@
+# MIT licensed
+# Copyright (c) 2024 Rocket Aaron <i@rocka.me>, et al.
+
+import json
+import jq
+
+from nvchecker.api import session, GetVersionError
+
+async def get_version(name, conf, *, cache, **kwargs):
+  key = tuple(sorted(conf.items()))
+  return await cache.get(key, get_version_impl)
+
+async def get_version_impl(info):
+  conf = dict(info)
+
+  try:
+    program = jq.compile(conf.get('filter', '.'))
+  except ValueError as e:
+    raise GetVersionError('bad jq filter', exc_info=e)
+
+  data = conf.get('post_data')
+  if data is None:
+    res = await session.get(conf['url'])
+  else:
+    res = await session.post(conf['url'], body = data, headers = {
+        'Content-Type': conf.get('post_data_type', 'application/json')
+      })
+
+  try:
+    obj = json.loads(res.body)
+  except json.decoder.JSONDecodeError as e:
+    raise GetVersionError('bad json string', exc_info=e)
+
+  try:
+    version = program.input(obj).all()
+    if version == [None] and not conf.get('missing_ok', False):
+      raise GetVersionError('version string not found.')
+    version = [str(v) for v in version]
+  except ValueError as e:
+    raise GetVersionError('failed to filter json', exc_info=e)
+
+  return version

nvchecker_source/launchpad.py

@@ -0,0 +1,20 @@
+# MIT Licensed
+# Copyright (c) 2024 Bert Peters <bertptrs@archlinux.org>, et al.
+from __future__ import annotations
+from nvchecker.api import AsyncCache, Entry, RichResult
+
+PROJECT_INFO_URL = "https://api.launchpad.net/1.0/{launchpad}"
+
+async def get_version(name: str, conf: Entry, *, cache: AsyncCache, **kwargs):
+  launchpad = conf["launchpad"]
+
+  project_data = await cache.get_json(PROJECT_INFO_URL.format(launchpad=launchpad))
+  data = await cache.get_json(project_data['releases_collection_link'])
+
+  return [
+    RichResult(version=entry["version"], url=entry["web_link"])
+    for entry in data["entries"]
+  ]
+
+
+

nvchecker_source/mercurial.py

@@ -0,0 +1,10 @@
+# MIT licensed
+# Copyright (c) 2020 Felix Yan <felixonmars@archlinux.org>, et al.
+
+async def get_version(name, conf, *, cache, **kwargs):
+  url = conf['mercurial'] + '/json-tags'
+
+  data = await cache.get_json(url)
+
+  version = [tag['tag'] for tag in data['tags']]
+  return version

nvchecker_source/npm.py

@@ -3,7 +3,7 @@
 
 import json
 import re
-from nvchecker.api import session
+from nvchecker.api import session, RichResult
 
 NPM_URL = 'https://registry.npmjs.org/%s'
 
@@ -26,4 +26,13 @@ async def get_version(name, conf, *, cache, **kwargs):
   data = await cache.get(NPM_URL % key, get_first_1k)
 
   dist_tags = json.loads(re.search(b'"dist-tags":({.*?})', data).group(1))
-  return dist_tags['latest']
+  version = dist_tags['latest']
+
+  # There is no standardised URL scheme, so we only return an URL for the default registry
+  if NPM_URL.startswith('https://registry.npmjs.org/'):
+    return RichResult(
+      version = version,
+      url = f'https://www.npmjs.com/package/{key}/v/{version}',
+    )
+  else:
+    return version

nvchecker_source/opam.py

@@ -0,0 +1,71 @@
+# MIT licensed
+# Copyright (c) 2024 Daniel Peukert <daniel@peukert.cc>, et al.
+
+import asyncio
+from io import BytesIO
+import tarfile
+from typing import List
+
+from nvchecker.api import (
+  session, VersionResult,
+  Entry, AsyncCache,
+  KeyManager, RichResult
+)
+
+OPAM_REPO_INDEX_URL = "%s/index.tar.gz"
+OPAM_VERSION_PATH_PREFIX = "packages/%s/%s."
+OPAM_VERSION_PATH_SUFFIX = "/opam"
+
+OPAM_DEFAULT_REPO = 'https://opam.ocaml.org'
+OPAM_DEFAULT_REPO_VERSION_URL = "%s/packages/%s/%s.%s"
+
+def _decompress_and_list_files(data: bytes) -> List[str]:
+  # Convert the bytes to a file object and get a list of files
+  archive = tarfile.open(mode='r', fileobj=BytesIO(data))
+  return archive.getnames()
+
+async def get_files(url: str) -> List[str]:
+  # Download the file and get its contents
+  res = await session.get(url)
+  data = res.body
+
+  # Get the file list of the archive
+  loop = asyncio.get_running_loop()
+  return await loop.run_in_executor(None, _decompress_and_list_files, data)
+
+async def get_package_versions(files: List[str], pkg: str) -> List[str]:
+  # Prepare the filename prefix based on the package name
+  prefix = OPAM_VERSION_PATH_PREFIX % (pkg , pkg)
+
+  # Only keep opam files that are relevant to the package we're working with
+  filtered_files = []
+
+  for filename in files:
+    if filename.startswith(prefix) and filename.endswith(OPAM_VERSION_PATH_SUFFIX):
+      filtered_files.append(filename[len(prefix):-1*len(OPAM_VERSION_PATH_SUFFIX)])
+
+  return filtered_files
+
+async def get_version(
+  name: str, conf: Entry, *,
+  cache: AsyncCache, keymanager: KeyManager,
+  **kwargs,
+):
+  pkg = conf.get('pkg', name)
+  repo = conf.get('repo', OPAM_DEFAULT_REPO).rstrip('/')
+
+  # Get the list of files in the repo index (see https://opam.ocaml.org/doc/Manual.html#Repositories for repo structure)
+  files = await cache.get(OPAM_REPO_INDEX_URL % repo, get_files) # type: ignore
+
+  # Parse the version strings from the file names
+  raw_versions = await get_package_versions(files, pkg)
+
+  # Convert the version strings into RichResults
+  versions = []
+  for version in raw_versions:
+    versions.append(RichResult(
+      version = version,
+      # There is no standardised URL scheme, so we only return an URL for the default registry
+      url = OPAM_DEFAULT_REPO_VERSION_URL % (repo, pkg, pkg, version) if repo == OPAM_DEFAULT_REPO else None,
+    ))
+  return versions

nvchecker_source/openvsx.py

@@ -1,6 +1,8 @@
 # MIT licensed
 # Copyright (c) 2013-2021 Th3Whit3Wolf <the.white.wolf.is.1337@gmail.com>, et al.
 
+from nvchecker.api import RichResult
+
 API_URL = 'https://open-vsx.org/api/%s/%s'
 
 async def get_version(name, conf, *, cache, **kwargs):
@@ -10,4 +12,7 @@ async def get_version(name, conf, *, cache, **kwargs):
   extension = splitName[1]
   data = await cache.get_json(API_URL % (publisher, extension))
   version = data['version']
-  return version
+  return RichResult(
+    version = version,
+    url = f'https://open-vsx.org/extension/{publisher}/{extension}/{version}',
+  )

nvchecker_source/packagist.py

@@ -1,6 +1,8 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+from nvchecker.api import RichResult
+
 PACKAGIST_URL = 'https://packagist.org/packages/%s.json'
 
 async def get_version(name, conf, *, cache, **kwargs):
@@ -14,4 +16,8 @@ async def get_version(name, conf, *, cache, **kwargs):
   }
 
   if len(versions):
-    return max(versions, key=lambda version: versions[version]["time"])
+    version = max(versions, key=lambda version: versions[version]["time"])
+    return RichResult(
+      version = version,
+      url = f'https://packagist.org/packages/{data["package"]["name"]}#{version}',
+    )

nvchecker_source/pacman.py

@@ -1,7 +1,7 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from nvchecker_source import cmd # type: ignore
+from nvchecker_source import cmd
 
 async def get_version(name, conf, **kwargs):
   referree = conf.get('pacman') or name

nvchecker_source/pagure.py

@@ -6,10 +6,10 @@ import urllib.parse
 import structlog
 
 from nvchecker.api import (
-  VersionResult, Entry, AsyncCache, KeyManager,
+  VersionResult, RichResult, Entry, AsyncCache, KeyManager,
 )
 
-PAGURE_URL = 'https://%s/api/0/%s/git/tags'
+PAGURE_URL = 'https://%s/api/0/%s/git/tags?with_commits=true'
 
 logger = structlog.get_logger(logger_name=__name__)
 
@@ -24,5 +24,9 @@ async def get_version(
   url = PAGURE_URL % (host, repo)
 
   data = await cache.get_json(url)
-  version = data["tags"]
-  return version
+  return [
+    RichResult(
+      version = version,
+      url = f'https://{host}/{repo}/tree/{version_hash}',
+    ) for version, version_hash in data["tags"].items()
+  ]

nvchecker_source/pypi.py

@@ -1,9 +1,16 @@
 # MIT licensed
-# Copyright (c) 2013-2021 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2021,2023-2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from packaging.version import Version
+import structlog
+from packaging.version import Version, InvalidVersion
+
+from nvchecker.api import RichResult
+
+logger = structlog.get_logger(logger_name=__name__)
 
 async def get_version(name, conf, *, cache, **kwargs):
+  ret = []
+
   package = conf.get('pypi') or name
   use_pre_release = conf.get('use_pre_release', False)
 
@@ -11,11 +18,26 @@ async def get_version(name, conf, *, cache, **kwargs):
 
   data = await cache.get_json(url)
 
-  if use_pre_release:
-    version = sorted(
-      data['releases'].keys(),
-      key = Version,
-    )[-1]
-  else:
-    version = data['info']['version']
-  return version
+  for version in data['releases'].keys():
+    # Skip versions that are marked as yanked.
+    if (vers := data['releases'][version]) and vers[0]['yanked']:
+      continue
+
+    try:
+      parsed_version = Version(version)
+    except InvalidVersion:
+      if data['releases'][version]:
+        # emit a warning if there is something under the invalid version
+        # sympy has an empty "0.5.13-hg" version
+        logger.warning('ignoring invalid version', version=version)
+      continue
+
+    if not use_pre_release and parsed_version.is_prerelease:
+      continue
+
+    ret.append(RichResult(
+      version = version,
+      url = f'https://pypi.org/project/{package}/{version}/',
+    ))
+
+  return ret

nvchecker_source/regex.py

@@ -2,35 +2,38 @@
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import re
-import sre_constants
 
 from nvchecker.api import session, GetVersionError
 
 async def get_version(name, conf, *, cache, **kwargs):
-  key = tuple(sorted(conf.items()))
-  return await cache.get(key, get_version_impl)
-
-async def get_version_impl(info):
-  conf = dict(info)
-
   try:
     regex = re.compile(conf['regex'])
-  except sre_constants.error as e:
+  except re.error as e:
     raise GetVersionError('bad regex', exc_info=e)
   if regex.groups > 1:
     raise GetVersionError('multi-group regex')
 
-  encoding = conf.get('encoding', 'latin1')
+  key = (
+    conf['url'],
+    conf.get('encoding', 'latin1'),
+    conf.get('post_data'),
+    conf.get('post_data_type', 'application/x-www-form-urlencoded'),
+  )
+  body = await cache.get(key, get_url)
 
-  data = conf.get('post_data')
-  if data is None:
-    res = await session.get(conf['url'])
-  else:
-    res = await session.post(conf['url'], body = data, headers = {
-        'Content-Type': conf.get('post_data_type', 'application/x-www-form-urlencoded')
-      })
-  body = res.body.decode(encoding)
   versions = regex.findall(body)
   if not versions and not conf.get('missing_ok', False):
     raise GetVersionError('version string not found.')
   return versions
+
+async def get_url(info):
+  url, encoding, post_data, post_data_type = info
+
+  if post_data is None:
+    res = await session.get(url)
+  else:
+    res = await session.post(url, body = post_data, headers = {
+      'Content-Type': post_data_type,
+    })
+  body = res.body.decode(encoding)
+  return body

nvchecker_source/repology.py

@@ -1,7 +1,7 @@
 # MIT licensed
 # Copyright (c) 2019 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from nvchecker.api import GetVersionError
+from nvchecker.api import RichResult, GetVersionError
 
 API_URL = 'https://repology.org/api/v1/project/{}'
 
@@ -25,5 +25,9 @@ async def get_version(name, conf, *, cache, **kwargs):
         raise GetVersionError('package is not found in subrepo',
                               repo=repo, subrepo=subrepo)
 
-  versions = [pkg['version'] for pkg in pkgs]
-  return versions
+  return [
+    RichResult(
+      version = pkg['version'],
+      url = f'https://repology.org/project/{project}/packages',
+    ) for pkg in pkgs
+  ]

nvchecker_source/rpmrepo.py

@@ -0,0 +1,84 @@
+# MIT licensed
+# Copyright (c) 2024 Jakub Ružička <jru@debian.org>, et al.
+
+import asyncio
+import gzip
+import pathlib
+import urllib
+from typing import Set
+
+import lxml.etree
+from nvchecker.api import session, AsyncCache, Entry, KeyManager, VersionResult
+
+
+# XML namespaces used in repodata (dead links haha)
+NS = {
+    'common': 'http://linux.duke.edu/metadata/common',
+    'repo':   'http://linux.duke.edu/metadata/repo',
+    'rpm':    'http://linux.duke.edu/metadata/rpm'
+}
+
+
+async def get_version(
+  name: str, conf: Entry, *,
+  cache: AsyncCache, keymanager: KeyManager,
+  **kwargs,
+) -> VersionResult:
+  repo = conf['repo']
+  arch = conf.get('arch', 'binary')
+  pkg = conf.get('pkg')
+  if not pkg:
+    pkg = conf.get('rpmrepo', name)
+
+  repo_url = urllib.parse.urlparse(repo)
+  repo_path = pathlib.PurePosixPath(repo_url.path)
+
+  # get the url of repomd.xml
+  repomd_path = repo_path / 'repodata' / 'repomd.xml'
+  repomd_url = repo_url._replace(path=str(repomd_path)).geturl()
+  # download repomd.xml (use cache)
+  repomd_body = await cache.get(repomd_url, get_file) # type: ignore
+  # parse repomd.xml
+  repomd_xml = lxml.etree.fromstring(repomd_body)
+
+  # get the url of *primary.xml.gz
+  primary_element = repomd_xml.find('repo:data[@type="primary"]/repo:location', namespaces=NS)
+  primary_path = repo_path / primary_element.get('href') # type: ignore
+  primary_url = repo_url._replace(path=str(primary_path)).geturl()
+  # download and decompress *primary.xml.gz (use cache)
+  primary_body = await cache.get(primary_url, get_file_gz) # type: ignore
+  # parse *primary.xml metadata
+  metadata = lxml.etree.fromstring(primary_body)
+
+  # use set to eliminate duplication
+  versions_set: Set[str] = set()
+  # iterate package metadata
+  for el in metadata.findall(f'common:package[common:name="{pkg}"]', namespaces=NS):
+    pkg_arch = el.findtext('common:arch', namespaces=NS)
+
+    # filter bych arch
+    if arch == 'binary':
+      if pkg_arch == 'src':
+        continue
+    elif arch != 'any':
+      if pkg_arch != arch:
+        continue
+
+    version_info = el.find('common:version', namespaces=NS)
+    version = version_info.get('ver') # type: ignore
+    versions_set.add(version) # type: ignore
+
+  versions = list(versions_set)
+  return versions # type: ignore
+
+
+async def get_file(url: str) -> bytes:
+  res = await session.get(url)
+  return res.body
+
+
+async def get_file_gz(url: str) -> bytes:
+  res = await session.get(url)
+  loop = asyncio.get_running_loop()
+  return await loop.run_in_executor(
+    None, gzip.decompress, res.body)

nvchecker_source/snapcraft.py

@@ -0,0 +1,33 @@
+# MIT licensed
+# Copyright (c) 2025 Maxim Slipenko <maxim@slipenko.com>, et al.
+
+from nvchecker.api import (
+  GetVersionError
+)
+from nvchecker.httpclient.base import HTTPError
+
+URL="https://api.snapcraft.io/v2/snaps/info/%(snap)s"
+
+async def get_version(
+  name: str, conf, *,
+  cache, keymanager,
+  **kwargs,
+):
+  try:
+    snap = conf.get("snap")
+    channel = conf.get("channel")
+
+    result = await cache.get_json(
+      URL % { "snap": snap },
+      headers={
+        "Snap-Device-Series": "16",
+      },
+    )
+  except HTTPError:
+    raise GetVersionError(f"Failed to request snap info for {snap}")
+  
+  for c in result['channel-map']:
+    if c['channel']['name'] == channel:
+      return c['version']
+  
+  raise GetVersionError(f"Failed to find version for {snap}")

nvchecker_source/sparkle.py

@@ -4,23 +4,25 @@
 
 from xml.etree import ElementTree
 
-from nvchecker.api import session
-
-NAMESPACE = 'http://www.andymatuschak.org/xml-namespaces/sparkle'
+from nvchecker.api import session, RichResult
 
+XML_NAMESPACE = 'http://www.w3.org/XML/1998/namespace'
+SPARKLE_NAMESPACE = 'http://www.andymatuschak.org/xml-namespaces/sparkle'
 
 async def get_version(name, conf, *, cache, **kwargs):
   sparkle = conf['sparkle']
-  return await cache.get(sparkle, get_version_impl)
+  release_notes_language = conf.get('release_notes_language', 'en')
+  return await cache.get((sparkle, release_notes_language), get_version_impl)
 
 
-async def get_version_impl(sparkle):
+async def get_version_impl(info):
+  sparkle, release_notes_language = info
   res = await session.get(sparkle)
-  root = ElementTree.fromstring(res.body)
-  item = root.find('./channel/item[1]/enclosure')
+  root = ElementTree.fromstring(res.body).find('./channel/item[1]')
+  item = root.find('./enclosure')
 
-  version_string = item.get(f'{{{NAMESPACE}}}shortVersionString')
-  build_number = item.get(f'{{{NAMESPACE}}}version')
+  version_string = item.get(f'{{{SPARKLE_NAMESPACE}}}shortVersionString')
+  build_number = item.get(f'{{{SPARKLE_NAMESPACE}}}version')
 
   if (version_string and version_string.isdigit()) and (
     build_number and not build_number.isdigit()
@@ -34,4 +36,25 @@ async def get_version_impl(sparkle):
   if build_number and (build_number not in version):
     version.append(build_number)
 
-  return '-'.join(version) if version else None
+  version_str = '-'.join(version) if version else None
+
+  release_notes_link = None
+  for release_notes in root.findall(f'./{{{SPARKLE_NAMESPACE}}}releaseNotesLink'):
+    language = release_notes.get(f'{{{XML_NAMESPACE}}}lang')
+
+    # If the release notes have no language set, store them, but keep looking for our preferred language
+    if language is None:
+      release_notes_link = release_notes.text.strip()
+
+    # If the release notes match our preferred language, store them and stop looking
+    if language == release_notes_language:
+      release_notes_link = release_notes.text.strip()
+      break
+
+  if release_notes_link is not None:
+    return RichResult(
+      version = version_str,
+      url = release_notes_link,
+    )
+  else:
+    return version_str

nvchecker_source/ubuntupkg.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 # Copyright (c) 2017 Felix Yan <felixonmars@archlinux.org>, et al.
 
-from nvchecker.api import GetVersionError
+from nvchecker.api import RichResult, GetVersionError
 
 URL = 'https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name=%s&exact_match=true'
 
@@ -42,4 +42,7 @@ async def get_version(name, conf, *, cache, **kwargs):
   else:
     version = releases[0]['source_package_version']
 
-  return version
+  return RichResult(
+    version = version,
+    url = f'https://packages.ubuntu.com/{releases[0]["distro_series_link"].rsplit("/", 1)[-1]}/{pkg}',
+  )

nvchecker_source/vsmarketplace.py

@@ -3,7 +3,7 @@
 
 from nvchecker.api import (
   VersionResult, Entry, AsyncCache, KeyManager,
-  TemporaryError, session, GetVersionError,
+  TemporaryError, session, RichResult, GetVersionError,
 )
 
 API_URL = 'https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery'
@@ -51,4 +51,7 @@ async def get_version(name: str, conf: Entry, *, cache: AsyncCache, **kwargs):
   j = res.json()
 
   version = j['results'][0]['extensions'][0]['versions'][0]['version']
-  return version
+  return RichResult(
+    version = version,
+    url = f'https://marketplace.visualstudio.com/items?itemName={name}',
+  )

pyproject.toml

@@ -0,0 +1,11 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+# addopts = -n auto
+asyncio_mode = "strict"
+asyncio_default_fixture_loop_scope = "session"
+
+# build and upload
+# rm -rf dist && python -m build --no-isolation && twine check dist/* && twine upload dist/*

sample_config.toml

@@ -2,11 +2,6 @@
 oldver = "old_ver.json"
 newver = "new_ver.json"
 
-[vim]
-source = "regex"
-regex = "7\\.3\\.\\d+"
-url = "http://ftp.vim.org/pub/vim/patches/7.3/"
-
 [google-chrome]
 source = "cmd"
 cmd = '''wget -qO- http://dl.google.com/linux/chrome/rpm/stable/x86_64/repodata/other.xml.gz | zgrep -A1 "google-chrome-stable" | awk -F\" '/version/ {print $4"-"$6}' '''
@@ -25,17 +20,13 @@ github = "lilydjwg/nvchecker"
 [ssed]
 source = "regex"
 regex = "The current version is ([\\d.]+)\\."
-url = "http://sed.sourceforge.net/grabbag/ssed/"
+url = "https://sed.sourceforge.net/grabbag/ssed/"
 proxy = "http://localhost:8087"
 
 [PySide]
 source = "pypi"
-pypi = "PySide"
+pypi = "nvchecker"
 
 [test]
 source = "manual"
 manual = "0.1"
-
-["Sparkle Test App"]
-source = "sparkle"
-sparkle = "https://sparkle-project.org/files/sparkletestcast.xml"

scripts/nvchecker-ini2toml

@@ -18,7 +18,7 @@ _handler_precedence = (
 BOOL_KEYS = [
   'strip_release', 'use_last_modified',
   'use_latest_release', 'use_latest_tag',
-  'use_max_tag', 'use_pre_release',
+  'use_max_release', 'use_max_tag', 'use_pre_release',
 ]
 
 INT_KEYS = [

scripts/nvchecker-notify

@@ -26,6 +26,11 @@ def get_args():
   parser.add_argument('-k', '--keyfile',
                       metavar='FILE', type=str,
                       help='use specified keyfile (override the one in configuration file)')
+  parser.add_argument('-t', '--tries', default=1, type=int, metavar='N',
+                      help='try N times when network errors occur')
+  parser.add_argument('--failures', action='store_true',
+                      help='exit with code 3 if failures / errors happen during checking')
+
   return parser.parse_args()
 
 def main():
@@ -43,6 +48,10 @@ def main():
     cmd.extend(['-c', args.file])
   if args.keyfile:
     cmd.extend(['-k', args.keyfile])
+  if args.tries:
+    cmd.extend(['-t', str(args.tries)])
+  if args.failures:
+    cmd.append('--failures')
 
   process = subprocess.Popen(cmd, pass_fds=(wfd,))
   os.close(wfd)

scripts/run_cached_tests

@@ -0,0 +1,40 @@
+#!/bin/bash -e
+
+mitmdump=${mitmdump:-mitmdump}
+
+if [[ -f ~/.mitmproxy/nvdump ]]; then
+  $mitmdump -S ~/.mitmproxy/nvdump -p 7890 --ignore-hosts '127\.0\.0\.1' --server-replay-reuse --server-replay-extra=forward -w newdump >mitmdump_output &
+else
+  $mitmdump -w ~/.mitmproxy/nvdump -p 7890 --ignore-hosts '127\.0\.0\.1' >mitmdump_output &
+fi
+
+mitm_pid=$!
+
+on_exit () {
+  kill -INT $mitm_pid
+
+  if [[ -s newdump ]]; then
+    cat newdump >> ~/.mitmproxy/nvdump
+  fi
+
+  cat mitmdump_output
+}
+
+trap on_exit EXIT
+
+if [[ -f keyfile.toml ]]; then 
+  export KEYFILE=keyfile.toml
+fi
+
+for _ in {1..10}; do
+  if [[ -s ~/.mitmproxy/mitmproxy-ca-cert.pem ]]; then
+    break
+  fi
+  sleep 1
+done
+
+export SSL_CERT_FILE=$HOME/.mitmproxy/mitmproxy-ca-cert.pem
+export GIT_SSL_CAINFO=$SSL_CERT_FILE
+export http_proxy=http://localhost:7890 https_proxy=http://localhost:7890
+
+pytest

setup.cfg

@@ -1,5 +1,79 @@
+# The complex upload command:
+# rm -rf dist && python -m build --sdist && twine check dist/* && twine upload -s dist/*
+
+[metadata]
+name = nvchecker
+version = attr: nvchecker.__version__
+author = lilydjwg
+author_email = lilydjwg@gmail.com
+description = New version checker for software
+license = MIT
+keywords = new, version, build, check
+url = https://github.com/lilydjwg/nvchecker
+long_description = file: README.rst
+long_description_content_type = text/x-rst
+platforms = any
+
+classifiers =
+  Development Status :: 5 - Production/Stable
+  Environment :: Console
+  Intended Audience :: Developers
+  Intended Audience :: System Administrators
+  License :: OSI Approved :: MIT License
+  Operating System :: OS Independent
+  Programming Language :: Python
+  Programming Language :: Python :: 3
+  Programming Language :: Python :: 3 :: Only
+  Programming Language :: Python :: 3.8
+  Programming Language :: Python :: 3.9
+  Programming Language :: Python :: 3.10
+  Programming Language :: Python :: 3.11
+  Programming Language :: Python :: 3.12
+  Programming Language :: Python :: 3.13
+  Topic :: Internet
+  Topic :: Internet :: WWW/HTTP
+  Topic :: Software Development
+  Topic :: System :: Archiving :: Packaging
+  Topic :: System :: Software Distribution
+  Topic :: Utilities
+
+[options]
+zip_safe = True
+python_requires = >=3.8
+
+packages = find_namespace:
+install_requires =
+  tomli; python_version<"3.11"
+  structlog
+  platformdirs
+  tornado>=6
+  pycurl
+scripts =
+  scripts/nvchecker-ini2toml
+  scripts/nvchecker-notify
+
+[options.packages.find]
+exclude = tests, build*, docs*
+
+[options.extras_require]
+vercmp =
+  pyalpm
+awesomeversion =
+  awesomeversion
+pypi =
+  packaging
+htmlparser =
+  lxml
+rpmrepo =
+  lxml
+jq =
+  jq
+
+[options.entry_points]
+console_scripts =
+  nvchecker = nvchecker.__main__:main
+  nvtake = nvchecker.tools:take
+  nvcmp = nvchecker.tools:cmp
+
 [flake8]
 ignore = E111, E302, E501
-
-[tool:pytest]
-addopts = --asyncio-mode=auto

setup.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-
-from setuptools import setup, find_namespace_packages
-import nvchecker
-
-# The complex upload command:
-# rm -rf dist && python setup.py sdist && twine check dist/* && twine upload -s dist/*
-
-setup(
-  name = 'nvchecker',
-  version = nvchecker.__version__,
-  author = 'lilydjwg',
-  author_email = 'lilydjwg@gmail.com',
-  description = 'New version checker for software',
-  license = 'MIT',
-  keywords = 'new version build check',
-  url = 'https://github.com/lilydjwg/nvchecker',
-  long_description = open('README.rst', encoding='utf-8').read(),
-  long_description_content_type = 'text/x-rst',
-  platforms = 'any',
-  zip_safe = True,
-
-  packages = find_namespace_packages(include=['nvchecker*']),
-  install_requires = ['setuptools; python_version<"3.8"', 'tomli', 'structlog', 'appdirs', 'tornado>=6', 'pycurl'],
-  extras_require = {
-    'vercmp': ['pyalpm'],
-    'awesomeversion': ['awesomeversion'],
-    'pypi': ['packaging'],
-    'htmlparser': ['lxml'],
-  },
-  tests_require = [
-    'pytest',
-    'pytest-asyncio',
-    'pytest-httpbin',
-    'flaky',
-  ],
-  entry_points = {
-    'console_scripts': [
-      'nvchecker = nvchecker.__main__:main',
-      'nvtake = nvchecker.tools:take',
-      'nvcmp = nvchecker.tools:cmp',
-    ],
-  },
-  scripts = [
-    'scripts/nvchecker-ini2toml',
-    'scripts/nvchecker-notify',
-  ],
-
-  classifiers = [
-    "Development Status :: 5 - Production/Stable",
-    "Environment :: Console",
-    "Intended Audience :: Developers",
-    "Intended Audience :: System Administrators",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
-    "Programming Language :: Python",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
-    "Topic :: Internet",
-    "Topic :: Internet :: WWW/HTTP",
-    "Topic :: Software Development",
-    "Topic :: System :: Archiving :: Packaging",
-    "Topic :: System :: Software Distribution",
-    "Topic :: Utilities",
-  ],
-)

tests/conftest.py

@@ -1,23 +1,32 @@
 # MIT licensed
-# Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2020, 2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import asyncio
 import structlog
 import os
 from pathlib import Path
+from typing import TYPE_CHECKING, Dict
+
+if TYPE_CHECKING:
+  import tomli as tomllib
+else:
+  try:
+    import tomllib
+  except ModuleNotFoundError:
+    import tomli as tomllib
 
-import tomli
 import pytest
+import pytest_asyncio
 
 from nvchecker import core
 from nvchecker import __main__ as main
-from nvchecker.util import Entries, VersData, RawResult
+from nvchecker.util import Entries, ResultData, RawResult
 
 use_keyfile = False
 
 async def run(
   entries: Entries, max_concurrency: int = 20,
-) -> VersData:
+) -> Dict[str, str]:
   task_sem = asyncio.Semaphore(max_concurrency)
   result_q: asyncio.Queue[RawResult] = asyncio.Queue()
   keyfile = os.environ.get('KEYFILE')
@@ -34,14 +43,14 @@ async def run(
     keymanager, entry_waiter, 1, {},
   )
 
-  oldvers: VersData = {}
+  oldvers: ResultData = {}
   result_coro = core.process_result(oldvers, result_q, entry_waiter)
   runner_coro = core.run_tasks(futures)
 
-  vers, _has_failures = await main.run(result_coro, runner_coro)
-  return vers
+  results, _has_failures = await main.run(result_coro, runner_coro)
+  return {k: r.version for k, r in results.items()}
 
-@pytest.fixture(scope="module")
+@pytest_asyncio.fixture(scope="session")
 async def get_version():
   async def __call__(name, config):
     entries = {name: config}
@@ -50,33 +59,24 @@ async def get_version():
 
   return __call__
 
-@pytest.fixture(scope="module")
+@pytest_asyncio.fixture(scope="session")
 async def run_str():
   async def __call__(str):
-    entries = tomli.loads(str)
+    entries = tomllib.loads(str)
     newvers = await run(entries)
     return newvers.popitem()[1]
 
   return __call__
 
-@pytest.fixture(scope="module")
+@pytest_asyncio.fixture(scope="session")
 async def run_str_multi():
   async def __call__(str):
-    entries = tomli.loads(str)
+    entries = tomllib.loads(str)
     newvers = await run(entries)
     return newvers
 
   return __call__
 
-loop = asyncio.new_event_loop()
-@pytest.fixture(scope="session")
-def event_loop(request):
-  """Override pytest-asyncio's event_loop fixture,
-     Don't create an instance of the default event loop for each test case.
-     We need the same ioloop across tests for the aiohttp support.
-  """
-  yield loop
-
 @pytest.fixture(scope="session", autouse=True)
 def raise_on_logger_msg():
   def proc(logger, method_name, event_dict):

tests/test_alpm.py

@@ -30,6 +30,7 @@ def setup_module(module):
       'pkgrel=4\n'
       'arch=(any)\n'
       'provides=("test-provides=5.6-7" "test-provides-unversioned")\n'
+      'options=(!debug)\n'
     )
   subprocess.check_call(['makepkg', '--nosign'], cwd=pkg_path)
   pkg_file = subprocess.check_output(['makepkg', '--packagelist'], cwd=pkg_path, text=True).strip()

tests/test_alpmfiles.py

@@ -0,0 +1,53 @@
+# MIT licensed
+# Copyright (c) 2023 Pekka Ristola <pekkarr [at] protonmail [dot] com>, et al.
+
+import pathlib
+import shutil
+import subprocess
+import tempfile
+
+import pytest
+
+pytestmark = [
+  pytest.mark.asyncio,
+  pytest.mark.skipif(shutil.which('pacman') is None, reason='requires pacman command'),
+  pytest.mark.skipif(shutil.which('fakeroot') is None, reason='requires fakeroot command'),
+]
+
+global temp_dir, db_path
+
+
+def setup_module(module):
+  global temp_dir, db_path
+
+  temp_dir = tempfile.TemporaryDirectory()
+  temp_path = pathlib.Path(temp_dir.name)
+  db_path = temp_path / 'test-db'
+
+  db_path.mkdir(exist_ok=True)
+
+  cmd = ['fakeroot', 'pacman', '-Fy', '--dbpath', db_path]
+  subprocess.check_call(cmd)
+
+
+def teardown_module(module):
+  temp_dir.cleanup()
+
+
+async def test_alpmfiles(get_version):
+  assert await get_version('test', {
+    'source': 'alpmfiles',
+    'pkgname': 'libuv',
+    'filename': 'usr/lib/libuv\\.so\\.([^.]+)',
+    'dbpath': db_path,
+  }) == '1'
+
+async def test_alpmfiles_strip(get_version):
+  assert await get_version('test', {
+    'source': 'alpmfiles',
+    'pkgname': 'glibc',
+    'repo': 'core',
+    'filename': 'libc\\.so\\.[^.]+',
+    'strip_dir': True,
+    'dbpath': db_path,
+  }) == 'libc.so.6'

tests/test_android_sdk.py

@@ -5,6 +5,7 @@
 import pytest
 pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 
+@pytest.mark.flaky(reruns=10)
 async def test_android_addon(get_version):
     assert await get_version("android-google-play-apk-expansion", {
         "source": "android_sdk",
@@ -13,11 +14,12 @@ async def test_android_addon(get_version):
     }) == "1.r03"
 
 async def test_android_package(get_version):
-    assert await get_version("android-sdk-cmake", {
+    version = await get_version("android-sdk-cmake", {
         "source": "android_sdk",
         "android_sdk": "cmake;",
         "repo": "package",
-    }) == "3.22.1"
+    })
+    assert version.startswith("3.")
 
 
 async def test_android_package_channel(get_version):
@@ -26,7 +28,7 @@ async def test_android_package_channel(get_version):
         "android_sdk": "ndk;",
         "repo": "package",
         "channel": "beta,dev,canary",
-    }) == "25.0.8528842"
+    }) == "26.0.10636728"
 
 async def test_android_list(get_version):
     assert await get_version("android-sdk-cmake-older", {
@@ -37,7 +39,7 @@ async def test_android_list(get_version):
     }) == "3.10.2"
 
 async def test_android_package_os(get_version):
-    await get_version("android-usb-driver", {
+    assert await get_version("android-usb-driver", {
         "source": "android_sdk",
         "android_sdk": "extras;google;usb_driver",
         "repo": "addon",
@@ -45,7 +47,7 @@ async def test_android_package_os(get_version):
     }) == "13"
 
 async def test_android_package_os_missing(get_version):
-    await get_version("android-usb-driver", {
+    assert await get_version("android-usb-driver", {
         "source": "android_sdk",
         "android_sdk": "extras;google;usb_driver",
         "repo": "addon",

tests/test_anitya.py

@@ -13,3 +13,10 @@ async def test_anitya(get_version):
     "anitya": "fedora/shutter",
   })
   assert re.match(r"[0-9.]+", version)
+
+async def test_anitya_by_id(get_version):
+  version = await get_version("shutter", {
+    "source": "anitya",
+    "anitya_id": "4813",
+  })
+  assert re.match(r"[0-9.]+", version)

tests/test_apt.py

@@ -2,19 +2,18 @@
 # Copyright (c) 2020-2021 lilydjwg <lilydjwg@gmail.com>, et al.
 # Copyright (c) 2017 Felix Yan <felixonmars@archlinux.org>, et al.
 
-from flaky import flaky
 import pytest
 pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_apt(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "apt",
         "mirror": "http://deb.debian.org/debian/",
         "suite": "sid",
-    }) == "0.1.7-1"
+    }) == "0.1.7-3"
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_apt_srcpkg(get_version):
     ver = await get_version("test", {
         "source": "apt",
@@ -24,7 +23,7 @@ async def test_apt_srcpkg(get_version):
     })
     assert ver.startswith("0.0~git20150829.56e4718-")
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_apt_strip_release(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "apt",
@@ -33,7 +32,8 @@ async def test_apt_strip_release(get_version):
         "strip_release": 1,
     }) == "0.1.7"
 
-@flaky(max_runs=10)
+@pytest.mark.skip
+@pytest.mark.flaky(reruns=10)
 async def test_apt_deepin(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "apt",
@@ -41,14 +41,3 @@ async def test_apt_deepin(get_version):
         "suite": "apricot",
     }) == "0.1.6-1"
 
-@flaky(max_runs=10)
-async def test_apt_multiversions(get_version):
-    ver = await get_version("ms-teams", {
-        "source": "apt",
-        "mirror": "https://packages.microsoft.com/repos/ms-teams",
-        "pkg": "teams",
-        "suite": "stable",
-        "repo": "main",
-        "arch": "amd64",
-    })
-    assert ver.startswith("1.5.00.")

tests/test_archpkg.py

@@ -1,35 +1,34 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from flaky import flaky
 import pytest
 pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 
-@flaky
+@pytest.mark.flaky
 async def test_archpkg(get_version):
     assert await get_version("base", {
         "source": "archpkg",
-    }) == "3-1"
+    }) == "3-2"
 
-@flaky
+@pytest.mark.flaky
 async def test_archpkg_strip_release(get_version):
     assert await get_version("base", {
         "source": "archpkg",
         "strip_release": True,
     }) == "3"
 
-@flaky
+@pytest.mark.flaky
 async def test_archpkg_provided(get_version):
     assert await get_version("dbus", {
         "source": "archpkg",
         "provided": "libdbus-1.so",
     }) == "3-64"
 
-@flaky
+@pytest.mark.flaky
 async def test_archpkg_provided_strip(get_version):
-    assert await get_version("jsoncpp", {
+    int(await get_version("jsoncpp", {
         "source": "archpkg",
         "provided": "libjsoncpp.so",
         "strip_release": True,
-    }) == "25"
+    }))
 

tests/test_aur.py

@@ -1,29 +1,24 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-import os
-
-from flaky import flaky
 import pytest
 pytestmark = [pytest.mark.asyncio,
-              pytest.mark.needs_net,
-              pytest.mark.skipif(os.environ.get('TRAVIS') == 'true',
-                                 reason="fail too often")]
+              pytest.mark.needs_net]
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_aur(get_version):
     assert await get_version("ssed", {
         "source": "aur",
     }) == "3.62-2"
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_aur_strip_release(get_version):
     assert await get_version("ssed", {
         "source": "aur",
         "strip_release": 1,
     }) == "3.62"
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_aur_use_last_modified(get_version):
     assert await get_version("ssed", {
         "source": "aur",

tests/test_container.py

@@ -1,8 +1,12 @@
 # MIT licensed
 # Copyright (c) 2020 Chih-Hsuan Yen <yan12125 at gmail dot com>
 
+import os
+import datetime
+
 import pytest
-pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net,
+             pytest.mark.skipif(bool(os.environ.get('GITHUB_RUN_ID')), reason="400 very often")]
 
 async def test_container(get_version):
   assert await get_version("hello-world", {
@@ -11,6 +15,31 @@ async def test_container(get_version):
     "include_regex": "linux",
   }) == "linux"
 
+async def test_container_with_tag(get_version):
+  update_time = await get_version("bitnami/mongodb:5.0", {
+    "source": "container",
+    "container": "bitnami/mongodb:5.0",
+  })
+  # the update time is changing occasionally, so we can not compare the exact time, otherwise the test will be failed in the future
+  assert datetime.date.fromisoformat(update_time.split('T')[0]) > datetime.date(2023, 12, 1)
+
+async def test_container_with_tag_and_multi_arch(get_version):
+  update_time = await get_version("hello-world:linux", {
+    "source": "container",
+    "container": "library/hello-world:linux",
+  })
+  # the update time is changing occasionally, so we can not compare the exact time, otherwise the test will be failed in the future
+  assert datetime.date.fromisoformat(update_time.split('T')[0]) > datetime.date(2023, 1, 1)
+
+async def test_container_with_tag_and_registry(get_version):
+  update_time = await get_version("hello-world-nginx:v1.0", {
+    "source": "container",
+    "registry": "quay.io",
+    "container": "redhattraining/hello-world-nginx:v1.0",
+  })
+  # the update time probably won't be changed
+  assert datetime.date.fromisoformat(update_time.split('T')[0]) == datetime.date(2019, 6, 26)
+
 async def test_container_paging(get_version):
   assert await get_version("prometheus-operator", {
     "source": "container",

tests/test_cran.py

@@ -7,4 +7,4 @@ pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 async def test_cran(get_version):
     assert await get_version("xml2", {
         "source": "cran",
-    }) == "1.3.3"
+    }) == "1.3.7"

tests/test_cratesio.py

@@ -7,4 +7,24 @@ pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 async def test_cratesio(get_version):
     assert await get_version("example", {
         "source": "cratesio",
-    }) == "0.1.0"
+    }) == "1.1.0"
+
+async def test_cratesio_list(get_version):
+    assert await get_version("example", {
+        "source": "cratesio",
+        "include_regex": r"^1\.0.*",
+    }) == "1.0.2"
+
+async def test_cratesio_skip_prerelease(get_version):
+    with pytest.raises(RuntimeError, match='include_regex matched no versions'):
+        await get_version("cargo-lock", {
+            "source": "cratesio",
+            "include_regex": r".*-.*",
+        })
+
+async def test_cratesio_use_prerelease(get_version):
+    await get_version("cargo-lock", {
+        "source": "cratesio",
+        "use_pre_release": "true",
+        "include_regex": r".*-.*",
+    })

tests/test_debianpkg.py

@@ -2,24 +2,23 @@
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 # Copyright (c) 2017 Felix Yan <felixonmars@archlinux.org>, et al.
 
-from flaky import flaky
 import pytest
 pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_debianpkg(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "debianpkg",
-    }) == "0.1.7-1"
+    }) == "0.1.7-3"
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_debianpkg_strip_release(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "debianpkg",
         "strip_release": 1,
     }) == "0.1.7"
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_debianpkg_suite(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "debianpkg",

tests/test_gitea.py

@@ -1,20 +1,19 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from flaky import flaky
 import pytest
 pytestmark = [pytest.mark.asyncio,
               pytest.mark.needs_net]
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_gitea(get_version):
     ver = await get_version("example", {
         "source": "gitea",
         "gitea": "gitea/tea"})
-    assert len(ver) == 8
-    assert ver.isdigit()
+    assert ver.startswith('20')
+    assert 'T' in ver
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_gitea_max_tag_with_include(get_version):
     assert await get_version("example", {
         "source": "gitea",

tests/test_github.py

@@ -24,9 +24,33 @@ async def test_github_default_not_master(get_version):
 async def test_github_latest_release(get_version):
     assert await get_version("example", {
         "source": "github",
-        "github": "harry-sanabria/ReleaseTestRepo",
+        "github": "dpeukert/ReleaseTestRepo",
         "use_latest_release": True,
-    }) == "release3"
+    }) == "v0.0.0"
+
+async def test_github_latest_release_include_prereleases(get_version):
+    assert await get_version("example", {
+        "source": "github",
+        "github": "dpeukert/ReleaseTestRepo",
+        "use_latest_release": True,
+        "include_prereleases": True,
+    }) == "v0.0.1-pre"
+
+async def test_github_latest_release_with_release_name(get_version):
+    version = await get_version("example", {
+        "source": "github",
+        "github": "mamba-org/mamba",
+        "use_latest_release": True,
+    })
+    assert version.startswith('20') # tag name
+
+    version = await get_version("example", {
+        "source": "github",
+        "github": "mamba-org/mamba",
+        "use_latest_release": True,
+        "use_release_name": True,
+    })
+    assert not version.startswith('20') # release name
 
 async def test_github_max_tag(get_version):
     assert await get_version("example", {
@@ -35,6 +59,20 @@ async def test_github_max_tag(get_version):
         "use_max_tag": True,
     }) == "second_release"
 
+async def test_github_max_release(get_version):
+    assert await get_version("example", {
+        "source": "github",
+        "github": "harry-sanabria/ReleaseTestRepo",
+        "use_max_release": True,
+    }) == "second_release"
+
+    assert await get_version("example", {
+        "source": "github",
+        "github": "harry-sanabria/ReleaseTestRepo",
+        "use_max_release": True,
+        "use_release_name": True,
+    }) == "second_release"
+
 async def test_github_max_tag_with_ignored(get_version):
     assert await get_version("example", {
         "source": "github",
@@ -43,6 +81,21 @@ async def test_github_max_tag_with_ignored(get_version):
         "ignored": "second_release release3",
     }) == "first_release"
 
+async def test_github_max_release_with_ignored(get_version):
+    assert await get_version("example", {
+        "source": "github",
+        "github": "harry-sanabria/ReleaseTestRepo",
+        "use_max_release": True,
+        "ignored": "second_release release3",
+    }) == "first_release"
+    assert await get_version("example", {
+        "source": "github",
+        "github": "harry-sanabria/ReleaseTestRepo",
+        "use_max_release": True,
+        "ignored": "second_release",
+        "use_release_name": True,
+    }) == "release #3"
+
 async def test_github_with_path(get_version):
     assert await get_version("example", {
         "source": "github",
@@ -67,6 +120,16 @@ async def test_github_max_tag_with_include(get_version):
     })
     assert re.match(r'chrome-[\d.]+', version)
 
+async def test_github_max_release_with_include(get_version):
+    version = await get_version("example", {
+        "source": "github",
+        "github": "EFForg/https-everywhere",
+        "use_max_release": True,
+        "use_release_name": True,
+        "include_regex": r"Release \d.*",
+    })
+    assert re.match(r'Release [\d.]+', version)
+
 async def test_github_latest_tag(get_version):
     assert await get_version("example", {
         "source": "github",

tests/test_go.py

@@ -0,0 +1,38 @@
+# MIT licensed
+# Copyright (c) 2024 bgme <i@bgme.me>.
+
+import pytest
+
+from nvchecker.api import HTTPError
+
+try:
+  import lxml
+  lxml_available = True
+except ImportError:
+  lxml_available = False
+
+pytestmark = [
+  pytest.mark.asyncio,
+  pytest.mark.needs_net,
+  pytest.mark.skipif(not lxml_available, reason="needs lxml")
+]
+
+
+async def test_go(get_version):
+  ver = await get_version("one version", {
+    "source": "go",
+    "go": "github.com/caddyserver/replace-response",
+  })
+
+  assert ver.startswith("v0.0.0-")
+
+  assert await get_version("multiple version", {
+    "source": "go",
+    "go": "github.com/corazawaf/coraza-caddy",
+  }) == "v1.2.2"
+
+  with pytest.raises(HTTPError):
+    await get_version("not found", {
+      "source": "go",
+      "go": "github.com/asdas/sadfasdf",
+    })

tests/test_hackage.py

@@ -1,11 +1,10 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from flaky import flaky
 import pytest
 pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 
-@flaky(max_runs=10)
+@pytest.mark.flaky(reruns=10)
 async def test_hackage(get_version):
     assert await get_version("sessions", {
         "source": "hackage",

tests/test_htmlparser.py

@@ -0,0 +1,35 @@
+# MIT licensed
+# Copyright (c) 2021 ypsilik <tt2laurent.maud@gmail.com>, et al.
+
+import pytest
+
+lxml_available = True
+try:
+  import lxml
+except ImportError:
+  lxml_available = False
+
+pytestmark = [
+  pytest.mark.asyncio,
+  pytest.mark.needs_net,
+  pytest.mark.skipif(not lxml_available, reason="needs lxml"),
+]
+
+async def test_xpath_ok(get_version):
+    ver = await get_version("aur", {
+        "source": "htmlparser",
+        "url": "https://aur.archlinux.org/",
+        "xpath": '//div[@id="footer"]/p[1]/a/text()',
+    })
+    assert ver.startswith('v')
+    assert '.' in ver
+
+async def test_xpath_element(get_version):
+    ver = await get_version("aur", {
+        "source": "htmlparser",
+        "url": "https://aur.archlinux.org/",
+        "xpath": '//div[@id="footer"]/p[1]/a',
+    })
+    assert ver.startswith('v')
+    assert '.' in ver
+

tests/test_htmlpasrer.py

@@ -1,29 +0,0 @@
-# MIT licensed
-# Copyright (c) 2021 ypsilik <tt2laurent.maud@gmail.com>, et al.
-
-import pytest
-
-pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
-
-async def test_xpath_ok(get_version):
-    assert await get_version("unifiedremote", {
-        "source": "htmlparser",
-        "url": "http://httpbin.org/",
-        "xpath": '//pre[@class="version"]/text()',
-    }) != None
-
-async def test_xpath_missing_ok(get_version):
-    assert await get_version("unifiedremote", {
-        "source": "htmlparser",
-        "url": "http://httpbin.org/",
-        "xpath": '//pre[@class="test-is-ok"]/text()',
-        "missing_ok": True,
-    }) is None
-
-async def test_xpath_element(get_version):
-    assert await get_version("unifiedremote", {
-        "source": "htmlparser",
-        "url": "http://httpbin.org/",
-        "xpath": '//pre[@class="version"]',
-    }) != None
-

tests/test_httpheader.py

@@ -1,24 +1,31 @@
 # MIT licensed
-# Copyright (c) 2021 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2021,2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import pytest
-import pytest_httpbin
-assert pytest_httpbin # for pyflakes
 
-pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+httpbin_available = True
+try:
+  import pytest_httpbin
+  assert pytest_httpbin # for pyflakes
+except ImportError:
+  httpbin_available = False
 
+pytestmark = pytest.mark.asyncio
+
+@pytest.mark.needs_net
 async def test_redirection(get_version):
-    assert await get_version("unifiedremote", {
-        "source": "httpheader",
-        "url": "https://www.unifiedremote.com/download/linux-x64-deb",
-        "regex": r'urserver-([\d.]+).deb',
-    }) is not None
+  assert await get_version("unifiedremote", {
+    "source": "httpheader",
+    "url": "https://www.unifiedremote.com/download/linux-x64-deb",
+    "regex": r'urserver-([\d.]+).deb',
+  }) is not None
 
+@pytest.mark.skipif(not httpbin_available, reason="needs pytest_httpbin")
 async def test_get_version_withtoken(get_version, httpbin):
-    assert await get_version("unifiedremote", {
-        "source": "httpheader",
-        "url": httpbin.url + "/basic-auth/username/superpassword",
-        "httptoken": "Basic dXNlcm5hbWU6c3VwZXJwYXNzd29yZA==",
-        "header": "server",
-        "regex": r'([0-9.]+)*',
-    }) is not None
+  assert await get_version("unifiedremote", {
+    "source": "httpheader",
+    "url": httpbin.url + "/basic-auth/username/superpassword",
+    "httptoken": "Basic dXNlcm5hbWU6c3VwZXJwYXNzd29yZA==",
+    "header": "server",
+    "regex": r'([0-9.]+)*',
+  }) is not None

tests/test_jq.py

@@ -0,0 +1,33 @@
+# MIT licensed
+# Copyright (c) 2024 Rocket Aaron <i@rocka.me>, et al.
+
+import pytest
+
+jq_available = True
+try:
+  import jq
+except ImportError:
+  jq_available = False
+
+pytestmark = [
+  pytest.mark.asyncio,
+  pytest.mark.needs_net,
+  pytest.mark.skipif(not jq_available, reason="needs jq"),
+]
+
+async def test_jq(get_version):
+    ver = await get_version("aur", {
+        "source": "jq",
+        "url": "https://aur.archlinux.org/rpc/v5/info?arg[]=nvchecker-git"
+    })
+    ver = ver.strip()
+    assert ver.startswith("{")
+    assert ver.endswith("}")
+
+async def test_jq_filter(get_version):
+    ver = await get_version("aur", {
+        "source": "jq",
+        "url": "https://aur.archlinux.org/rpc/v5/info?arg[]=nvchecker-git",
+        "filter": '.results[0].PackageBase',
+    })
+    assert ver == "nvchecker-git"

tests/test_launchpad.py

@@ -0,0 +1,16 @@
+# MIT Licensed
+# Copyright (c) 2024 Bert Peters <bertptrs@archlinux.org>, et al.
+import pytest
+pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+
+@pytest.mark.flaky(reruns=10)
+async def test_launchpad(get_version):
+  version = await get_version(
+    "sakura",
+    {
+      "source": "launchpad",
+      "launchpad": "sakura",
+    }
+  )
+
+  assert version == '3.8.8'

tests/test_mercurial.py

@@ -0,0 +1,15 @@
+# MIT licensed
+# Copyright (c) 2020 Felix Yan <felixonmars@archlinux.org>, et al.
+
+import pytest
+pytestmark = [
+  pytest.mark.asyncio,
+  pytest.mark.needs_net,
+]
+
+@pytest.mark.skip
+async def test_mercurial(get_version):
+  assert await get_version("example", {
+    "source": "mercurial",
+    "mercurial": "https://repo.mercurial-scm.org/hg-website/json-tags",
+  }) == "v1.0"

tests/test_opam.py

@@ -0,0 +1,25 @@
+# MIT licensed
+# Copyright (c) 2024 Daniel Peukert <daniel@peukert.cc>, et al.
+
+import pytest
+pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+
+async def test_opam_official(get_version):
+    assert await get_version("test", {
+        "source": "opam",
+        "pkg": "omigrate",
+    }) == "0.3.2"
+
+async def test_opam_coq(get_version):
+    assert await get_version("test", {
+        "source": "opam",
+        "repo": "https://coq.inria.fr/opam/released",
+        "pkg": "coq-abp",
+    }) == "8.10.0"
+
+async def test_opam_coq_trailing_slash(get_version):
+    assert await get_version("test", {
+        "source": "opam",
+        "repo": "https://coq.inria.fr/opam/released/",
+        "pkg": "coq-abp",
+    }) == "8.10.0"

tests/test_pacman.py

@@ -13,7 +13,7 @@ pytestmark = [pytest.mark.asyncio,
 async def test_pacman(get_version):
     assert await get_version("base", {
         "source": "pacman",
-    }) == "3-1"
+    }) == "3-2"
 
 async def test_pacman_strip_release(get_version):
     assert await get_version("base", {

tests/test_pypi.py

@@ -20,3 +20,20 @@ async def test_pypi_pre_release(get_version):
         "source": "pypi",
         "use_pre_release": 1,
     }) == "1.0.1a1"
+
+async def test_pypi_list(get_version):
+    assert await get_version("urllib3", {
+        "source": "pypi",
+        "include_regex": "^1\\..*",
+    }) == "1.26.20"
+
+async def test_pypi_invalid_version(get_version):
+    await get_version("sympy", {
+        "source": "pypi",
+    })
+
+async def test_pypi_yanked_version(get_version):
+    assert await get_version("urllib3", {
+        "source": "pypi",
+        "include_regex": "^(1\\..*)|(2\\.0\\.[0,1])",
+    }) == "1.26.20"

tests/test_regex.py

@@ -1,130 +1,138 @@
 # MIT licensed
-# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020,2024 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import base64
 
 import pytest
-import pytest_httpbin
-assert pytest_httpbin # for pyflakes
 
-pytestmark = pytest.mark.asyncio
+httpbin_available = True
+try:
+  import pytest_httpbin
+  assert pytest_httpbin # for pyflakes
+except ImportError:
+  httpbin_available = False
+
+pytestmark = [
+  pytest.mark.asyncio,
+  pytest.mark.skipif(not httpbin_available, reason="needs pytest_httpbin"),
+]
 
 def base64_encode(s):
-    return base64.b64encode(s.encode('utf-8')).decode('ascii')
+  return base64.b64encode(s.encode('utf-8')).decode('ascii')
 
 async def test_regex_httpbin_default_user_agent(get_version, httpbin):
-    ua = await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/get",
-        "regex": r'"User-Agent":\s*"([^"]+)"',
-    })
-    assert ua.startswith("lilydjwg/nvchecker")
+  ua = await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/get",
+    "regex": r'"User-Agent":\s*"([^"]+)"',
+  })
+  assert ua.startswith("lilydjwg/nvchecker")
 
 async def test_regex_httpbin_user_agent(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/get",
-        "regex": r'"User-Agent":\s*"(\w+)"',
-        "user_agent": "Meow",
-    }) == "Meow"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/get",
+    "regex": r'"User-Agent":\s*"(\w+)"',
+    "user_agent": "Meow",
+  }) == "Meow"
 
 async def test_regex(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/base64/" + base64_encode("version 1.12 released"),
-        "regex": r'version ([0-9.]+)',
-    }) == "1.12"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/base64/" + base64_encode("version 1.12 released"),
+    "regex": r'version ([0-9.]+)',
+  }) == "1.12"
 
 async def test_missing_ok(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/base64/" + base64_encode("something not there"),
-        "regex": "foobar",
-        "missing_ok": True,
-    }) is None
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/base64/" + base64_encode("something not there"),
+    "regex": "foobar",
+    "missing_ok": True,
+  }) is None
 
 async def test_missing(get_version, httpbin):
-    with pytest.raises(RuntimeError):
-      await get_version("example", {
-          "source": "regex",
-          "url": httpbin.url + "/base64/" + base64_encode("something not there"),
-          "regex": "foobar",
-      })
+  with pytest.raises(RuntimeError):
+    await get_version("example", {
+      "source": "regex",
+      "url": httpbin.url + "/base64/" + base64_encode("something not there"),
+      "regex": "foobar",
+    })
 
 async def test_multi_group(get_version, httpbin):
-    with pytest.raises(RuntimeError):
-      await get_version("example", {
-          "source": "regex",
-          "url": httpbin.url + "/base64/" + base64_encode("1.2"),
-          "regex": r"(\d+)\.(\d+)",
-      })
+  with pytest.raises(RuntimeError):
+    await get_version("example", {
+      "source": "regex",
+      "url": httpbin.url + "/base64/" + base64_encode("1.2"),
+      "regex": r"(\d+)\.(\d+)",
+    })
 
 async def test_regex_with_tokenBasic(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/basic-auth/username/superpassword",
-        "httptoken": "Basic dXNlcm5hbWU6c3VwZXJwYXNzd29yZA==",
-        "regex": r'"user":"([a-w]+)"',
-    }) == "username"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/basic-auth/username/superpassword",
+    "httptoken": "Basic dXNlcm5hbWU6c3VwZXJwYXNzd29yZA==",
+    "regex": r'"user":\s*"([a-w]+)"',
+  }) == "username"
 
 async def test_regex_with_tokenBearer(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/bearer",
-        "httptoken": "Bearer username:password",
-        "regex": r'"token":"([a-w]+):.*"',
-    }) == "username"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/bearer",
+    "httptoken": "Bearer username:password",
+    "regex": r'"token":\s*"([a-w]+):.*"',
+  }) == "username"
 
 async def test_regex_no_verify_ssl(get_version, httpbin_secure):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin_secure.url + "/base64/" + base64_encode("version 1.12 released"),
-        "regex": r'version ([0-9.]+)',
-        "verify_cert": False,
-    }) == "1.12"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin_secure.url + "/base64/" + base64_encode("version 1.12 released"),
+    "regex": r'version ([0-9.]+)',
+    "verify_cert": False,
+  }) == "1.12"
 
 async def test_regex_bad_ssl(get_version, httpbin_secure):
-    try:
-        await get_version("example", {
-            "source": "regex",
-            "url": httpbin_secure.url + "/base64/" + base64_encode("version 1.12 released"),
-            "regex": r'version ([0-9.]+)',
-        })
-    except Exception:
-        pass
-    else:
-        assert False, 'certificate should not be trusted'
+  try:
+    await get_version("example", {
+      "source": "regex",
+      "url": httpbin_secure.url + "/base64/" + base64_encode("version 1.12 released"),
+      "regex": r'version ([0-9.]+)',
+    })
+  except Exception:
+    pass
+  else:
+    assert False, 'certificate should not be trusted'
 
 async def test_regex_post(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/post",
-        "regex": r'"ABCDEF":\s*"(\w+)"',
-        "post_data": "ABCDEF=234&CDEFG=xyz"
-    }) == "234"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/post",
+    "regex": r'"ABCDEF":\s*"(\w+)"',
+    "post_data": "ABCDEF=234&CDEFG=xyz"
+  }) == "234"
 
 async def test_regex_post2(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/post",
-        "regex": r'"CDEFG":\s*"(\w+)"',
-        "post_data": "ABCDEF=234&CDEFG=xyz"
-    }) == "xyz"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/post",
+    "regex": r'"CDEFG":\s*"(\w+)"',
+    "post_data": "ABCDEF=234&CDEFG=xyz"
+  }) == "xyz"
 
 async def test_regex_post_json(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/post",
-        "regex": r'"ABCDEF":\s*(\w+)',
-        "post_data": '{"ABCDEF":234,"CDEFG":"xyz"}',
-        "post_data_type": "application/json"
-    }) == "234"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/post",
+    "regex": r'"ABCDEF":\s*(\w+)',
+    "post_data": '{"ABCDEF":234,"CDEFG":"xyz"}',
+    "post_data_type": "application/json"
+  }) == "234"
 
 async def test_regex_post_json2(get_version, httpbin):
-    assert await get_version("example", {
-        "source": "regex",
-        "url": httpbin.url + "/post",
-        "regex": r'"CDEFG":\s*"(\w+)"',
-        "post_data": '{"ABCDEF":234,"CDEFG":"xyz"}',
-        "post_data_type": "application/json"
-    }) == "xyz"
+  assert await get_version("example", {
+    "source": "regex",
+    "url": httpbin.url + "/post",
+    "regex": r'"CDEFG":\s*"(\w+)"',
+    "post_data": '{"ABCDEF":234,"CDEFG":"xyz"}',
+    "post_data_type": "application/json"
+  }) == "xyz"

tests/test_repology.py

@@ -5,12 +5,14 @@ import pytest
 pytestmark = [pytest.mark.asyncio,
               pytest.mark.needs_net]
 
+@pytest.mark.flaky(reruns=10)
 async def test_repology(get_version):
   assert await get_version("ssed", {
         "source": "repology",
         "repo": "aur",
   }) == "3.62"
 
+@pytest.mark.flaky(reruns=10)
 async def test_repology_subrepo(get_version):
   assert await get_version("asciiquarium", {
         "source": "repology",

tests/test_rpmrepo.py

@@ -0,0 +1,19 @@
+# MIT licensed
+# Copyright (c) 2024 Jakub Ružička <jru@debian.org>, et al.
+
+import pytest
+pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+
+async def test_rpmrepo_fedora(get_version):
+    assert await get_version("knot_fedora-39", {
+        "source": "rpmrepo",
+        "pkg": "knot",
+        "repo": "http://ftp.sh.cvut.cz/fedora/linux/updates/39/Everything/x86_64/",
+    }) == "3.3.9"
+
+async def test_rpmrepo_alma(get_version):
+    assert await get_version("knot_fedora-39", {
+        "source": "rpmrepo",
+        "pkg": "tmux",
+        "repo": "http://ftp.sh.cvut.cz/almalinux/9.5/BaseOS/x86_64/os/",
+    }) == "3.2a"

tests/test_snapcraft.py

@@ -0,0 +1,28 @@
+# MIT licensed
+# Copyright (c) 2025 Maxim Slipenko <maxim@slipenko.com>, et al.
+
+import pytest
+pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+
+async def test_snapcraft(get_version):
+    assert await get_version("test", {
+        "source": "snapcraft",
+        "snap": "test-snapd-public",
+        "channel": "edge",
+    }) == "2.0"
+
+async def test_snapcraft_non_existent_snap(get_version):
+    with pytest.raises(RuntimeError, match='Failed to request snap info for not-existent-snap'):
+        assert await get_version("test", {
+            "source": "snapcraft",
+            "snap": "not-existent-snap",
+            "channel": "stable",
+        })
+
+async def test_snapcraft_non_existent_channel(get_version):
+    with pytest.raises(RuntimeError, match='Failed to find version for test-snapd-public'):
+        assert await get_version("test", {
+            "source": "snapcraft",
+            "snap": "test-snapd-public",
+            "channel": "non-existent-channel",
+        })

tests/test_substitute.py

@@ -45,8 +45,8 @@ async def test_substitute_regex_empty_to_pattern(get_version):
 async def test_substitute_prefix_has_higher_priority(get_version):
     assert await get_version("example", {
         "source": "manual",
-        "manual": "r15",
-        "prefix": "r",
-        "from_pattern": r"r(\d+)",
-        "to_pattern": r"R\1",
-    }) == "15"
+        "manual": "Version 1.2 Beta 3",
+        "prefix": "Version ",
+        "from_pattern": r" Beta ",
+        "to_pattern": r"b",
+    }) == "1.2b3"

tests/test_ubuntupkg.py

@@ -1,32 +1,32 @@
 # MIT licensed
-# Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2020,2024 lilydjwg <lilydjwg@gmail.com>, et al.
 # Copyright (c) 2017 Felix Yan <felixonmars@archlinux.org>, et al.
 
-from flaky import flaky
 import pytest
 pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 
-@flaky
+@pytest.mark.flaky
 async def test_ubuntupkg(get_version):
-    assert await get_version("sigrok-firmware-fx2lafw", {
+    v = await get_version("sigrok-firmware-fx2lafw", {
         "source": "ubuntupkg",
-    }) == "0.1.7-1"
+    })
+    assert v.startswith("0.1.7-")
 
-@flaky
+@pytest.mark.flaky
 async def test_ubuntupkg_strip_release(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "ubuntupkg",
         "strip_release": True,
     }) == "0.1.7"
 
-@flaky
+@pytest.mark.flaky
 async def test_ubuntupkg_suite(get_version):
     assert await get_version("sigrok-firmware-fx2lafw", {
         "source": "ubuntupkg",
         "suite": "xenial",
     }) == "0.1.2-1"
 
-@flaky
+@pytest.mark.flaky(reruns=10)
 async def test_ubuntupkg_suite_with_paging(get_version):
     assert await get_version("ffmpeg", {
         "source": "ubuntupkg",

tests/test_vsmarketplace.py

@@ -7,4 +7,4 @@ pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
 async def test_vsmarketplace(get_version):
     assert await get_version("usernamehw.indent-one-space", {
         "source": "vsmarketplace",
-    }) == "0.3.0"
+    }) == "1.0.0"

tox.ini

@@ -0,0 +1,16 @@
+[tox]
+isolated_build = True
+# you may find `tox --skip-missing-interpreters=true` helpful.
+envlist = py3{8,9,10,11,12}
+
+[testenv]
+usedevelop = false
+deps =
+  pytest
+  pytest-asyncio
+  pytest-httpbin
+  flaky
+extras =
+  htmlparser
+passenv = KEYFILE
+commands = pytest -r fEs {posargs}