From 8b929322cb3e92d15e00cd736b7e256f1e447f01 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 10:55:19 -0800 Subject: [PATCH 1/9] defensively fix reStructuredText syntax a ` : ` (space, colon, space) like that can be confused for a definition list classifier, especially in a definition list, so prefer to backslash-escape the colon. see the "reStructedText Markup Specification", section "Syntax Details" > "Body Elements" > "Definition Lists". --- docs/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 79b4c5b..c5ea879 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -204,8 +204,8 @@ httptoken A personal authorization token used to fetch the url with the ``Authorization`` header. The type of token depends on the authorization required. - - For Bearer token set : ``Bearer `` - - For Basic token set : ``Basic `` + - For Bearer token set \: ``Bearer `` + - For Basic token set \: ``Basic `` In the keyfile add ``httptoken_{name}`` token. From 15020dfcd606bbf2067b5bf4104a56ff49fdadc2 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 12:57:47 -0800 Subject: [PATCH 2/9] switch from setup.py to PEP 517 & setup.cfg --- .github/workflows/tests.yaml | 2 +- .gitignore | 1 + README.rst | 2 +- pyproject.toml | 6 +++ setup.cfg | 72 ++++++++++++++++++++++++++++++++++-- setup.py | 69 ---------------------------------- 6 files changed, 78 insertions(+), 74 deletions(-) create mode 100644 pyproject.toml delete mode 100755 setup.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 97304a1..f7998b6 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -34,7 +34,7 @@ jobs: cache-name: cache-pip with: path: ~/.cache/pip - key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.deps }}-${{ hashFiles('setup.py') }} + key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.deps }}-${{ hashFiles('pyproject.toml', 'setup.cfg') }} restore-keys: | ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.deps }}- ${{ runner.os }}-${{ env.cache-name }}- diff --git a/.gitignore b/.gitignore index dda2b4b..b841bb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.egg-info/ __pycache__/ /build/ +/dist/ .cache/ .eggs/ *.pyc diff --git a/README.rst b/README.rst index 4ce6ee3..08ef28f 100644 --- a/README.rst +++ b/README.rst @@ -42,7 +42,7 @@ To install:: To use the latest code, you can also clone this repository and run:: - python3 setup.py install + pip install . To see available options:: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..aa7f987 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +# addopts = -n auto diff --git a/setup.cfg b/setup.cfg index 08b89b4..343d3e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,71 @@ +# The complex upload command: +# rm -rf dist && python -m build --sdist && twine check dist/* && twine upload -s dist/* + +[metadata] +name = nvchecker +version = attr: nvchecker.__version__ +author = lilydjwg +author_email = lilydjwg@gmail.com +description = New version checker for software +license = MIT +keywords = new, version, build, check +url = https://github.com/lilydjwg/nvchecker +long_description = file: README.rst +long_description_content_type = text/x-rst +platforms = any + +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Intended Audience :: Developers + Intended Audience :: System Administrators + License :: OSI Approved :: MIT License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Topic :: Internet + Topic :: Internet :: WWW/HTTP + Topic :: Software Development + Topic :: System :: Archiving :: Packaging + Topic :: System :: Software Distribution + Topic :: Utilities + +[options] +zip_safe = True + +packages = find_namespace: +install_requires = + setuptools; python_version<"3.8" + tomli + structlog + appdirs + tornado>=6 + pycurl +scripts = + scripts/nvchecker-ini2toml + scripts/nvchecker-notify + +[options.packages.find] +exclude = tests, build*, docs* + +[options.extras_require] +vercmp = + pyalpm +pypi = + packaging +htmlparser = + lxml + +[options.entry_points] +console_scripts = + nvchecker = nvchecker.__main__:main + nvtake = nvchecker.tools:take + nvcmp = nvchecker.tools:cmp + [flake8] ignore = E111, E302, E501 - -[tool:pytest] -# addopts = -n auto diff --git a/setup.py b/setup.py deleted file mode 100755 index 7848538..0000000 --- a/setup.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 - -from setuptools import setup, find_namespace_packages -import nvchecker - -# The complex upload command: -# rm -rf dist && python setup.py sdist && twine check dist/* && twine upload -s dist/* - -setup( - name = 'nvchecker', - version = nvchecker.__version__, - author = 'lilydjwg', - author_email = 'lilydjwg@gmail.com', - description = 'New version checker for software', - license = 'MIT', - keywords = 'new version build check', - url = 'https://github.com/lilydjwg/nvchecker', - long_description = open('README.rst', encoding='utf-8').read(), - long_description_content_type = 'text/x-rst', - platforms = 'any', - zip_safe = True, - - packages = find_namespace_packages(exclude=['tests', 'build*', 'docs*']), - install_requires = ['setuptools; python_version<"3.8"', 'tomli', 'structlog', 'appdirs', 'tornado>=6', 'pycurl'], - extras_require = { - 'vercmp': ['pyalpm'], - 'pypi': ['packaging'], - 'htmlparser': ['lxml'], - }, - tests_require = [ - 'pytest', - 'pytest-asyncio', - 'pytest-httpbin', - 'flaky', - ], - entry_points = { - 'console_scripts': [ - 'nvchecker = nvchecker.__main__:main', - 'nvtake = nvchecker.tools:take', - 'nvcmp = nvchecker.tools:cmp', - ], - }, - scripts = [ - 'scripts/nvchecker-ini2toml', - 'scripts/nvchecker-notify', - ], - - classifiers = [ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Internet", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development", - "Topic :: System :: Archiving :: Packaging", - "Topic :: System :: Software Distribution", - "Topic :: Utilities", - ], -) From a62866a2d3e4c0a66fcccf0113be4719ae840717 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 14:07:12 -0800 Subject: [PATCH 3/9] tests: spell test_htmlparser correctly --- tests/{test_htmlpasrer.py => test_htmlparser.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_htmlpasrer.py => test_htmlparser.py} (100%) diff --git a/tests/test_htmlpasrer.py b/tests/test_htmlparser.py similarity index 100% rename from tests/test_htmlpasrer.py rename to tests/test_htmlparser.py From 7cf3b06c7cca0fd6f9f434736357f98c59350bb4 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 13:01:27 -0800 Subject: [PATCH 4/9] introduce tox support --- .gitignore | 1 + pyproject.toml | 20 ++++++++++++++++++++ tests/test_htmlparser.py | 12 +++++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b841bb7..4857e9f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ __pycache__/ *.pyo .travis.pub .pytest_cache/ +.tox/ keyfile.toml diff --git a/pyproject.toml b/pyproject.toml index aa7f987..3f7a142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,23 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] # addopts = -n auto + +[tool.tox] +legacy_tox_ini = """ +[tox] +isolated_build = True +# you may find `tox --skip-missing-interpreters=true` helpful. +envlist = py3{7,8,9,10} + +[testenv] +usedevelop = false +deps = + pytest + pytest-asyncio + pytest-httpbin + flaky +extras = + htmlparser +passenv = KEYFILE +commands = pytest -r fEs {posargs} +""" diff --git a/tests/test_htmlparser.py b/tests/test_htmlparser.py index f3be52b..79ccd7e 100644 --- a/tests/test_htmlparser.py +++ b/tests/test_htmlparser.py @@ -3,7 +3,17 @@ import pytest -pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net] +lxml_available = True +try: + import lxml +except ImportError: + lxml_available = False + +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.needs_net, + pytest.mark.skipif(not lxml_available, reason="needs lxml"), +] async def test_xpath_ok(get_version): assert await get_version("unifiedremote", { From 45b5593f26eb605369fe0d15f788855bf45557e0 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 14:28:23 -0800 Subject: [PATCH 5/9] tests: pytest-asyncio `asyncio_mode` -> `strict` the `asyncio_mode` of `legacy` (current default) is deprecated. `asyncio_mode` will be `strict` by default in the future. --- pyproject.toml | 1 + tests/conftest.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3f7a142..2a7c198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] # addopts = -n auto +asyncio_mode = "strict" [tool.tox] legacy_tox_ini = """ diff --git a/tests/conftest.py b/tests/conftest.py index 17a5b6d..e89778b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ from pathlib import Path import tomli import pytest +import pytest_asyncio from nvchecker import core from nvchecker import __main__ as main @@ -41,7 +42,7 @@ async def run( vers, _has_failures = await main.run(result_coro, runner_coro) return vers -@pytest.fixture(scope="module") +@pytest_asyncio.fixture(scope="module") async def get_version(): async def __call__(name, config): entries = {name: config} @@ -50,7 +51,7 @@ async def get_version(): return __call__ -@pytest.fixture(scope="module") +@pytest_asyncio.fixture(scope="module") async def run_str(): async def __call__(str): entries = tomli.loads(str) @@ -59,7 +60,7 @@ async def run_str(): return __call__ -@pytest.fixture(scope="module") +@pytest_asyncio.fixture(scope="module") async def run_str_multi(): async def __call__(str): entries = tomli.loads(str) From d422205ad415fb6c3138ed1731cd4a1bd44f0c71 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 14:20:53 -0800 Subject: [PATCH 6/9] github: partial `use_commit_name` support --- docs/usage.rst | 8 +++- nvchecker_source/github.py | 75 ++++++++++++++++++++++++-------------- tests/test_github.py | 23 ++++++++++++ 3 files changed, 77 insertions(+), 29 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index c5ea879..60f3cc4 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -365,8 +365,8 @@ Check GitHub source = "github" Check `GitHub `_ for updates. The version returned is in -date format ``%Y%m%d.%H%M%S``, e.g. ``20130701.012212``, unless ``use_latest_release`` -or ``use_max_tag`` is used. See below. +date format ``%Y%m%d.%H%M%S``, e.g. ``20130701.012212``, unless ``use_latest_release``, +``use_max_tag``, or ``use_commit_name`` is used. See below. github The github repository, with author, e.g. ``lilydjwg/nvchecker``. @@ -393,6 +393,10 @@ use_latest_tag This requires a token because it's using the v4 GraphQL API. +use_commit_name + Set this to ``true`` to append a plus and the commit name to the version, e.g. + ``20130701.012212+e1457aadd30f53f4d50d6c4828d517355c09b8ae``. + query When ``use_latest_tag`` is ``true``, this sets a query for the tag. The exact matching method is not documented by GitHub. diff --git a/nvchecker_source/github.py b/nvchecker_source/github.py index 0920d54..84eedde 100644 --- a/nvchecker_source/github.py +++ b/nvchecker_source/github.py @@ -1,9 +1,10 @@ # MIT licensed # Copyright (c) 2013-2020 lilydjwg , et al. +import itertools import time from urllib.parse import urlencode -from typing import Tuple +from typing import Optional, Tuple import structlog @@ -14,6 +15,9 @@ from nvchecker.api import ( logger = structlog.get_logger(logger_name=__name__) +def add_commit_name(version: str, commit_name: Optional[str]) -> str: + return version if commit_name is None else version + '+' + commit_name + GITHUB_URL = 'https://api.github.com/repos/%s/commits' GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest' # https://developer.github.com/v3/git/refs/#get-all-references @@ -27,38 +31,45 @@ async def get_version(name, conf, **kwargs): check_ratelimit(e, name) QUERY_LATEST_TAG = ''' -{{ - repository(name: "{name}", owner: "{owner}") {{ - refs(refPrefix: "refs/tags/", first: 1, - query: "{query}", - orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{ - edges {{ - node {{ +query latestTag( + $owner: String!, $name: String!, + $query: String, $includeCommitName: Boolean = false, +) { + repository(owner: $owner, name: $name) { + refs( + refPrefix: "refs/tags/", query: $query, + first: 1, orderBy: {field: TAG_COMMIT_DATE, direction: DESC}, + ) { + edges { + node { name - }} - }} - }} - }} -}} + ... @include(if: $includeCommitName) { target { oid } } + } + } + } + } +} ''' -async def get_latest_tag(key: Tuple[str, str, str]) -> str: - repo, query, token = key +async def get_latest_tag(key: Tuple[str, Optional[str], str, bool]) -> str: + repo, query, token, use_commit_name = key owner, reponame = repo.split('/') headers = { 'Authorization': f'bearer {token}', 'Content-Type': 'application/json', } - q = QUERY_LATEST_TAG.format( - owner = owner, - name = reponame, - query = query, - ) + variables = { + 'owner': owner, + 'name': reponame, + 'includeCommitName': use_commit_name, + } + if query is not None: + variables['query'] = query res = await session.post( GITHUB_GRAPHQL_URL, headers = headers, - json = {'query': q}, + json = {'query': QUERY_LATEST_TAG, 'variables': variables}, ) j = res.json() @@ -66,7 +77,10 @@ async def get_latest_tag(key: Tuple[str, str, str]) -> str: if not refs: raise GetVersionError('no tag found') - return refs[0]['node']['name'] + return next(add_commit_name( + ref['node']['name'], + ref['node']['target']['oid'] if use_commit_name else None, + ) for ref in refs) async def get_version_real( name: str, conf: Entry, *, @@ -82,12 +96,13 @@ async def get_version_real( token = keymanager.get_key('github') use_latest_tag = conf.get('use_latest_tag', False) + use_commit_name = conf.get('use_commit_name', False) if use_latest_tag: if not token: raise GetVersionError('token not given but it is required') - query = conf.get('query', '') - return await cache.get((repo, query, token), get_latest_tag) # type: ignore + query = conf.get('query') + return await cache.get((repo, query, token, use_commit_name), get_latest_tag) # type: ignore br = conf.get('branch') path = conf.get('path') @@ -114,7 +129,10 @@ async def get_version_real( data = await cache.get_json(url, headers = headers) if use_max_tag: - tags = [ref['ref'].split('/', 2)[-1] for ref in data] + tags = [add_commit_name( + ref['ref'].split('/', 2)[-1], + ref['object']['sha'] if use_commit_name else None, + ) for ref in data] if not tags: raise GetVersionError('No tag found in upstream repository.') return tags @@ -126,8 +144,11 @@ async def get_version_real( else: # YYYYMMDD.HHMMSS - version = data[0]['commit']['committer']['date'] \ - .rstrip('Z').replace('-', '').replace(':', '').replace('T', '.') + version = add_commit_name( + data[0]['commit']['committer']['date'] \ + .rstrip('Z').replace('-', '').replace(':', '').replace('T', '.'), + data[0]['sha'] if use_commit_name else None, + ) return version diff --git a/tests/test_github.py b/tests/test_github.py index c505c83..2c9b3e6 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -15,6 +15,13 @@ async def test_github(get_version): "github": "harry-sanabria/ReleaseTestRepo", }) == "20140122.012101" +async def test_github_commit_name(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_commit_name": True, + }) == "20140122.012101+2b3cdf6134b07ae6ac77f11b586dc1ae6d1521db" + async def test_github_default_not_master(get_version): assert await get_version("example", { "source": "github", @@ -35,6 +42,14 @@ async def test_github_max_tag(get_version): "use_max_tag": True, }) == "second_release" +async def test_github_max_tag_commit_name(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_max_tag": True, + "use_commit_name": True, + }) == "second_release+0f01b10ee72809e7ec0d903db95bcb6eef18c925" + async def test_github_max_tag_with_ignored(get_version): assert await get_version("example", { "source": "github", @@ -74,3 +89,11 @@ async def test_github_latest_tag(get_version): "use_latest_tag": True, }) == "release3" +async def test_github_latest_tag_commit_name(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_latest_tag": True, + "use_commit_name": True, + }) == "release3+2b3cdf6134b07ae6ac77f11b586dc1ae6d1521db" + From cdd31a01e45dfd1fea1bb8234ef64d3366500140 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 18:00:51 -0800 Subject: [PATCH 7/9] github: factor out GitHub API querying --- nvchecker/util.py | 26 +++-- nvchecker_source/github.py | 188 ++++++++++++++++++++++--------------- 2 files changed, 129 insertions(+), 85 deletions(-) diff --git a/nvchecker/util.py b/nvchecker/util.py index fb50750..bf7657d 100644 --- a/nvchecker/util.py +++ b/nvchecker/util.py @@ -146,6 +146,14 @@ class BaseWorker: '''Run the `tasks`. Subclasses should implement this method.''' raise NotImplementedError +def _normalize(x: Any) -> Any: + if isinstance(x, list): + return tuple(sorted(_normalize(y) for y in x)) + elif isinstance(x, dict): + return tuple(sorted((_normalize(k), _normalize(v)) for k, v in x.items())) + else: + return x + class AsyncCache: '''A cache for use with async functions.''' cache: Dict[Hashable, Any] @@ -156,28 +164,32 @@ class AsyncCache: self.lock = asyncio.Lock() async def _get_json( - self, key: Tuple[str, str, Tuple[Tuple[str, str], ...]], + self, key: Tuple[str, str, Tuple[Tuple[str, str], ...], object], extra: Any, ) -> Any: - _, url, headers = key - res = await session.get(url, headers=dict(headers)) + _, url, headers, json = key + json = extra # denormalizing json would be a pain, so we sneak it through + res = await (session.get(url=url, headers=dict(headers)) if json is None \ + else session.post(url=url, headers=dict(headers), json=json)) return res.json() async def get_json( self, url: str, *, headers: Dict[str, str] = {}, + json: Optional[object] = None, ) -> Any: '''Get specified ``url`` and return the response content as JSON. The returned data will be cached for reuse. ''' - key = '_jsonurl', url, tuple(sorted(headers.items())) + key = '_jsonurl', url, _normalize(headers), _normalize(json) return await self.get( - key , self._get_json) # type: ignore + key, self._get_json, extra=json) # type: ignore async def get( self, key: Hashable, - func: Callable[[Hashable], Coroutine[Any, Any, Any]], + func: Callable[[Hashable, Optional[Any]], Coroutine[Any, Any, Any]], + extra: Optional[Any] = None, ) -> Any: '''Run async ``func`` and cache its return value by ``key``. @@ -189,7 +201,7 @@ class AsyncCache: async with self.lock: cached = self.cache.get(key) if cached is None: - coro = func(key) + coro = func(key, extra) fu = asyncio.create_task(coro) self.cache[key] = fu diff --git a/nvchecker_source/github.py b/nvchecker_source/github.py index 84eedde..f8e4fa1 100644 --- a/nvchecker_source/github.py +++ b/nvchecker_source/github.py @@ -4,7 +4,7 @@ import itertools import time from urllib.parse import urlencode -from typing import Optional, Tuple +from typing import Any, Dict, Optional, Tuple import structlog @@ -30,6 +30,53 @@ async def get_version(name, conf, **kwargs): except TemporaryError as e: check_ratelimit(e, name) +async def query_graphql( + *, + cache: AsyncCache, + token: Optional[str] = None, + headers: Optional[Dict[str, str]] = None, + query: str, + variables: Optional[Dict[str, object]] = None, + json: Optional[Dict[str, object]] = None, + url: Optional[str] = None, + **kwargs, +) -> Any: + if not token: + raise GetVersionError('token not given but it is required') + if headers is None: + headers = {} + headers.setdefault('Authorization', f'bearer {token}') + headers.setdefault('Content-Type', 'application/json') + + if json is None: + json = {} + json['query'] = query + if variables is not None: + json.setdefault('variables', {}).update(variables) + + if url is None: + url = GITHUB_GRAPHQL_URL + return await cache.get_json(url = url, headers = headers, json = json) + +async def query_rest( + *, + cache: AsyncCache, + token: Optional[str] = None, + headers: Optional[Dict[str, str]] = None, + parameters: Optional[Dict[str, str]] = None, + url: str, +) -> Any: + if headers is None: + headers = {} + if token: + headers.setdefault('Authorization', f'token {token}') + headers.setdefault('Accept', 'application/vnd.github.quicksilver-preview+json') + + if parameters: + url += '?' + urlencode(parameters) + + return await cache.get_json(url = url, headers = headers) + QUERY_LATEST_TAG = ''' query latestTag( $owner: String!, $name: String!, @@ -51,43 +98,13 @@ query latestTag( } ''' -async def get_latest_tag(key: Tuple[str, Optional[str], str, bool]) -> str: - repo, query, token, use_commit_name = key - owner, reponame = repo.split('/') - headers = { - 'Authorization': f'bearer {token}', - 'Content-Type': 'application/json', - } - variables = { - 'owner': owner, - 'name': reponame, - 'includeCommitName': use_commit_name, - } - if query is not None: - variables['query'] = query - - res = await session.post( - GITHUB_GRAPHQL_URL, - headers = headers, - json = {'query': QUERY_LATEST_TAG, 'variables': variables}, - ) - j = res.json() - - refs = j['data']['repository']['refs']['edges'] - if not refs: - raise GetVersionError('no tag found') - - return next(add_commit_name( - ref['node']['name'], - ref['node']['target']['oid'] if use_commit_name else None, - ) for ref in refs) - async def get_version_real( name: str, conf: Entry, *, cache: AsyncCache, keymanager: KeyManager, **kwargs, ) -> VersionResult: repo = conf['github'] + use_commit_name = conf.get('use_commit_name', False) # Load token from config token = conf.get('token') @@ -95,62 +112,77 @@ async def get_version_real( if token is None: token = keymanager.get_key('github') - use_latest_tag = conf.get('use_latest_tag', False) - use_commit_name = conf.get('use_commit_name', False) - if use_latest_tag: - if not token: - raise GetVersionError('token not given but it is required') - - query = conf.get('query') - return await cache.get((repo, query, token, use_commit_name), get_latest_tag) # type: ignore - - br = conf.get('branch') - path = conf.get('path') - use_latest_release = conf.get('use_latest_release', False) - use_max_tag = conf.get('use_max_tag', False) - if use_latest_release: - url = GITHUB_LATEST_RELEASE % repo - elif use_max_tag: - url = GITHUB_MAX_TAG % repo - else: - url = GITHUB_URL % repo - parameters = {} - if br: - parameters['sha'] = br - if path: - parameters['path'] = path - url += '?' + urlencode(parameters) - headers = { - 'Accept': 'application/vnd.github.quicksilver-preview+json', - } - if token: - headers['Authorization'] = f'token {token}' - - data = await cache.get_json(url, headers = headers) - - if use_max_tag: - tags = [add_commit_name( - ref['ref'].split('/', 2)[-1], - ref['object']['sha'] if use_commit_name else None, - ) for ref in data] + if conf.get('use_latest_tag', False): + owner, reponame = repo.split('/') + j = await query_graphql( + cache = cache, + token = token, + query = QUERY_LATEST_TAG, + variables = { + 'owner': owner, + 'name': reponame, + 'query': conf.get('query'), + 'includeCommitName': use_commit_name, + }, + ) + refs = j['data']['repository']['refs']['edges'] + if not refs: + raise GetVersionError('no tag found') + ref = next( + add_commit_name( + ref['node']['name'], + ref['node']['target']['oid'] if use_commit_name else None, + ) + for ref in refs + ) + return ref + elif conf.get('use_latest_release', False): + data = await query_rest( + cache = cache, + token = token, + url = GITHUB_LATEST_RELEASE % repo, + ) + if 'tag_name' not in data: + raise GetVersionError('No release found in upstream repository.') + tag = data['tag_name'] + return tag + elif conf.get('use_max_tag', False): + data = await query_rest( + cache = cache, + token = token, + url = GITHUB_MAX_TAG % repo, + ) + tags = [ + add_commit_name( + ref['ref'].split('/', 2)[-1], + ref['object']['sha'] if use_commit_name else None, + ) + for ref in data + ] if not tags: raise GetVersionError('No tag found in upstream repository.') return tags - - if use_latest_release: - if 'tag_name' not in data: - raise GetVersionError('No release found in upstream repository.') - version = data['tag_name'] - else: + br = conf.get('branch') + path = conf.get('path') + parameters = {} + if br is not None: + parameters['sha'] = br + if path is not None: + parameters['path'] = path + data = await query_rest( + cache = cache, + token = token, + url = GITHUB_URL % repo, + parameters = parameters, + ) # YYYYMMDD.HHMMSS version = add_commit_name( data[0]['commit']['committer']['date'] \ .rstrip('Z').replace('-', '').replace(':', '').replace('T', '.'), data[0]['sha'] if use_commit_name else None, ) - - return version + return version def check_ratelimit(exc, name): res = exc.response From 03a83d82782f251a35771a8b221468eeb74ea029 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 18:21:01 -0800 Subject: [PATCH 8/9] github: full `use_commit_name` support --- docs/usage.rst | 2 ++ nvchecker_source/github.py | 49 ++++++++++++++++++++++++++++++++------ tests/test_github.py | 8 +++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 60f3cc4..6fb7156 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -397,6 +397,8 @@ use_commit_name Set this to ``true`` to append a plus and the commit name to the version, e.g. ``20130701.012212+e1457aadd30f53f4d50d6c4828d517355c09b8ae``. + If this isn't showing up, provide a token so it can use the v4 GraphQL API. + query When ``use_latest_tag`` is ``true``, this sets a query for the tag. The exact matching method is not documented by GitHub. diff --git a/nvchecker_source/github.py b/nvchecker_source/github.py index f8e4fa1..06e2f07 100644 --- a/nvchecker_source/github.py +++ b/nvchecker_source/github.py @@ -98,6 +98,20 @@ query latestTag( } ''' +QUERY_LATEST_RELEASE = ''' +query latestRelease( + $owner: String!, $name: String!, + $includeCommitName: Boolean = false, +) { + repository(owner: $owner, name: $name) { + latestRelease { + tagName + ... @include(if: $includeCommitName) { tagCommit { oid } } + } + } +} +''' + async def get_version_real( name: str, conf: Entry, *, cache: AsyncCache, keymanager: KeyManager, @@ -137,14 +151,35 @@ async def get_version_real( ) return ref elif conf.get('use_latest_release', False): - data = await query_rest( - cache = cache, - token = token, - url = GITHUB_LATEST_RELEASE % repo, - ) - if 'tag_name' not in data: + tag = None + if token: + owner, reponame = repo.split('/') + j = await query_graphql( + cache = cache, + token = token, + query = QUERY_LATEST_RELEASE, + variables = { + 'owner': owner, + 'name': reponame, + 'includeCommitName': use_commit_name, + }, + ) + release = j['data']['repository']['latestRelease'] + if release is not None: + tag = add_commit_name( + release['tagName'], + release['tagCommit']['oid'] if use_commit_name else None, + ) + else: + data = await query_rest( + cache = cache, + token = token, + url = GITHUB_LATEST_RELEASE % repo, + ) + if 'tag_name' in data: + tag = data['tag_name'] + if tag is None: raise GetVersionError('No release found in upstream repository.') - tag = data['tag_name'] return tag elif conf.get('use_max_tag', False): data = await query_rest( diff --git a/tests/test_github.py b/tests/test_github.py index 2c9b3e6..7b30c0b 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -35,6 +35,14 @@ async def test_github_latest_release(get_version): "use_latest_release": True, }) == "release3" +async def test_github_latest_release_commit_name(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_latest_release": True, + "use_commit_name": True, + }) == "release3+2b3cdf6134b07ae6ac77f11b586dc1ae6d1521db" + async def test_github_max_tag(get_version): assert await get_version("example", { "source": "github", From 4d5c102f152ffddb0c9953e35ed70267e81cbd94 Mon Sep 17 00:00:00 2001 From: Dusk Banks Date: Tue, 22 Feb 2022 20:45:11 -0800 Subject: [PATCH 9/9] github: full GraphQL API support there is currently not GraphQL API equivalent to the REST API's `/repos/{owner}/{repo}/commits/{ref}` endpoint. --- docs/usage.rst | 8 ++ nvchecker_source/github.py | 148 +++++++++++++++++++++++++------------ 2 files changed, 108 insertions(+), 48 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 6fb7156..489b27e 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -409,6 +409,12 @@ use_max_tag lightweight ones, and return the largest one sorted by the ``sort_version_key`` option. Will return the tag name instead of date. + This defaults ``list_count`` to 100. + +list_count + When supporting :ref:`list options` through the v4 GraphQL API, this sets a + maximum count of items in the list. By default, ``list_count`` is set to 1. + token A personal authorization token used to call the API. @@ -421,6 +427,8 @@ To set an authorization token, you can set: - the token option This source supports :ref:`list options` when ``use_max_tag`` is set. +Options of this source that support :ref:`list options` may be effected by +``list_count``. Check Gitea ~~~~~~~~~~~ diff --git a/nvchecker_source/github.py b/nvchecker_source/github.py index 06e2f07..f51146c 100644 --- a/nvchecker_source/github.py +++ b/nvchecker_source/github.py @@ -1,10 +1,9 @@ # MIT licensed # Copyright (c) 2013-2020 lilydjwg , et al. -import itertools import time from urllib.parse import urlencode -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import structlog @@ -77,27 +76,76 @@ async def query_rest( return await cache.get_json(url = url, headers = headers) -QUERY_LATEST_TAG = ''' -query latestTag( +QUERY_LATEST_TAGS = ''' +query latestTags( $owner: String!, $name: String!, - $query: String, $includeCommitName: Boolean = false, + $query: String, $orderByCommitDate: Boolean!, $count: Int = 1, + $includeCommitName: Boolean = false, ) { repository(owner: $owner, name: $name) { - refs( + ... @include(if: $orderByCommitDate) { latestRefs: refs( refPrefix: "refs/tags/", query: $query, - first: 1, orderBy: {field: TAG_COMMIT_DATE, direction: DESC}, - ) { - edges { - node { - name - ... @include(if: $includeCommitName) { target { oid } } - } - } + first: $count, orderBy: {field: TAG_COMMIT_DATE, direction: DESC} + ) { ...tagData } } + ... @skip(if: $orderByCommitDate) { maxRefs: refs( + refPrefix: "refs/tags/", query: $query, + last: $count + ) { ...tagData } } + } +} +fragment tagData on RefConnection { + edges { + node { + name + ... @include(if: $includeCommitName) { target { ...commitOid } } } } } +fragment commitOid on GitObject { + ... on Commit { commitOid: oid } + ... on Tag { tagTarget: target { + ... on Commit { commitOid: oid } + } } +} ''' +async def query_latest_tags( + *, + cache: AsyncCache, + token: Optional[str] = None, + owner: str, + name: str, + query: Optional[str], + order_by_commit_date: bool, + count: Optional[int] = None, + use_commit_name: bool, +) -> List[str]: + j = await query_graphql( + cache = cache, + token = token, + query = QUERY_LATEST_TAGS, + variables = { + 'owner': owner, + 'name': name, + 'query': query, + 'orderByCommitDate': order_by_commit_date, + 'count': count, + 'includeCommitName': use_commit_name, + }, + ) + refsAlias = 'latestRefs' if order_by_commit_date else 'maxRefs' + refs = j['data']['repository'][refsAlias]['edges'] + if not order_by_commit_date: + refs = reversed(refs) + tags = [ + add_commit_name( + ref['node']['name'], + ref['node']['target']['commitOid'] if use_commit_name else None, + ) + for ref in refs + ] + return tags + QUERY_LATEST_RELEASE = ''' query latestRelease( $owner: String!, $name: String!, @@ -128,28 +176,18 @@ async def get_version_real( if conf.get('use_latest_tag', False): owner, reponame = repo.split('/') - j = await query_graphql( + tags = await query_latest_tags( cache = cache, token = token, - query = QUERY_LATEST_TAG, - variables = { - 'owner': owner, - 'name': reponame, - 'query': conf.get('query'), - 'includeCommitName': use_commit_name, - }, + owner = owner, + name = reponame, + query = conf.get('query'), + order_by_commit_date = True, + use_commit_name = use_commit_name, ) - refs = j['data']['repository']['refs']['edges'] - if not refs: - raise GetVersionError('no tag found') - ref = next( - add_commit_name( - ref['node']['name'], - ref['node']['target']['oid'] if use_commit_name else None, - ) - for ref in refs - ) - return ref + if not tags: + raise GetVersionError('No tag found in upstream repository.') + return tags[0] elif conf.get('use_latest_release', False): tag = None if token: @@ -182,18 +220,31 @@ async def get_version_real( raise GetVersionError('No release found in upstream repository.') return tag elif conf.get('use_max_tag', False): - data = await query_rest( - cache = cache, - token = token, - url = GITHUB_MAX_TAG % repo, - ) - tags = [ - add_commit_name( - ref['ref'].split('/', 2)[-1], - ref['object']['sha'] if use_commit_name else None, + if token: + owner, reponame = repo.split('/') + tags = await query_latest_tags( + cache = cache, + token = token, + owner = owner, + name = reponame, + query = conf.get('query'), + order_by_commit_date = False, + count = conf.get('list_count', 100), + use_commit_name = use_commit_name, ) - for ref in data - ] + else: + data = await query_rest( + cache = cache, + token = token, + url = GITHUB_MAX_TAG % repo, + ) + tags = [ + add_commit_name( + ref['ref'].split('/', 2)[-1], + ref['object']['sha'] if use_commit_name else None, + ) + for ref in data + ] if not tags: raise GetVersionError('No tag found in upstream repository.') return tags @@ -211,11 +262,12 @@ async def get_version_real( url = GITHUB_URL % repo, parameters = parameters, ) - # YYYYMMDD.HHMMSS + date = data[0]['commit']['committer']['date'] + commit_name = data[0]['sha'] if use_commit_name else None version = add_commit_name( - data[0]['commit']['committer']['date'] \ - .rstrip('Z').replace('-', '').replace(':', '').replace('T', '.'), - data[0]['sha'] if use_commit_name else None, + # YYYYMMDD.HHMMSS + date.rstrip('Z').replace('-', '').replace(':', '').replace('T', '.'), + commit_name, ) return version