Merge pull request #162 from yan12125/container

Add container source plugin

docs/usage.rst

@@ -662,6 +662,22 @@ git
 
 This source returns tags and supports :ref:`list options`.
 
+Check container registry
+~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "container"
+
+This enables you to check tags of images on a container registry like Docker.
+
+container
+  The path for the container image. For official Docker images, use namespace ``library/`` (e.g. ``library/python``).
+
+registry
+  The container registry host. Default: ``docker.io``
+
+This source returns tags and supports :ref:`list options`.
+
 Manually updating
 ~~~~~~~~~~~~~~~~~
 ::

nvchecker/api.py

@@ -1,7 +1,7 @@
 # MIT licensed
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from .httpclient import session, TemporaryError
+from .httpclient import session, TemporaryError, HTTPError
 from .util import (
   Entry, BaseWorker, RawResult, VersionResult,
   AsyncCache, KeyManager, GetVersionError,

nvchecker/httpclient/__init__.py

@@ -3,7 +3,7 @@
 
 from typing import Optional
 
-from .base import TemporaryError
+from .base import TemporaryError, HTTPError
 
 class Proxy:
   _obj = None

nvchecker/httpclient/aiohttp_httpclient.py

@@ -8,7 +8,7 @@ from typing import Optional, Dict
 import structlog
 import aiohttp
 
-from .base import BaseSession, TemporaryError, Response
+from .base import BaseSession, TemporaryError, Response, HTTPError
 
 __all__ = ['session']
 
@@ -54,10 +54,13 @@ class AiohttpSession(BaseSession):
     ) as e:
       raise TemporaryError(599, repr(e), e)
 
+    err_cls: Optional[type] = None
     if res.status >= 500:
-      raise TemporaryError(res.status, res.reason, res)
+      err_cls = TemporaryError
-    else:
+    elif res.status >= 400:
-      res.raise_for_status()
+      err_cls = HTTPError
+    if err_cls is not None:
+      raise err_cls(res.status, res.reason, res)
 
     body = await res.content.read()
     return Response(body)

nvchecker/httpclient/base.py

@@ -86,10 +86,14 @@ class BaseSession:
     ''':meta private:'''
     raise NotImplementedError
 
-class TemporaryError(Exception):
+class BaseHTTPError(Exception):
-  '''A temporary error (e.g. network error) happens.'''
   def __init__(self, code, message, response):
     self.code = code
     self.message = message
     self.response = response
 
+class TemporaryError(BaseHTTPError):
+  '''A temporary error (e.g. network error) happens.'''
+
+class HTTPError(BaseHTTPError):
+  ''' An HTTP 4xx error happens '''

nvchecker/httpclient/httpx_httpclient.py

@@ -6,7 +6,7 @@ from typing import Dict, Optional
 
 import httpx
 
-from .base import BaseSession, TemporaryError, Response
+from .base import BaseSession, TemporaryError, Response, HTTPError
 
 __all__ = ['session']
 
@@ -42,14 +42,17 @@ class HttpxSession(BaseSession):
         headers = headers,
         params = params,
       )
+      err_cls: Optional[type] = None
       if r.status_code >= 500:
-        raise TemporaryError(
+        err_cls = TemporaryError
+      elif r.status_code >= 400:
+        err_cls = HTTPError
+      if err_cls is not None:
+        raise err_cls(
           r.status_code,
           r.reason_phrase,
           r,
         )
-      else:
-        r.raise_for_status()
 
     except httpx.TransportError as e:
       raise TemporaryError(599, repr(e), e)

nvchecker/httpclient/tornado_httpclient.py

@@ -12,7 +12,7 @@ try:
 except ImportError:
   pycurl = None # type: ignore
 
-from .base import BaseSession, TemporaryError, Response
+from .base import BaseSession, TemporaryError, Response, HTTPError
 
 __all__ = ['session']
 
@@ -74,12 +74,15 @@ class TornadoSession(BaseSession):
     r = HTTPRequest(url, **kwargs)
     res = await AsyncHTTPClient().fetch(
       r, raise_error=False)
+    err_cls: Optional[type] = None
     if res.code >= 500:
-      raise TemporaryError(
+      err_cls = TemporaryError
+    elif res.code >= 400:
+      err_cls = HTTPError
+    if err_cls is not None:
+      raise err_cls(
         res.code, res.reason, res
       )
-    else:
-      res.rethrow()
 
     return Response(res.body)
 

nvchecker_source/android_sdk.py

@@ -1,6 +1,6 @@
 # MIT licensed
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
-# Copyright (c) 2017 Yen Chi Hsuan <yan12125 at gmail dot com>
+# Copyright (c) 2017 Chih-Hsuan Yen <yan12125 at gmail dot com>
 
 import os
 import re

nvchecker_source/container.py

@@ -0,0 +1,83 @@
+# MIT licensed
+# Copyright (c) 2020 Chih-Hsuan Yen <yan12125 at gmail dot com>
+
+from typing import Dict, List, NamedTuple, Optional, Tuple
+from urllib.request import parse_http_list
+
+from nvchecker.api import session, HTTPError
+
+class AuthInfo(NamedTuple):
+  service: Optional[str]
+  realm: str
+
+def parse_www_authenticate_header(header: str) -> Tuple[str, Dict[str, str]]:
+  '''
+  Parse WWW-Authenticate header used in OAuth2 authentication for container
+  registries. This is NOT RFC-compliant!
+
+  Simplified from http.parse_www_authenticate_header in Werkzeug (BSD license)
+  '''
+  auth_type, auth_info = header.split(None, 1)
+  result = {}
+  for item in parse_http_list(auth_info):
+    name, value = item.split("=", 1)
+    if value[:1] == value[-1:] == '"':
+      value = value[1:-1]
+    result[name] = value
+  return auth_type, result
+
+# Inspired by https://stackoverflow.com/a/51921869
+# Reference: https://github.com/containers/image/blob/v5.6.0/docker/docker_client.go
+
+class UnsupportedAuthenticationError(NotImplementedError):
+  def __init__(self):
+    super().__init__('Only Bearer authentication supported for now')
+
+async def get_registry_auth_info(registry_host: str) -> AuthInfo:
+  auth_service = auth_realm = None
+
+  try:
+    await session.get(f'https://{registry_host}/v2/')
+    raise UnsupportedAuthenticationError  # No authentication needed
+  except HTTPError as e:
+    if e.code != 401:
+      raise
+
+    auth_type, auth_info = parse_www_authenticate_header(e.response.headers['WWW-Authenticate'])
+    if auth_type.lower() != 'bearer':
+      raise UnsupportedAuthenticationError
+
+    # Although 'service' is needed as per https://docs.docker.com/registry/spec/auth/token/,
+    # ghcr.io (GitHub container registry) does not provide it
+    auth_service = auth_info.get('service')
+    auth_realm = auth_info['realm']
+
+    return AuthInfo(auth_service, auth_realm)
+
+async def get_container_tags(info: Tuple[str, str, AuthInfo]) -> List[str]:
+  image_path, registry_host, auth_info = info
+
+  auth_params = {
+    'scope': f'repository:{image_path}:pull',
+  }
+  if auth_info.service:
+    auth_params['service'] = auth_info.service
+  res = await session.get(auth_info.realm, params=auth_params)
+  token = res.json()['token']
+
+  res = await session.get(f'https://{registry_host}/v2/{image_path}/tags/list', headers={
+    'Authorization': f'Bearer {token}',
+    'Accept': 'application/json',
+  })
+  return res.json()['tags']
+
+async def get_version(name, conf, *, cache, **kwargs):
+  image_path = conf.get('container', name)
+  registry_host = conf.get('registry', 'docker.io')
+  if registry_host == 'docker.io':
+    registry_host = 'registry-1.docker.io'
+
+  auth_info = await cache.get(registry_host, get_registry_auth_info)
+
+  key = image_path, registry_host, auth_info
+  return await cache.get(key, get_container_tags)

tests/test_android_sdk.py

@@ -1,6 +1,6 @@
 # MIT licensed
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
-# Copyright (c) 2017 Yen Chi Hsuan <yan12125 at gmail dot com>
+# Copyright (c) 2017 Chih-Hsuan Yen <yan12125 at gmail dot com>
 
 import pytest
 pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]

tests/test_container.py

@@ -0,0 +1,12 @@
+# MIT licensed
+# Copyright (c) 2020 Chih-Hsuan Yen <yan12125 at gmail dot com>
+
+import pytest
+pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+
+async def test_container(get_version):
+  assert await get_version("hello-world", {
+    "source": "container",
+    "container": "library/hello-world",
+    "include_regex": "linux",
+  }) == "linux"