pypi: Use xmlrpc instead of json api

It makes the transfer much smaller as the xmlrpc method returns only exactly what we need here.

nvchecker/httpclient/aiohttp_httpclient.py

@@ -30,6 +30,7 @@ class AiohttpSession(BaseSession):
     headers: Dict[str, str] = {},
     params = (),
     json = None,
+    body = None,
   ) -> Response:
     kwargs = {
       'headers': headers,
@@ -40,6 +41,8 @@ class AiohttpSession(BaseSession):
       kwargs['proxy'] = proxy
     if json is not None:
       kwargs['json'] = json
+    if body is not None:
+      kwargs['body'] = body
 
     try:
       logger.debug('send request', method=method, url=url, kwargs=kwargs)

nvchecker/httpclient/base.py

@@ -40,6 +40,7 @@ class BaseSession:
     headers: Dict[str, str] = {},
     params = (),
     json = None,
+    body = None,
   ) -> Response:
     t = tries.get()
     p = proxy.get()
@@ -56,6 +57,7 @@ class BaseSession:
           headers = headers,
           params = params,
           json = json,
+          body = body,
           proxy = p or None,
         )
       except TemporaryError as e:

nvchecker/httpclient/httpx_httpclient.py

@@ -21,6 +21,7 @@ class HttpxSession(BaseSession):
     headers: Dict[str, str] = {},
     params = (),
     json = None,
+    body = None,
   ) -> Response:
     client = self.clients.get(proxy)
     if not client:
@@ -36,6 +37,7 @@ class HttpxSession(BaseSession):
         method, url, json = json,
         headers = headers,
         params = params,
+        content = body,
       )
       if r.status_code >= 500:
         raise TemporaryError(

nvchecker/httpclient/tornado_httpclient.py

@@ -38,6 +38,7 @@ class TornadoSession(BaseSession):
     headers: Dict[str, str] = {},
     params = (),
     json = None,
+    body = None,
   ) -> Response:
     kwargs: Dict[str, Any] = {
       'method': method,
@@ -46,6 +47,8 @@ class TornadoSession(BaseSession):
 
     if json:
       kwargs['body'] = _json.dumps(json)
+    if body:
+      kwargs['body'] = body
     kwargs['prepare_curl_callback'] = try_use_http2
 
     if proxy:

nvchecker_source/pypi.py

@@ -1,21 +1,32 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+import xmlrpc.client
+from typing import List, Tuple, Union
 from pkg_resources import parse_version
 
+from nvchecker.api import session
+
+async def pypi_xmlrpc_request(key: Tuple[str, str, bool]) -> Union[str, List[str]]:
+  url, package, use_pre_release = key
+  payload = xmlrpc.client.dumps((package, use_pre_release), "package_releases")
+
+  res = await session.post(url, headers={"Content-Type": "text/xml"}, body=payload)
+  params, methodname = xmlrpc.client.loads(res.body)
+  return params[0] # type: ignore
+
 async def get_version(name, conf, *, cache, **kwargs):
   package = conf.get('pypi') or name
-  use_pre_release = conf.get('use_pre_release', False)
+  use_pre_release = bool(conf.get('use_pre_release', False))
 
-  url = 'https://pypi.org/pypi/{}/json'.format(package)
+  url = 'https://pypi.org/pypi'
-
+  data = await cache.get((url, package, use_pre_release), pypi_xmlrpc_request)
-  data = await cache.get_json(url)
 
   if use_pre_release:
     version = sorted(
-      data['releases'].keys(),
+      data,
       key = parse_version,
     )[-1]
   else:
-    version = data['info']['version']
+    version = data
   return version