support both tornado and aiohttp

also fix throttling with aiohttp

nvchecker/core.py

@@ -88,26 +88,40 @@ class Source:
       self.oldvers = {}
     self.curvers = self.oldvers.copy()
 
-    futures = []
+    futures = set()
     config = self.config
     for name in config.sections():
       if name == '__config__':
         continue
       conf = config[name]
       conf['oldver'] = self.oldvers.get(name, None)
-      futures.append(get_version(name, conf))
+      fu = asyncio.ensure_future(get_version(name, conf))
+      fu.name = name
+      futures.add(fu)
 
-    for fu in asyncio.as_completed(futures):
-      try:
-        name, version = await fu
-        if version is not None:
-          self.print_version_update(name, version)
-      except Exception:
-        logger.exception('error happened dealing with %s', name)
+      if len(futures) >= 20: # TODO: use __config__
+        (done, futures) = await asyncio.wait(
+          futures, return_when = asyncio.FIRST_COMPLETED)
+        for fu in done:
+          self.future_done(fu)
+
+    if futures:
+      (done, _) = await asyncio.wait(futures)
+      for fu in done:
+        self.future_done(fu)
 
     if self.newver:
       write_verfile(self.newver, self.curvers)
 
+  def future_done(self, fu):
+    name = fu.name
+    try:
+      _, version = fu.result()
+      if version is not None:
+        self.print_version_update(name, version)
+    except Exception:
+      logger.exception('unexpected error happened with %s', name)
+
   def print_version_update(self, name, version):
     oldver = self.oldvers.get(name, None)
     if not oldver or oldver != version:

nvchecker/source/__init__.py

@@ -1,18 +1,14 @@
 # MIT licensed
 # Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
 
-import atexit
-import aiohttp
-connector = aiohttp.TCPConnector(limit=20)
+try:
+  import tornado, pycurl
+  which = 'tornado'
+except ImportError:
+  import aiohttp
+  which = 'aiohttp'
 
-config = None
-
-class BetterClientSession(aiohttp.ClientSession):
-    async def _request(self, *args, **kwargs):
-        if hasattr(self, "nv_config") and self.nv_config.get("proxy"):
-            kwargs.setdefault("proxy", self.nv_config.get("proxy"))
-
-        return await super(BetterClientSession, self)._request(*args, **kwargs)
-
-session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5)
-atexit.register(session.close)
+m = __import__('%s_httpclient' % which, globals(), locals(), level=1)
+__all__ = m.__all__
+for x in __all__:
+  globals()[x] = getattr(m, x)

nvchecker/source/aiohttp_httpclient.py

@@ -0,0 +1,18 @@
+# MIT licensed
+# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+
+import atexit
+import aiohttp
+connector = aiohttp.TCPConnector(limit=20)
+
+__all__ = ['session']
+
+class BetterClientSession(aiohttp.ClientSession):
+    async def _request(self, *args, **kwargs):
+        if hasattr(self, "nv_config") and self.nv_config.get("proxy"):
+            kwargs.setdefault("proxy", self.nv_config.get("proxy"))
+
+        return await super(BetterClientSession, self)._request(*args, **kwargs)
+
+session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5)
+atexit.register(session.close)

nvchecker/source/tornado_httpclient.py

@@ -0,0 +1,63 @@
+# MIT licensed
+# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+
+import json
+import asyncio
+from urllib.parse import urlencode
+
+from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPResponse
+from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future
+AsyncIOMainLoop().install()
+
+try:
+  import pycurl
+  AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
+except ImportError:
+  pycurl = None
+
+__all__ = ['session']
+
+client = AsyncHTTPClient()
+
+def try_use_http2(curl):
+  if pycurl:
+    curl.setopt(pycurl.HTTP_VERSION, 4)
+
+class Session:
+  def get(self, url, **kwargs):
+    kwargs['prepare_curl_callback'] = try_use_http2
+
+    proxy = kwargs.get('proxy')
+    if proxy:
+      del kwargs['proxy']
+    elif hasattr(self, 'nv_config') and self.nv_config.get('proxy'):
+      proxy = self.nv_config.get('proxy')
+    if proxy:
+      host, port = proxy.rsplit(':', 1)
+      kwargs['proxy_host'] = host
+      kwargs['proxy_port'] = int(port)
+
+    params = kwargs.get('params')
+    if params:
+      del kwargs['params']
+      q = urlencode(params)
+      url += '?' + q
+
+    r = HTTPRequest(url, **kwargs)
+    return ResponseManager(r)
+
+class ResponseManager:
+  def __init__(self, req):
+    self.req = req
+
+  async def __aenter__(self):
+    return await to_asyncio_future(client.fetch(self.req))
+
+  async def __aexit__(self, exc_type, exc, tb):
+    pass
+
+async def json_response(self):
+  return json.loads(self.body.decode('utf-8'))
+
+HTTPResponse.json = json_response
+session = Session()