diff --git a/nvchecker/core.py b/nvchecker/core.py index d369231..f113fe8 100644 --- a/nvchecker/core.py +++ b/nvchecker/core.py @@ -88,26 +88,40 @@ class Source: self.oldvers = {} self.curvers = self.oldvers.copy() - futures = [] + futures = set() config = self.config for name in config.sections(): if name == '__config__': continue conf = config[name] conf['oldver'] = self.oldvers.get(name, None) - futures.append(get_version(name, conf)) + fu = asyncio.ensure_future(get_version(name, conf)) + fu.name = name + futures.add(fu) - for fu in asyncio.as_completed(futures): - try: - name, version = await fu - if version is not None: - self.print_version_update(name, version) - except Exception: - logger.exception('error happened dealing with %s', name) + if len(futures) >= 20: # TODO: use __config__ + (done, futures) = await asyncio.wait( + futures, return_when = asyncio.FIRST_COMPLETED) + for fu in done: + self.future_done(fu) + + if futures: + (done, _) = await asyncio.wait(futures) + for fu in done: + self.future_done(fu) if self.newver: write_verfile(self.newver, self.curvers) + def future_done(self, fu): + name = fu.name + try: + _, version = fu.result() + if version is not None: + self.print_version_update(name, version) + except Exception: + logger.exception('unexpected error happened with %s', name) + def print_version_update(self, name, version): oldver = self.oldvers.get(name, None) if not oldver or oldver != version: diff --git a/nvchecker/source/__init__.py b/nvchecker/source/__init__.py index fa45a66..a823ee6 100644 --- a/nvchecker/source/__init__.py +++ b/nvchecker/source/__init__.py @@ -1,18 +1,14 @@ # MIT licensed # Copyright (c) 2013-2017 lilydjwg , et al. -import atexit -import aiohttp -connector = aiohttp.TCPConnector(limit=20) +try: + import tornado, pycurl + which = 'tornado' +except ImportError: + import aiohttp + which = 'aiohttp' -config = None - -class BetterClientSession(aiohttp.ClientSession): - async def _request(self, *args, **kwargs): - if hasattr(self, "nv_config") and self.nv_config.get("proxy"): - kwargs.setdefault("proxy", self.nv_config.get("proxy")) - - return await super(BetterClientSession, self)._request(*args, **kwargs) - -session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5) -atexit.register(session.close) +m = __import__('%s_httpclient' % which, globals(), locals(), level=1) +__all__ = m.__all__ +for x in __all__: + globals()[x] = getattr(m, x) diff --git a/nvchecker/source/aiohttp_httpclient.py b/nvchecker/source/aiohttp_httpclient.py new file mode 100644 index 0000000..9bd9e59 --- /dev/null +++ b/nvchecker/source/aiohttp_httpclient.py @@ -0,0 +1,18 @@ +# MIT licensed +# Copyright (c) 2013-2017 lilydjwg , et al. + +import atexit +import aiohttp +connector = aiohttp.TCPConnector(limit=20) + +__all__ = ['session'] + +class BetterClientSession(aiohttp.ClientSession): + async def _request(self, *args, **kwargs): + if hasattr(self, "nv_config") and self.nv_config.get("proxy"): + kwargs.setdefault("proxy", self.nv_config.get("proxy")) + + return await super(BetterClientSession, self)._request(*args, **kwargs) + +session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5) +atexit.register(session.close) diff --git a/nvchecker/source/tornado_httpclient.py b/nvchecker/source/tornado_httpclient.py new file mode 100644 index 0000000..e5a6737 --- /dev/null +++ b/nvchecker/source/tornado_httpclient.py @@ -0,0 +1,63 @@ +# MIT licensed +# Copyright (c) 2013-2017 lilydjwg , et al. + +import json +import asyncio +from urllib.parse import urlencode + +from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPResponse +from tornado.platform.asyncio import AsyncIOMainLoop, to_asyncio_future +AsyncIOMainLoop().install() + +try: + import pycurl + AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") +except ImportError: + pycurl = None + +__all__ = ['session'] + +client = AsyncHTTPClient() + +def try_use_http2(curl): + if pycurl: + curl.setopt(pycurl.HTTP_VERSION, 4) + +class Session: + def get(self, url, **kwargs): + kwargs['prepare_curl_callback'] = try_use_http2 + + proxy = kwargs.get('proxy') + if proxy: + del kwargs['proxy'] + elif hasattr(self, 'nv_config') and self.nv_config.get('proxy'): + proxy = self.nv_config.get('proxy') + if proxy: + host, port = proxy.rsplit(':', 1) + kwargs['proxy_host'] = host + kwargs['proxy_port'] = int(port) + + params = kwargs.get('params') + if params: + del kwargs['params'] + q = urlencode(params) + url += '?' + q + + r = HTTPRequest(url, **kwargs) + return ResponseManager(r) + +class ResponseManager: + def __init__(self, req): + self.req = req + + async def __aenter__(self): + return await to_asyncio_future(client.fetch(self.req)) + + async def __aexit__(self, exc_type, exc, tb): + pass + +async def json_response(self): + return json.loads(self.body.decode('utf-8')) + +HTTPResponse.json = json_response +session = Session()