diff --git a/.travis.yml b/.travis.yml index b009407..8e25359 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ python: - "3.6" - "3.7" - "nightly" - - "pypy3.5" -install: pip install -U $DEPS pytest pytest-asyncio flaky structlog + - "pypy3.6-7.1.1" +install: pip install -U $DEPS pytest pytest-asyncio pytest-httpbin flaky structlog script: pytest env: global: @@ -26,5 +26,10 @@ matrix: fast_finish: true allow_failures: # doesn't work well, see https://travis-ci.org/lilydjwg/nvchecker/jobs/376326582 - - python: pypy3.5 + - python: pypy3.6-7.1.1 env: DEPS=aiohttp + +addons: + apt: + packages: + - libgnutls-dev diff --git a/nvchecker/source/aiohttp_httpclient.py b/nvchecker/source/aiohttp_httpclient.py index 4fd8b4f..bc73107 100644 --- a/nvchecker/source/aiohttp_httpclient.py +++ b/nvchecker/source/aiohttp_httpclient.py @@ -4,6 +4,8 @@ import atexit import asyncio import aiohttp +from .httpclient import DEFAULT_USER_AGENT + connector = aiohttp.TCPConnector(limit=20) __all__ = ['session', 'HTTPError', 'NetworkErrors'] @@ -19,6 +21,8 @@ class BetterClientSession(aiohttp.ClientSession): if hasattr(self, "nv_config") and self.nv_config.get("proxy"): kwargs.setdefault("proxy", self.nv_config.get("proxy")) + kwargs.setdefault("headers", {}).setdefault('User-Agent', DEFAULT_USER_AGENT) + res = await super(BetterClientSession, self)._request( *args, **kwargs) if res.status >= 400: diff --git a/nvchecker/source/github.py b/nvchecker/source/github.py index 6ecd499..e19b804 100644 --- a/nvchecker/source/github.py +++ b/nvchecker/source/github.py @@ -46,7 +46,6 @@ async def get_version_real(name, conf, **kwargs): url += '?' + urlencode(parameters) headers = { 'Accept': 'application/vnd.github.quicksilver-preview+json', - 'User-Agent': 'lilydjwg/nvchecker', } if 'NVCHECKER_GITHUB_TOKEN' in os.environ: headers['Authorization'] = 'token %s' % os.environ['NVCHECKER_GITHUB_TOKEN'] diff --git a/nvchecker/source/httpclient.py b/nvchecker/source/httpclient.py new file mode 100644 index 0000000..a1666e1 --- /dev/null +++ b/nvchecker/source/httpclient.py @@ -0,0 +1 @@ +DEFAULT_USER_AGENT = 'lilydjwg/nvchecker' diff --git a/nvchecker/source/tornado_httpclient.py b/nvchecker/source/tornado_httpclient.py index dcf4ae8..74ab246 100644 --- a/nvchecker/source/tornado_httpclient.py +++ b/nvchecker/source/tornado_httpclient.py @@ -15,6 +15,8 @@ try: except ImportError: pycurl = None +from .httpclient import DEFAULT_USER_AGENT + __all__ = ['session', 'HTTPError', 'NetworkErrors'] client = AsyncHTTPClient() @@ -51,6 +53,7 @@ class Session: q = urlencode(params) url += '?' + q + kwargs.setdefault("headers", {}).setdefault('User-Agent', DEFAULT_USER_AGENT) r = HTTPRequest(url, **kwargs) return ResponseManager(r) diff --git a/setup.py b/setup.py index 8853f61..8446e8f 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup( tests_require = [ 'pytest', 'pytest-asyncio', + 'pytest-httpbin', 'flaky', ], entry_points = { diff --git a/tests/test_regex.py b/tests/test_regex.py index 0c1ac72..ee57c8a 100644 --- a/tests/test_regex.py +++ b/tests/test_regex.py @@ -4,12 +4,16 @@ import pytest pytestmark = pytest.mark.asyncio -@pytest.mark.skipif(True, - reason='httpbin is overloaded?') -async def test_regex_httpbin(get_version): +async def test_regex_httpbin_default_user_agent(get_version, httpbin): assert await get_version("example", { - "url": "https://httpbin.org/get", - "regex": r'"User-Agent": "(\w+)"', + "url": httpbin.url + "/get", + "regex": r'"User-Agent":\s*"([^"]+)"', + }) == "lilydjwg/nvchecker" + +async def test_regex_httpbin(get_version, httpbin): + assert await get_version("example", { + "url": httpbin.url + "/get", + "regex": r'"User-Agent":\s*"([^"]+)"', "user_agent": "Meow", }) == "Meow"