Merge pull request #73 from Xuanwo/gitlab-without-token

Add rate limit check support for gitlab

nvchecker/source/gitlab.py

@@ -2,11 +2,12 @@
 # Copyright (c) 2013-2018 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import os
+import time
 import urllib.parse
 
 import structlog
 
-from . import session
+from . import session, HTTPError
 from ..sortversion import sort_version_keys
 
 logger = structlog.get_logger(logger_name=__name__)
@@ -15,6 +16,12 @@ GITLAB_URL = 'https://%s/api/v4/projects/%s/repository/commits?ref_name=%s'
 GITLAB_MAX_TAG = 'https://%s/api/v4/projects/%s/repository/tags'
 
 async def get_version(name, conf, **kwargs):
+  try:
+    return await get_version_real(name, conf, **kwargs)
+  except HTTPError as e:
+    check_ratelimit(e, name)
+
+async def get_version_real(name, conf, **kwargs):
   repo = urllib.parse.quote_plus(conf.get('gitlab'))
   br = conf.get('branch', 'master')
   host = conf.get('host', "gitlab.com")
@@ -22,26 +29,27 @@ async def get_version(name, conf, **kwargs):
   ignored_tags = conf.get("ignored_tags", "").split()
   sort_version_key = sort_version_keys[conf.get("sort_version_key", "parse_version")]
 
-  token = conf.get('token')
-
-  if token is None:
-    env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_")
-    global_key = os.environ.get(env_name)
-    if not global_key:
-      key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_")
-      global_key = kwargs['keyman'].get_key(key_name)
-    token = global_key
-
-  if token is None:
-    logger.error('No gitlab token specified.', name=name)
-    return
-
   if use_max_tag:
     url = GITLAB_MAX_TAG % (host, repo)
   else:
     url = GITLAB_URL % (host, repo, br)
 
-  headers = {"PRIVATE-TOKEN": token}
+  # Load token from config
+  token = conf.get('token')
+  # Load token from environ
+  if token is None:
+    env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_")
+    token = os.environ.get(env_name)
+  # Load token from keyman
+  if token is None and 'keyman' in kwargs:
+    key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_")
+    token = kwargs['keyman'].get_key(key_name)
+
+  # Set private token if token is exist.
+  headers = {}
+  if token:
+    headers["PRIVATE-TOKEN"] = token
+
   async with session.get(url, headers=headers) as res:
     data = await res.json()
   if use_max_tag:
@@ -51,3 +59,13 @@ async def get_version(name, conf, **kwargs):
   else:
     version = data[0]['created_at'].split('T', 1)[0].replace('-', '')
   return version
+
+def check_ratelimit(exc, name):
+  res = exc.response
+  n = int(res.headers.get('RateLimit-Remaining'))
+  if n == 0:
+    logger.error('rate limited, resetting at (unknown). '
+                 'Or get an API token to increase the allowance if not yet',
+                 name = name)
+  else:
+    raise

tests/test_gitlab_local.py

@@ -0,0 +1,38 @@
+# MIT licensed
+# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+
+import os
+import pytest
+import contextlib
+pytestmark = [pytest.mark.asyncio,
+              pytest.mark.skipif(os.environ.get('TRAVIS') == 'true',
+                                 reason="rate-limited per IP")]
+
+@contextlib.contextmanager
+def unset_gitlab_token_env():
+  token = os.environ.get('NVCHECKER_GITLAB_TOKEN_GITLAB_COM')
+  try:
+    if token:
+      del os.environ['NVCHECKER_GITLAB_TOKEN_GITLAB_COM']
+    yield token
+  finally:
+    if token:
+      os.environ['NVCHECKER_GITLAB_TOKEN_GITLAB_COM'] = token
+
+async def test_gitlab(get_version):
+    with unset_gitlab_token_env():
+        ver = await get_version("example",
+                                {"gitlab": "gitlab-org/gitlab-test"})
+        assert len(ver) == 8
+        assert ver.isdigit()
+
+async def test_gitlab_max_tag(get_version):
+    with unset_gitlab_token_env():
+        assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1}) == "v1.1.0"
+
+async def test_gitlab_max_tag_with_ignored_tags(get_version):
+    with unset_gitlab_token_env():
+        ver = await get_version("example",
+                                {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, "ignored_tags": "v1.1.0"})
+        assert ver == "v1.0.0"
+