From 3f1f5ce11800850019a13f7079365b97125aaca1 Mon Sep 17 00:00:00 2001 From: involution Date: Tue, 19 Nov 2024 05:35:01 -0500 Subject: [PATCH] i --- nvchecker_source/github.py | 435 ++++++++++++++++--------------------- 1 file changed, 192 insertions(+), 243 deletions(-) diff --git a/nvchecker_source/github.py b/nvchecker_source/github.py index a310e50..cbeebab 100644 --- a/nvchecker_source/github.py +++ b/nvchecker_source/github.py @@ -19,283 +19,232 @@ RATE_LIMITED_ERROR = False GITHUB_URL = 'https://api.%s/repos/%s/commits' GITHUB_LATEST_RELEASE = 'https://api.%s/repos/%s/releases/latest' -# https://developer.github.com/v3/git/refs/#get-all-references GITHUB_MAX_TAG = 'https://api.%s/repos/%s/git/refs/tags' GITHUB_GRAPHQL_URL = 'https://api.%s/graphql' -async def get_version(name, conf, **kwargs): - global RATE_LIMITED_ERROR, ALLOW_REQUEST - - if RATE_LIMITED_ERROR: - raise RuntimeError('rate limited') - - if ALLOW_REQUEST is None: - ALLOW_REQUEST = asyncio.Event() - ALLOW_REQUEST.set() - - for _ in range(2): # retry once - try: - await ALLOW_REQUEST.wait() - return await get_version_real(name, conf, **kwargs) - except HTTPError as e: - if e.code in [403, 429]: - if n := check_ratelimit(e, name): - ALLOW_REQUEST.clear() - await asyncio.sleep(n+1) - ALLOW_REQUEST.set() - continue - RATE_LIMITED_ERROR = True - raise - -QUERY_LATEST_TAG = ''' -{{ - repository(name: "{name}", owner: "{owner}") {{ - refs(refPrefix: "refs/tags/", first: 1, - query: "{query}", - orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{ - edges {{ - node {{ - name - target {{ - oid - }} - }} - }} - }} - }} -}} -''' - -QUERY_LATEST_RELEASE_WITH_PRERELEASES = ''' -{{ - repository(name: "{name}", owner: "{owner}") {{ - releases(first: 1, orderBy: {{field: CREATED_AT, direction: DESC}}) {{ - edges {{ - node {{ - name - url - tag {{ - name - }} - tagCommit {{ - oid - }} - }} - }} - }} - }} -}} -''' -async def get_commit_count(url: str, headers: dict) -> int: - """Get the total commit count using pagination.""" - params = {'per_page': '1'} +async def enhance_version_with_commit_info( + result: RichResult, + host: str, + repo: str, + headers: dict, + use_commit_info: bool +) -> RichResult: + """Add commit count and SHA to version if use_commit_info is True.""" + if not use_commit_info: + return result + + url = GITHUB_URL % (host, repo) + commit_count = await get_commit_count(url, headers) - response = await session.get( - url, - params=params, - headers=headers + # Create new version string with commit info + enhanced_version = f"{result.version}.r{commit_count}.g{result.revision[:9]}" + + return RichResult( + version=enhanced_version, + gitref=result.gitref, + revision=result.revision, + url=result.url ) - - commit_count = 1 - if 'Link' in response.headers: - link_header = response.headers['Link'] - for link in link_header.split(', '): - if 'rel="last"' in link: - url = link[link.find("<") + 1:link.find(">")] - query_params = parse_qs(urlparse(url).query) - if 'page' in query_params: - commit_count = int(query_params['page'][0]) - break - - return commit_count async def get_latest_tag(key: Tuple[str, str, str, str]) -> RichResult: - host, repo, query, token = key - owner, reponame = repo.split('/') - headers = { - 'Authorization': f'bearer {token}', - 'Content-Type': 'application/json', - } - q = QUERY_LATEST_TAG.format( - owner = owner, - name = reponame, - query = query, - ) + host, repo, query, token = key + owner, reponame = repo.split('/') + headers = { + 'Authorization': f'bearer {token}', + 'Content-Type': 'application/json', + } + q = QUERY_LATEST_TAG.format( + owner=owner, + name=reponame, + query=query, + ) - res = await session.post( - GITHUB_GRAPHQL_URL % host, - headers = headers, - json = {'query': q}, - ) - j = res.json() + res = await session.post( + GITHUB_GRAPHQL_URL % host, + headers=headers, + json={'query': q}, + ) + j = res.json() - refs = j['data']['repository']['refs']['edges'] - if not refs: - raise GetVersionError('no tag found') + refs = j['data']['repository']['refs']['edges'] + if not refs: + raise GetVersionError('no tag found') - version = refs[0]['node']['name'] - revision = refs[0]['node']['target']['oid'] - return RichResult( - version = version, - gitref = f"refs/tags/{version}", - revision = revision, - url = f'https://github.com/{repo}/releases/tag/{version}', - ) + version = refs[0]['node']['name'] + revision = refs[0]['node']['target']['oid'] + return RichResult( + version=version, + gitref=f"refs/tags/{version}", + revision=revision, + url=f'https://github.com/{repo}/releases/tag/{version}', + ) async def get_latest_release_with_prereleases(key: Tuple[str, str, str, str]) -> RichResult: - host, repo, token, use_release_name = key - owner, reponame = repo.split('/') - headers = { - 'Authorization': f'bearer {token}', - 'Content-Type': 'application/json', - } - q = QUERY_LATEST_RELEASE_WITH_PRERELEASES.format( - owner = owner, - name = reponame, - ) + host, repo, token, use_release_name = key + owner, reponame = repo.split('/') + headers = { + 'Authorization': f'bearer {token}', + 'Content-Type': 'application/json', + } + q = QUERY_LATEST_RELEASE_WITH_PRERELEASES.format( + owner=owner, + name=reponame, + ) - res = await session.post( - GITHUB_GRAPHQL_URL % host, - headers = headers, - json = {'query': q}, - ) - j = res.json() + res = await session.post( + GITHUB_GRAPHQL_URL % host, + headers=headers, + json={'query': q}, + ) + j = res.json() - refs = j['data']['repository']['releases']['edges'] - if not refs: - raise GetVersionError('no release found') + refs = j['data']['repository']['releases']['edges'] + if not refs: + raise GetVersionError('no release found') - tag_name = refs[0]['node']['tag']['name'] - if use_release_name: - version = refs[0]['node']['name'] - else: - version = tag_name + tag_name = refs[0]['node']['tag']['name'] + if use_release_name: + version = refs[0]['node']['name'] + else: + version = tag_name - return RichResult( - version = version, - gitref = f"refs/tags/{tag_name}", - revision = refs[0]['node']['tagCommit']['oid'], - url = refs[0]['node']['url'], - ) + return RichResult( + version=version, + gitref=f"refs/tags/{tag_name}", + revision=refs[0]['node']['tagCommit']['oid'], + url=refs[0]['node']['url'], + ) async def get_version_real( - name: str, conf: Entry, *, - cache: AsyncCache, keymanager: KeyManager, - **kwargs, + name: str, conf: Entry, *, + cache: AsyncCache, keymanager: KeyManager, + **kwargs, ) -> VersionResult: - repo = conf['github'] - host = conf.get('host', "github.com") + repo = conf['github'] + host = conf.get('host', "github.com") + use_commit_info = conf.get('use_commit_info', False) - # Load token from config - token = conf.get('token') - # Load token from keyman - if token is None: - token = keymanager.get_key(host.lower(), 'github') + # Load token from config + token = conf.get('token') + # Load token from keyman + if token is None: + token = keymanager.get_key(host.lower(), 'github') - use_latest_tag = conf.get('use_latest_tag', False) - if use_latest_tag: - if not token: - raise GetVersionError('token not given but it is required') + headers = { + 'Accept': 'application/vnd.github.quicksilver-preview+json', + } + if token: + headers['Authorization'] = f'token {token}' - query = conf.get('query', '') - return await cache.get((host, repo, query, token), get_latest_tag) # type: ignore + use_latest_tag = conf.get('use_latest_tag', False) + if use_latest_tag: + if not token: + raise GetVersionError('token not given but it is required') - use_latest_release = conf.get('use_latest_release', False) - include_prereleases = conf.get('include_prereleases', False) - use_release_name = conf.get('use_release_name', False) - if use_latest_release and include_prereleases: - if not token: - raise GetVersionError('token not given but it is required') + query = conf.get('query', '') + result = await cache.get((host, repo, query, token), get_latest_tag) + return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info) - return await cache.get( - (host, repo, token, use_release_name), - get_latest_release_with_prereleases) # type: ignore + use_latest_release = conf.get('use_latest_release', False) + include_prereleases = conf.get('include_prereleases', False) + use_release_name = conf.get('use_release_name', False) + if use_latest_release and include_prereleases: + if not token: + raise GetVersionError('token not given but it is required') - br = conf.get('branch') - path = conf.get('path') - use_max_tag = conf.get('use_max_tag', False) - if use_latest_release: - url = GITHUB_LATEST_RELEASE % (host, repo) - elif use_max_tag: - url = GITHUB_MAX_TAG % (host, repo) - else: - url = GITHUB_URL % (host, repo) - parameters = {} - if br: - parameters['sha'] = br - if path: - parameters['path'] = path - url += '?' + urlencode(parameters) - headers = { - 'Accept': 'application/vnd.github.quicksilver-preview+json', - } - if token: - headers['Authorization'] = f'token {token}' + result = await cache.get( + (host, repo, token, use_release_name), + get_latest_release_with_prereleases) + return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info) - data = await cache.get_json(url, headers = headers) - - if use_max_tag: - tags: List[Union[str, RichResult]] = [ - RichResult( - version = ref['ref'].split('/', 2)[-1], - gitref = ref['ref'], - revision = ref['object']['sha'], - url = f'https://github.com/{repo}/releases/tag/{ref["ref"].split("/", 2)[-1]}', - ) for ref in data - ] - if not tags: - raise GetVersionError('No tag found in upstream repository.') - return tags - - if use_latest_release: - if 'tag_name' not in data: - raise GetVersionError('No release found in upstream repository.') - - if use_release_name: - version = data['name'] + br = conf.get('branch') + path = conf.get('path') + use_max_tag = conf.get('use_max_tag', False) + if use_latest_release: + url = GITHUB_LATEST_RELEASE % (host, repo) + elif use_max_tag: + url = GITHUB_MAX_TAG % (host, repo) else: - version = data['tag_name'] + url = GITHUB_URL % (host, repo) + parameters = {} + if br: + parameters['sha'] = br + if path: + parameters['path'] = path + url += '?' + urlencode(parameters) - return RichResult( - version = version, - gitref = f"refs/tags/{data['tag_name']}", - url = data['html_url'], - ) + data = await cache.get_json(url, headers=headers) - else: + if use_max_tag: + tags: List[Union[str, RichResult]] = [ + RichResult( + version=ref['ref'].split('/', 2)[-1], + gitref=ref['ref'], + revision=ref['object']['sha'], + url=f'https://github.com/{repo}/releases/tag/{ref["ref"].split("/", 2)[-1]}', + ) for ref in data + ] + if not tags: + raise GetVersionError('No tag found in upstream repository.') + + # Enhance all tags with commit info if enabled + if use_commit_info: + enhanced_tags = [] + for tag in tags: + if isinstance(tag, RichResult): + enhanced_tag = await enhance_version_with_commit_info( + tag, host, repo, headers, use_commit_info + ) + enhanced_tags.append(enhanced_tag) + else: + enhanced_tags.append(tag) + return enhanced_tags + return tags - version = data[0]['commit']['committer']['date'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.') + if use_latest_release: + if 'tag_name' not in data: + raise GetVersionError('No release found in upstream repository.') - # Only add commit info if configured - if conf.get('use_commit_info', False): - commit_count = await get_commit_count(url, headers) - version = f"{version}.r{commit_count}.g{data[0]['sha'][:9]}" + if use_release_name: + version = data['name'] + else: + version = data['tag_name'] - return RichResult( - # YYYYMMDD.HHMMSS - version = version, - revision = data[0]['sha'], - url = data[0]['html_url'], - ) + result = RichResult( + version=version, + gitref=f"refs/tags/{data['tag_name']}", + url=data['html_url'], + ) + return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info) + + else: + version = data[0]['commit']['committer']['date'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.') + + result = RichResult( + version=version, + revision=data[0]['sha'], + url=data[0]['html_url'], + ) + return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info) def check_ratelimit(exc: HTTPError, name: str) -> Optional[int]: - res = exc.response - if not res: - raise exc + res = exc.response + if not res: + raise exc - if v := res.headers.get('retry-after'): - n = int(v) - logger.warning('retry-after', n=n) - return n + if v := res.headers.get('retry-after'): + n = int(v) + logger.warning('retry-after', n=n) + return n - # default -1 is used to re-raise the exception - n = int(res.headers.get('X-RateLimit-Remaining', -1)) - if n == 0: - reset = int(res.headers.get('X-RateLimit-Reset')) - logger.error(f'rate limited, resetting at {time.ctime(reset)}. ' - 'Or get an API token to increase the allowance if not yet', - name = name, - reset = reset) - return None + # default -1 is used to re-raise the exception + n = int(res.headers.get('X-RateLimit-Remaining', -1)) + if n == 0: + reset = int(res.headers.get('X-RateLimit-Reset')) + logger.error(f'rate limited, resetting at {time.ctime(reset)}. ' + 'Or get an API token to increase the allowance if not yet', + name=name, + reset=reset) + return None - raise exc + raise exc \ No newline at end of file