BioArchLinux/nvchecker
2
0
Fork 0
mirror of https://github.com/lilydjwg/nvchecker.git synced 2025-03-10 06:14:02 +00:00
Code Issues Projects Releases Packages Wiki Activity

i

Browse source
This commit is contained in:
involution 2024-11-19 05:35:01 -05:00
parent 267c00fb5e
commit 3f1f5ce118
1 changed files with 192 additions and 243 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

435
nvchecker_source/github.py
View file

@ -19,283 +19,232 @@ RATE_LIMITED_ERROR = False
GITHUB_URL = 'https://api.%s/repos/%s/commits' GITHUB_URL = 'https://api.%s/repos/%s/commits'
GITHUB_LATEST_RELEASE = 'https://api.%s/repos/%s/releases/latest' GITHUB_LATEST_RELEASE = 'https://api.%s/repos/%s/releases/latest'
# https://developer.github.com/v3/git/refs/#get-all-references
GITHUB_MAX_TAG = 'https://api.%s/repos/%s/git/refs/tags' GITHUB_MAX_TAG = 'https://api.%s/repos/%s/git/refs/tags'
GITHUB_GRAPHQL_URL = 'https://api.%s/graphql' GITHUB_GRAPHQL_URL = 'https://api.%s/graphql'
async def get_version(name, conf, **kwargs): async def enhance_version_with_commit_info(
global RATE_LIMITED_ERROR, ALLOW_REQUEST result: RichResult,
host: str,
if RATE_LIMITED_ERROR: repo: str,
raise RuntimeError('rate limited') headers: dict,
use_commit_info: bool
if ALLOW_REQUEST is None: ) -> RichResult:
ALLOW_REQUEST = asyncio.Event() """Add commit count and SHA to version if use_commit_info is True."""
ALLOW_REQUEST.set() if not use_commit_info:
return result
for _ in range(2): # retry once
try: url = GITHUB_URL % (host, repo)
await ALLOW_REQUEST.wait() commit_count = await get_commit_count(url, headers)
return await get_version_real(name, conf, **kwargs)
except HTTPError as e:
if e.code in [403, 429]:
if n := check_ratelimit(e, name):
ALLOW_REQUEST.clear()
await asyncio.sleep(n+1)
ALLOW_REQUEST.set()
continue
RATE_LIMITED_ERROR = True
raise
QUERY_LATEST_TAG = '''
{{
repository(name: "{name}", owner: "{owner}") {{
refs(refPrefix: "refs/tags/", first: 1,
query: "{query}",
orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{
edges {{
node {{
name
target {{
oid
}}
}}
}}
}}
}}
}}
'''
QUERY_LATEST_RELEASE_WITH_PRERELEASES = '''
{{
repository(name: "{name}", owner: "{owner}") {{
releases(first: 1, orderBy: {{field: CREATED_AT, direction: DESC}}) {{
edges {{
node {{
name
url
tag {{
name
}}
tagCommit {{
oid
}}
}}
}}
}}
}}
}}
'''
async def get_commit_count(url: str, headers: dict) -> int:
"""Get the total commit count using pagination."""
params = {'per_page': '1'}
response = await session.get( # Create new version string with commit info
url, enhanced_version = f"{result.version}.r{commit_count}.g{result.revision[:9]}"
params=params,
headers=headers return RichResult(
version=enhanced_version,
gitref=result.gitref,
revision=result.revision,
url=result.url
) )
commit_count = 1
if 'Link' in response.headers:
link_header = response.headers['Link']
for link in link_header.split(', '):
if 'rel="last"' in link:
url = link[link.find("<") + 1:link.find(">")]
query_params = parse_qs(urlparse(url).query)
if 'page' in query_params:
commit_count = int(query_params['page'][0])
break
return commit_count
async def get_latest_tag(key: Tuple[str, str, str, str]) -> RichResult: async def get_latest_tag(key: Tuple[str, str, str, str]) -> RichResult:
host, repo, query, token = key host, repo, query, token = key
owner, reponame = repo.split('/') owner, reponame = repo.split('/')
headers = { headers = {
'Authorization': f'bearer {token}', 'Authorization': f'bearer {token}',
'Content-Type': 'application/json', 'Content-Type': 'application/json',
} }
q = QUERY_LATEST_TAG.format( q = QUERY_LATEST_TAG.format(
owner = owner, owner=owner,
name = reponame, name=reponame,
query = query, query=query,
) )
res = await session.post( res = await session.post(
GITHUB_GRAPHQL_URL % host, GITHUB_GRAPHQL_URL % host,
headers = headers, headers=headers,
json = {'query': q}, json={'query': q},
) )
j = res.json() j = res.json()
refs = j['data']['repository']['refs']['edges'] refs = j['data']['repository']['refs']['edges']
if not refs: if not refs:
raise GetVersionError('no tag found') raise GetVersionError('no tag found')
version = refs[0]['node']['name'] version = refs[0]['node']['name']
revision = refs[0]['node']['target']['oid'] revision = refs[0]['node']['target']['oid']
return RichResult( return RichResult(
version = version, version=version,
gitref = f"refs/tags/{version}", gitref=f"refs/tags/{version}",
revision = revision, revision=revision,
url = f'https://github.com/{repo}/releases/tag/{version}', url=f'https://github.com/{repo}/releases/tag/{version}',
) )
async def get_latest_release_with_prereleases(key: Tuple[str, str, str, str]) -> RichResult: async def get_latest_release_with_prereleases(key: Tuple[str, str, str, str]) -> RichResult:
host, repo, token, use_release_name = key host, repo, token, use_release_name = key
owner, reponame = repo.split('/') owner, reponame = repo.split('/')
headers = { headers = {
'Authorization': f'bearer {token}', 'Authorization': f'bearer {token}',
'Content-Type': 'application/json', 'Content-Type': 'application/json',
} }
q = QUERY_LATEST_RELEASE_WITH_PRERELEASES.format( q = QUERY_LATEST_RELEASE_WITH_PRERELEASES.format(
owner = owner, owner=owner,
name = reponame, name=reponame,
) )
res = await session.post( res = await session.post(
GITHUB_GRAPHQL_URL % host, GITHUB_GRAPHQL_URL % host,
headers = headers, headers=headers,
json = {'query': q}, json={'query': q},
) )
j = res.json() j = res.json()
refs = j['data']['repository']['releases']['edges'] refs = j['data']['repository']['releases']['edges']
if not refs: if not refs:
raise GetVersionError('no release found') raise GetVersionError('no release found')
tag_name = refs[0]['node']['tag']['name'] tag_name = refs[0]['node']['tag']['name']
if use_release_name: if use_release_name:
version = refs[0]['node']['name'] version = refs[0]['node']['name']
else: else:
version = tag_name version = tag_name
return RichResult( return RichResult(
version = version, version=version,
gitref = f"refs/tags/{tag_name}", gitref=f"refs/tags/{tag_name}",
revision = refs[0]['node']['tagCommit']['oid'], revision=refs[0]['node']['tagCommit']['oid'],
url = refs[0]['node']['url'], url=refs[0]['node']['url'],
) )
async def get_version_real( async def get_version_real(
name: str, conf: Entry, *, name: str, conf: Entry, *,
cache: AsyncCache, keymanager: KeyManager, cache: AsyncCache, keymanager: KeyManager,
**kwargs, **kwargs,
) -> VersionResult: ) -> VersionResult:
repo = conf['github'] repo = conf['github']
host = conf.get('host', "github.com") host = conf.get('host', "github.com")
use_commit_info = conf.get('use_commit_info', False)
# Load token from config # Load token from config
token = conf.get('token') token = conf.get('token')
# Load token from keyman # Load token from keyman
if token is None: if token is None:
token = keymanager.get_key(host.lower(), 'github') token = keymanager.get_key(host.lower(), 'github')
use_latest_tag = conf.get('use_latest_tag', False) headers = {
if use_latest_tag: 'Accept': 'application/vnd.github.quicksilver-preview+json',
if not token: }
raise GetVersionError('token not given but it is required') if token:
headers['Authorization'] = f'token {token}'
query = conf.get('query', '') use_latest_tag = conf.get('use_latest_tag', False)
return await cache.get((host, repo, query, token), get_latest_tag) # type: ignore if use_latest_tag:
if not token:
raise GetVersionError('token not given but it is required')
use_latest_release = conf.get('use_latest_release', False) query = conf.get('query', '')
include_prereleases = conf.get('include_prereleases', False) result = await cache.get((host, repo, query, token), get_latest_tag)
use_release_name = conf.get('use_release_name', False) return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
if use_latest_release and include_prereleases:
if not token:
raise GetVersionError('token not given but it is required')
return await cache.get( use_latest_release = conf.get('use_latest_release', False)
(host, repo, token, use_release_name), include_prereleases = conf.get('include_prereleases', False)
get_latest_release_with_prereleases) # type: ignore use_release_name = conf.get('use_release_name', False)
if use_latest_release and include_prereleases:
if not token:
raise GetVersionError('token not given but it is required')
br = conf.get('branch') result = await cache.get(
path = conf.get('path') (host, repo, token, use_release_name),
use_max_tag = conf.get('use_max_tag', False) get_latest_release_with_prereleases)
if use_latest_release: return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
url = GITHUB_LATEST_RELEASE % (host, repo)
elif use_max_tag:
url = GITHUB_MAX_TAG % (host, repo)
else:
url = GITHUB_URL % (host, repo)
parameters = {}
if br:
parameters['sha'] = br
if path:
parameters['path'] = path
url += '?' + urlencode(parameters)
headers = {
'Accept': 'application/vnd.github.quicksilver-preview+json',
}
if token:
headers['Authorization'] = f'token {token}'
data = await cache.get_json(url, headers = headers) br = conf.get('branch')
path = conf.get('path')
if use_max_tag: use_max_tag = conf.get('use_max_tag', False)
tags: List[Union[str, RichResult]] = [ if use_latest_release:
RichResult( url = GITHUB_LATEST_RELEASE % (host, repo)
version = ref['ref'].split('/', 2)[-1], elif use_max_tag:
gitref = ref['ref'], url = GITHUB_MAX_TAG % (host, repo)
revision = ref['object']['sha'],
url = f'https://github.com/{repo}/releases/tag/{ref["ref"].split("/", 2)[-1]}',
) for ref in data
]
if not tags:
raise GetVersionError('No tag found in upstream repository.')
return tags
if use_latest_release:
if 'tag_name' not in data:
raise GetVersionError('No release found in upstream repository.')
if use_release_name:
version = data['name']
else: else:
version = data['tag_name'] url = GITHUB_URL % (host, repo)
parameters = {}
if br:
parameters['sha'] = br
if path:
parameters['path'] = path
url += '?' + urlencode(parameters)
return RichResult( data = await cache.get_json(url, headers=headers)
version = version,
gitref = f"refs/tags/{data['tag_name']}",
url = data['html_url'],
)
else: if use_max_tag:
tags: List[Union[str, RichResult]] = [
RichResult(
version=ref['ref'].split('/', 2)[-1],
gitref=ref['ref'],
revision=ref['object']['sha'],
url=f'https://github.com/{repo}/releases/tag/{ref["ref"].split("/", 2)[-1]}',
) for ref in data
]
if not tags:
raise GetVersionError('No tag found in upstream repository.')
# Enhance all tags with commit info if enabled
if use_commit_info:
enhanced_tags = []
for tag in tags:
if isinstance(tag, RichResult):
enhanced_tag = await enhance_version_with_commit_info(
tag, host, repo, headers, use_commit_info
)
enhanced_tags.append(enhanced_tag)
else:
enhanced_tags.append(tag)
return enhanced_tags
return tags
version = data[0]['commit']['committer']['date'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.') if use_latest_release:
if 'tag_name' not in data:
raise GetVersionError('No release found in upstream repository.')
# Only add commit info if configured if use_release_name:
if conf.get('use_commit_info', False): version = data['name']
commit_count = await get_commit_count(url, headers) else:
version = f"{version}.r{commit_count}.g{data[0]['sha'][:9]}" version = data['tag_name']
return RichResult( result = RichResult(
# YYYYMMDD.HHMMSS version=version,
version = version, gitref=f"refs/tags/{data['tag_name']}",
revision = data[0]['sha'], url=data['html_url'],
url = data[0]['html_url'], )
) return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
else:
version = data[0]['commit']['committer']['date'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.')
result = RichResult(
version=version,
revision=data[0]['sha'],
url=data[0]['html_url'],
)
return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
def check_ratelimit(exc: HTTPError, name: str) -> Optional[int]: def check_ratelimit(exc: HTTPError, name: str) -> Optional[int]:
res = exc.response res = exc.response
if not res: if not res:
raise exc raise exc
if v := res.headers.get('retry-after'): if v := res.headers.get('retry-after'):
n = int(v) n = int(v)
logger.warning('retry-after', n=n) logger.warning('retry-after', n=n)
return n return n
# default -1 is used to re-raise the exception # default -1 is used to re-raise the exception
n = int(res.headers.get('X-RateLimit-Remaining', -1)) n = int(res.headers.get('X-RateLimit-Remaining', -1))
if n == 0: if n == 0:
reset = int(res.headers.get('X-RateLimit-Reset')) reset = int(res.headers.get('X-RateLimit-Reset'))
logger.error(f'rate limited, resetting at {time.ctime(reset)}. ' logger.error(f'rate limited, resetting at {time.ctime(reset)}. '
'Or get an API token to increase the allowance if not yet', 'Or get an API token to increase the allowance if not yet',
name = name, name=name,
reset = reset) reset=reset)
return None return None
raise exc raise exc