mirror of
https://github.com/lilydjwg/nvchecker.git
synced 2025-03-10 06:14:02 +00:00
i
This commit is contained in:
involution
parent
267c00fb5e
commit
3f1f5ce118
1 changed files with 192 additions and 243 deletions
|
|
@ -19,283 +19,232 @@ RATE_LIMITED_ERROR = False
|
|||
|
||||
GITHUB_URL = 'https://api.%s/repos/%s/commits'
|
||||
GITHUB_LATEST_RELEASE = 'https://api.%s/repos/%s/releases/latest'
|
||||
# https://developer.github.com/v3/git/refs/#get-all-references
|
||||
GITHUB_MAX_TAG = 'https://api.%s/repos/%s/git/refs/tags'
|
||||
GITHUB_GRAPHQL_URL = 'https://api.%s/graphql'
|
||||
|
||||
async def get_version(name, conf, **kwargs):
|
||||
global RATE_LIMITED_ERROR, ALLOW_REQUEST
|
||||
|
||||
if RATE_LIMITED_ERROR:
|
||||
raise RuntimeError('rate limited')
|
||||
|
||||
if ALLOW_REQUEST is None:
|
||||
ALLOW_REQUEST = asyncio.Event()
|
||||
ALLOW_REQUEST.set()
|
||||
|
||||
for _ in range(2): # retry once
|
||||
try:
|
||||
await ALLOW_REQUEST.wait()
|
||||
return await get_version_real(name, conf, **kwargs)
|
||||
except HTTPError as e:
|
||||
if e.code in [403, 429]:
|
||||
if n := check_ratelimit(e, name):
|
||||
ALLOW_REQUEST.clear()
|
||||
await asyncio.sleep(n+1)
|
||||
ALLOW_REQUEST.set()
|
||||
continue
|
||||
RATE_LIMITED_ERROR = True
|
||||
raise
|
||||
|
||||
QUERY_LATEST_TAG = '''
|
||||
{{
|
||||
repository(name: "{name}", owner: "{owner}") {{
|
||||
refs(refPrefix: "refs/tags/", first: 1,
|
||||
query: "{query}",
|
||||
orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{
|
||||
edges {{
|
||||
node {{
|
||||
name
|
||||
target {{
|
||||
oid
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
'''
|
||||
|
||||
QUERY_LATEST_RELEASE_WITH_PRERELEASES = '''
|
||||
{{
|
||||
repository(name: "{name}", owner: "{owner}") {{
|
||||
releases(first: 1, orderBy: {{field: CREATED_AT, direction: DESC}}) {{
|
||||
edges {{
|
||||
node {{
|
||||
name
|
||||
url
|
||||
tag {{
|
||||
name
|
||||
}}
|
||||
tagCommit {{
|
||||
oid
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
'''
|
||||
async def get_commit_count(url: str, headers: dict) -> int:
|
||||
"""Get the total commit count using pagination."""
|
||||
params = {'per_page': '1'}
|
||||
async def enhance_version_with_commit_info(
|
||||
result: RichResult,
|
||||
host: str,
|
||||
repo: str,
|
||||
headers: dict,
|
||||
use_commit_info: bool
|
||||
) -> RichResult:
|
||||
"""Add commit count and SHA to version if use_commit_info is True."""
|
||||
if not use_commit_info:
|
||||
return result
|
||||
|
||||
url = GITHUB_URL % (host, repo)
|
||||
commit_count = await get_commit_count(url, headers)
|
||||
|
||||
response = await session.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers
|
||||
# Create new version string with commit info
|
||||
enhanced_version = f"{result.version}.r{commit_count}.g{result.revision[:9]}"
|
||||
|
||||
return RichResult(
|
||||
version=enhanced_version,
|
||||
gitref=result.gitref,
|
||||
revision=result.revision,
|
||||
url=result.url
|
||||
)
|
||||
|
||||
commit_count = 1
|
||||
if 'Link' in response.headers:
|
||||
link_header = response.headers['Link']
|
||||
for link in link_header.split(', '):
|
||||
if 'rel="last"' in link:
|
||||
url = link[link.find("<") + 1:link.find(">")]
|
||||
query_params = parse_qs(urlparse(url).query)
|
||||
if 'page' in query_params:
|
||||
commit_count = int(query_params['page'][0])
|
||||
break
|
||||
|
||||
return commit_count
|
||||
|
||||
async def get_latest_tag(key: Tuple[str, str, str, str]) -> RichResult:
|
||||
host, repo, query, token = key
|
||||
owner, reponame = repo.split('/')
|
||||
headers = {
|
||||
'Authorization': f'bearer {token}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
q = QUERY_LATEST_TAG.format(
|
||||
owner = owner,
|
||||
name = reponame,
|
||||
query = query,
|
||||
)
|
||||
host, repo, query, token = key
|
||||
owner, reponame = repo.split('/')
|
||||
headers = {
|
||||
'Authorization': f'bearer {token}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
q = QUERY_LATEST_TAG.format(
|
||||
owner=owner,
|
||||
name=reponame,
|
||||
query=query,
|
||||
)
|
||||
|
||||
res = await session.post(
|
||||
GITHUB_GRAPHQL_URL % host,
|
||||
headers = headers,
|
||||
json = {'query': q},
|
||||
)
|
||||
j = res.json()
|
||||
res = await session.post(
|
||||
GITHUB_GRAPHQL_URL % host,
|
||||
headers=headers,
|
||||
json={'query': q},
|
||||
)
|
||||
j = res.json()
|
||||
|
||||
refs = j['data']['repository']['refs']['edges']
|
||||
if not refs:
|
||||
raise GetVersionError('no tag found')
|
||||
refs = j['data']['repository']['refs']['edges']
|
||||
if not refs:
|
||||
raise GetVersionError('no tag found')
|
||||
|
||||
version = refs[0]['node']['name']
|
||||
revision = refs[0]['node']['target']['oid']
|
||||
return RichResult(
|
||||
version = version,
|
||||
gitref = f"refs/tags/{version}",
|
||||
revision = revision,
|
||||
url = f'https://github.com/{repo}/releases/tag/{version}',
|
||||
)
|
||||
version = refs[0]['node']['name']
|
||||
revision = refs[0]['node']['target']['oid']
|
||||
return RichResult(
|
||||
version=version,
|
||||
gitref=f"refs/tags/{version}",
|
||||
revision=revision,
|
||||
url=f'https://github.com/{repo}/releases/tag/{version}',
|
||||
)
|
||||
|
||||
async def get_latest_release_with_prereleases(key: Tuple[str, str, str, str]) -> RichResult:
|
||||
host, repo, token, use_release_name = key
|
||||
owner, reponame = repo.split('/')
|
||||
headers = {
|
||||
'Authorization': f'bearer {token}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
q = QUERY_LATEST_RELEASE_WITH_PRERELEASES.format(
|
||||
owner = owner,
|
||||
name = reponame,
|
||||
)
|
||||
host, repo, token, use_release_name = key
|
||||
owner, reponame = repo.split('/')
|
||||
headers = {
|
||||
'Authorization': f'bearer {token}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
q = QUERY_LATEST_RELEASE_WITH_PRERELEASES.format(
|
||||
owner=owner,
|
||||
name=reponame,
|
||||
)
|
||||
|
||||
res = await session.post(
|
||||
GITHUB_GRAPHQL_URL % host,
|
||||
headers = headers,
|
||||
json = {'query': q},
|
||||
)
|
||||
j = res.json()
|
||||
res = await session.post(
|
||||
GITHUB_GRAPHQL_URL % host,
|
||||
headers=headers,
|
||||
json={'query': q},
|
||||
)
|
||||
j = res.json()
|
||||
|
||||
refs = j['data']['repository']['releases']['edges']
|
||||
if not refs:
|
||||
raise GetVersionError('no release found')
|
||||
refs = j['data']['repository']['releases']['edges']
|
||||
if not refs:
|
||||
raise GetVersionError('no release found')
|
||||
|
||||
tag_name = refs[0]['node']['tag']['name']
|
||||
if use_release_name:
|
||||
version = refs[0]['node']['name']
|
||||
else:
|
||||
version = tag_name
|
||||
tag_name = refs[0]['node']['tag']['name']
|
||||
if use_release_name:
|
||||
version = refs[0]['node']['name']
|
||||
else:
|
||||
version = tag_name
|
||||
|
||||
return RichResult(
|
||||
version = version,
|
||||
gitref = f"refs/tags/{tag_name}",
|
||||
revision = refs[0]['node']['tagCommit']['oid'],
|
||||
url = refs[0]['node']['url'],
|
||||
)
|
||||
return RichResult(
|
||||
version=version,
|
||||
gitref=f"refs/tags/{tag_name}",
|
||||
revision=refs[0]['node']['tagCommit']['oid'],
|
||||
url=refs[0]['node']['url'],
|
||||
)
|
||||
|
||||
async def get_version_real(
|
||||
name: str, conf: Entry, *,
|
||||
cache: AsyncCache, keymanager: KeyManager,
|
||||
**kwargs,
|
||||
name: str, conf: Entry, *,
|
||||
cache: AsyncCache, keymanager: KeyManager,
|
||||
**kwargs,
|
||||
) -> VersionResult:
|
||||
repo = conf['github']
|
||||
host = conf.get('host', "github.com")
|
||||
repo = conf['github']
|
||||
host = conf.get('host', "github.com")
|
||||
use_commit_info = conf.get('use_commit_info', False)
|
||||
|
||||
# Load token from config
|
||||
token = conf.get('token')
|
||||
# Load token from keyman
|
||||
if token is None:
|
||||
token = keymanager.get_key(host.lower(), 'github')
|
||||
# Load token from config
|
||||
token = conf.get('token')
|
||||
# Load token from keyman
|
||||
if token is None:
|
||||
token = keymanager.get_key(host.lower(), 'github')
|
||||
|
||||
use_latest_tag = conf.get('use_latest_tag', False)
|
||||
if use_latest_tag:
|
||||
if not token:
|
||||
raise GetVersionError('token not given but it is required')
|
||||
headers = {
|
||||
'Accept': 'application/vnd.github.quicksilver-preview+json',
|
||||
}
|
||||
if token:
|
||||
headers['Authorization'] = f'token {token}'
|
||||
|
||||
query = conf.get('query', '')
|
||||
return await cache.get((host, repo, query, token), get_latest_tag) # type: ignore
|
||||
use_latest_tag = conf.get('use_latest_tag', False)
|
||||
if use_latest_tag:
|
||||
if not token:
|
||||
raise GetVersionError('token not given but it is required')
|
||||
|
||||
use_latest_release = conf.get('use_latest_release', False)
|
||||
include_prereleases = conf.get('include_prereleases', False)
|
||||
use_release_name = conf.get('use_release_name', False)
|
||||
if use_latest_release and include_prereleases:
|
||||
if not token:
|
||||
raise GetVersionError('token not given but it is required')
|
||||
query = conf.get('query', '')
|
||||
result = await cache.get((host, repo, query, token), get_latest_tag)
|
||||
return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
|
||||
|
||||
return await cache.get(
|
||||
(host, repo, token, use_release_name),
|
||||
get_latest_release_with_prereleases) # type: ignore
|
||||
use_latest_release = conf.get('use_latest_release', False)
|
||||
include_prereleases = conf.get('include_prereleases', False)
|
||||
use_release_name = conf.get('use_release_name', False)
|
||||
if use_latest_release and include_prereleases:
|
||||
if not token:
|
||||
raise GetVersionError('token not given but it is required')
|
||||
|
||||
br = conf.get('branch')
|
||||
path = conf.get('path')
|
||||
use_max_tag = conf.get('use_max_tag', False)
|
||||
if use_latest_release:
|
||||
url = GITHUB_LATEST_RELEASE % (host, repo)
|
||||
elif use_max_tag:
|
||||
url = GITHUB_MAX_TAG % (host, repo)
|
||||
else:
|
||||
url = GITHUB_URL % (host, repo)
|
||||
parameters = {}
|
||||
if br:
|
||||
parameters['sha'] = br
|
||||
if path:
|
||||
parameters['path'] = path
|
||||
url += '?' + urlencode(parameters)
|
||||
headers = {
|
||||
'Accept': 'application/vnd.github.quicksilver-preview+json',
|
||||
}
|
||||
if token:
|
||||
headers['Authorization'] = f'token {token}'
|
||||
result = await cache.get(
|
||||
(host, repo, token, use_release_name),
|
||||
get_latest_release_with_prereleases)
|
||||
return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
|
||||
|
||||
data = await cache.get_json(url, headers = headers)
|
||||
|
||||
if use_max_tag:
|
||||
tags: List[Union[str, RichResult]] = [
|
||||
RichResult(
|
||||
version = ref['ref'].split('/', 2)[-1],
|
||||
gitref = ref['ref'],
|
||||
revision = ref['object']['sha'],
|
||||
url = f'https://github.com/{repo}/releases/tag/{ref["ref"].split("/", 2)[-1]}',
|
||||
) for ref in data
|
||||
]
|
||||
if not tags:
|
||||
raise GetVersionError('No tag found in upstream repository.')
|
||||
return tags
|
||||
|
||||
if use_latest_release:
|
||||
if 'tag_name' not in data:
|
||||
raise GetVersionError('No release found in upstream repository.')
|
||||
|
||||
if use_release_name:
|
||||
version = data['name']
|
||||
br = conf.get('branch')
|
||||
path = conf.get('path')
|
||||
use_max_tag = conf.get('use_max_tag', False)
|
||||
if use_latest_release:
|
||||
url = GITHUB_LATEST_RELEASE % (host, repo)
|
||||
elif use_max_tag:
|
||||
url = GITHUB_MAX_TAG % (host, repo)
|
||||
else:
|
||||
version = data['tag_name']
|
||||
url = GITHUB_URL % (host, repo)
|
||||
parameters = {}
|
||||
if br:
|
||||
parameters['sha'] = br
|
||||
if path:
|
||||
parameters['path'] = path
|
||||
url += '?' + urlencode(parameters)
|
||||
|
||||
return RichResult(
|
||||
version = version,
|
||||
gitref = f"refs/tags/{data['tag_name']}",
|
||||
url = data['html_url'],
|
||||
)
|
||||
data = await cache.get_json(url, headers=headers)
|
||||
|
||||
else:
|
||||
if use_max_tag:
|
||||
tags: List[Union[str, RichResult]] = [
|
||||
RichResult(
|
||||
version=ref['ref'].split('/', 2)[-1],
|
||||
gitref=ref['ref'],
|
||||
revision=ref['object']['sha'],
|
||||
url=f'https://github.com/{repo}/releases/tag/{ref["ref"].split("/", 2)[-1]}',
|
||||
) for ref in data
|
||||
]
|
||||
if not tags:
|
||||
raise GetVersionError('No tag found in upstream repository.')
|
||||
|
||||
# Enhance all tags with commit info if enabled
|
||||
if use_commit_info:
|
||||
enhanced_tags = []
|
||||
for tag in tags:
|
||||
if isinstance(tag, RichResult):
|
||||
enhanced_tag = await enhance_version_with_commit_info(
|
||||
tag, host, repo, headers, use_commit_info
|
||||
)
|
||||
enhanced_tags.append(enhanced_tag)
|
||||
else:
|
||||
enhanced_tags.append(tag)
|
||||
return enhanced_tags
|
||||
return tags
|
||||
|
||||
version = data[0]['commit']['committer']['date'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.')
|
||||
if use_latest_release:
|
||||
if 'tag_name' not in data:
|
||||
raise GetVersionError('No release found in upstream repository.')
|
||||
|
||||
# Only add commit info if configured
|
||||
if conf.get('use_commit_info', False):
|
||||
commit_count = await get_commit_count(url, headers)
|
||||
version = f"{version}.r{commit_count}.g{data[0]['sha'][:9]}"
|
||||
if use_release_name:
|
||||
version = data['name']
|
||||
else:
|
||||
version = data['tag_name']
|
||||
|
||||
return RichResult(
|
||||
# YYYYMMDD.HHMMSS
|
||||
version = version,
|
||||
revision = data[0]['sha'],
|
||||
url = data[0]['html_url'],
|
||||
)
|
||||
result = RichResult(
|
||||
version=version,
|
||||
gitref=f"refs/tags/{data['tag_name']}",
|
||||
url=data['html_url'],
|
||||
)
|
||||
return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
|
||||
|
||||
else:
|
||||
version = data[0]['commit']['committer']['date'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.')
|
||||
|
||||
result = RichResult(
|
||||
version=version,
|
||||
revision=data[0]['sha'],
|
||||
url=data[0]['html_url'],
|
||||
)
|
||||
return await enhance_version_with_commit_info(result, host, repo, headers, use_commit_info)
|
||||
|
||||
def check_ratelimit(exc: HTTPError, name: str) -> Optional[int]:
|
||||
res = exc.response
|
||||
if not res:
|
||||
raise exc
|
||||
res = exc.response
|
||||
if not res:
|
||||
raise exc
|
||||
|
||||
if v := res.headers.get('retry-after'):
|
||||
n = int(v)
|
||||
logger.warning('retry-after', n=n)
|
||||
return n
|
||||
if v := res.headers.get('retry-after'):
|
||||
n = int(v)
|
||||
logger.warning('retry-after', n=n)
|
||||
return n
|
||||
|
||||
# default -1 is used to re-raise the exception
|
||||
n = int(res.headers.get('X-RateLimit-Remaining', -1))
|
||||
if n == 0:
|
||||
reset = int(res.headers.get('X-RateLimit-Reset'))
|
||||
logger.error(f'rate limited, resetting at {time.ctime(reset)}. '
|
||||
'Or get an API token to increase the allowance if not yet',
|
||||
name = name,
|
||||
reset = reset)
|
||||
return None
|
||||
# default -1 is used to re-raise the exception
|
||||
n = int(res.headers.get('X-RateLimit-Remaining', -1))
|
||||
if n == 0:
|
||||
reset = int(res.headers.get('X-RateLimit-Reset'))
|
||||
logger.error(f'rate limited, resetting at {time.ctime(reset)}. '
|
||||
'Or get an API token to increase the allowance if not yet',
|
||||
name=name,
|
||||
reset=reset)
|
||||
return None
|
||||
|
||||
raise exc
|
||||
raise exc
|
||||
Loading…
Add table
Reference in a new issue