mirror of
https://github.com/lilydjwg/nvchecker.git
synced 2025-03-10 06:14:02 +00:00
restructure and rename to -graphql
This commit is contained in:
envolution
parent
a0a5da6ebf
commit
965c26b34e
2 changed files with 179 additions and 329 deletions
179
nvchecker_source/github-graphql.py
Normal file
179
nvchecker_source/github-graphql.py
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import aiohttp
|
||||||
|
from typing import List, Tuple, Union, Optional
|
||||||
|
from nvchecker.api import RichResult, Entry, KeyManager, GetVersionError, AsyncCache
|
||||||
|
|
||||||
|
async def get_github_token(conf: Entry, host: str, keymanager: KeyManager) -> Optional[str]:
|
||||||
|
token = conf.get('token')
|
||||||
|
if token is None:
|
||||||
|
token = keymanager.get_key(host.lower(), 'github')
|
||||||
|
if token is None:
|
||||||
|
token = os.environ.get('GITHUB_TOKEN')
|
||||||
|
return token
|
||||||
|
|
||||||
|
def create_rich_result(conf, commits, sha, **kwargs) -> RichResult:
|
||||||
|
if conf.get('use_commit_number', False):
|
||||||
|
kwargs['version'] += f"+r{str(commits)}"
|
||||||
|
if conf.get('use_commit_hash', False):
|
||||||
|
kwargs['version'] += f"+g{sha[:9]}"
|
||||||
|
return RichResult(**kwargs)
|
||||||
|
|
||||||
|
async def get_version(
|
||||||
|
name: str, conf: Entry, *,
|
||||||
|
cache: AsyncCache, keymanager: KeyManager,
|
||||||
|
**kwargs,
|
||||||
|
) -> RichResult:
|
||||||
|
repo = conf['github']
|
||||||
|
owner, reponame = repo.split('/')
|
||||||
|
host = conf.get('host', "github.com")
|
||||||
|
token = await get_github_token(conf, host, keymanager)
|
||||||
|
|
||||||
|
if not token:
|
||||||
|
raise GetVersionError('token not given but it is required')
|
||||||
|
|
||||||
|
GITHUB_GRAPHQL_URL = 'https://api.github.com/graphql'
|
||||||
|
query = """
|
||||||
|
query {
|
||||||
|
repository(owner: "$owner", name: "$name") {
|
||||||
|
defaultBranchRef {
|
||||||
|
target {
|
||||||
|
... on Commit {
|
||||||
|
history(first: 1) {
|
||||||
|
totalCount
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
oid
|
||||||
|
committedDate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
refs(refPrefix: "refs/tags/", first: 100, orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
name
|
||||||
|
target {
|
||||||
|
oid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
releases(first: 100, orderBy: { field: CREATED_AT, direction: DESC }) {
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
name
|
||||||
|
url
|
||||||
|
tagName
|
||||||
|
isPrerelease
|
||||||
|
isLatest
|
||||||
|
createdAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
query_vars = query.replace("$owner", owner).replace("$name", reponame)
|
||||||
|
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
headers = {
|
||||||
|
'Authorization': f'bearer {token}',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with session.post(
|
||||||
|
GITHUB_GRAPHQL_URL,
|
||||||
|
headers=headers,
|
||||||
|
json={'query': query_vars}
|
||||||
|
) as response:
|
||||||
|
data = await response.json()
|
||||||
|
|
||||||
|
if 'errors' in data:
|
||||||
|
raise GetVersionError(f"GitHub API error: {data['errors']}")
|
||||||
|
|
||||||
|
repo_data = data['data']['repository']
|
||||||
|
commits = repo_data["defaultBranchRef"]["target"]["history"]["totalCount"]
|
||||||
|
sha = repo_data["defaultBranchRef"]["target"]["history"]["edges"][0]["node"]["oid"]
|
||||||
|
|
||||||
|
# Latest Tag Strategy
|
||||||
|
if conf.get('use_latest_tag', False):
|
||||||
|
refs = repo_data['refs']['edges']
|
||||||
|
if not refs:
|
||||||
|
raise GetVersionError('No tag found in upstream repository.')
|
||||||
|
latest_tag = refs[0]['node']
|
||||||
|
return create_rich_result(
|
||||||
|
conf=conf,
|
||||||
|
commits=commits,
|
||||||
|
sha=sha,
|
||||||
|
version=latest_tag['name'],
|
||||||
|
gitref=f"refs/tags/{latest_tag['name']}",
|
||||||
|
revision=latest_tag['target']['oid'],
|
||||||
|
url=f'https://github.com/{repo}/releases/tag/{latest_tag["name"]}'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Maximum Tag Strategy - Return first tag
|
||||||
|
if conf.get('use_max_tag', False):
|
||||||
|
refs = repo_data['refs']['edges']
|
||||||
|
if not refs:
|
||||||
|
raise GetVersionError('No tag found in upstream repository.')
|
||||||
|
first_tag = refs[0]['node']
|
||||||
|
return create_rich_result(
|
||||||
|
conf=conf,
|
||||||
|
commits=commits,
|
||||||
|
sha=sha,
|
||||||
|
version=first_tag['name'],
|
||||||
|
gitref=f"refs/tags/{first_tag['name']}",
|
||||||
|
revision=first_tag['target']['oid'],
|
||||||
|
url=f'https://github.com/{repo}/releases/tag/{first_tag["name"]}'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Release Strategies
|
||||||
|
if conf.get('use_latest_release', False) or conf.get('use_newest_release', False):
|
||||||
|
releases = repo_data['releases']['edges']
|
||||||
|
if not releases:
|
||||||
|
raise GetVersionError('No release found in upstream repository.')
|
||||||
|
|
||||||
|
include_prereleases = conf.get('use_prereleases', False)
|
||||||
|
|
||||||
|
if conf.get('use_latest_release', False):
|
||||||
|
latest_release = next(
|
||||||
|
(release['node'] for release in releases
|
||||||
|
if release['node']['isLatest'] or (include_prereleases and release['node']['isPrerelease'])),
|
||||||
|
None
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
latest_release = releases[0]['node']
|
||||||
|
|
||||||
|
if not latest_release:
|
||||||
|
raise GetVersionError('No suitable release found')
|
||||||
|
|
||||||
|
use_release_name = conf.get('use_release_name', False)
|
||||||
|
version = latest_release['name'] if use_release_name else latest_release['tagName']
|
||||||
|
|
||||||
|
return create_rich_result(
|
||||||
|
conf=conf,
|
||||||
|
commits=commits,
|
||||||
|
sha=sha,
|
||||||
|
version=version,
|
||||||
|
gitref=f"refs/tags/{latest_release['tagName']}",
|
||||||
|
url=latest_release['url']
|
||||||
|
)
|
||||||
|
|
||||||
|
# Default: Use commit date
|
||||||
|
commit = repo_data['defaultBranchRef']['target']['history']['edges'][0]['node']
|
||||||
|
return create_rich_result(
|
||||||
|
conf=conf,
|
||||||
|
commits=commits,
|
||||||
|
sha=sha,
|
||||||
|
version=commit['committedDate'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.'),
|
||||||
|
revision=commit['oid'],
|
||||||
|
url=f'https://github.com/{repo}/commit/{commit["oid"]}'
|
||||||
|
)
|
||||||
|
|
||||||
|
except aiohttp.ClientError as e:
|
||||||
|
raise GetVersionError(f"GitHub API request failed: {e}")
|
||||||
|
|
@ -1,329 +0,0 @@
|
||||||
import os # Added for environment variable access
|
|
||||||
import time
|
|
||||||
from urllib.parse import urlencode
|
|
||||||
from typing import List, Tuple, Union, Optional
|
|
||||||
import asyncio
|
|
||||||
import aiohttp
|
|
||||||
|
|
||||||
import structlog
|
|
||||||
from nvchecker.api import (
|
|
||||||
VersionResult, Entry, AsyncCache, KeyManager,
|
|
||||||
HTTPError, session, RichResult, GetVersionError,
|
|
||||||
)
|
|
||||||
DEFAULT_TIMEOUT = aiohttp.ClientTimeout(total=60)
|
|
||||||
logger = structlog.get_logger(logger_name=__name__)
|
|
||||||
ALLOW_REQUEST = None
|
|
||||||
RATE_LIMITED_ERROR = False
|
|
||||||
_http_client = None
|
|
||||||
|
|
||||||
GITHUB_GRAPHQL_URL = 'https://api.%s/graphql'
|
|
||||||
|
|
||||||
async def create_http_client():
|
|
||||||
"""Create a new aiohttp client session with proper configuration."""
|
|
||||||
return aiohttp.ClientSession(timeout=DEFAULT_TIMEOUT)
|
|
||||||
|
|
||||||
async def get_http_client():
|
|
||||||
"""Initialize and return the HTTP client."""
|
|
||||||
global _http_client
|
|
||||||
if _http_client is not None:
|
|
||||||
return _http_client
|
|
||||||
|
|
||||||
# Create a new client session if none exists
|
|
||||||
client = await create_http_client()
|
|
||||||
_http_client = client
|
|
||||||
return _http_client
|
|
||||||
|
|
||||||
def create_rich_result(conf, commits, sha, **kwargs) -> RichResult:
|
|
||||||
"""
|
|
||||||
Helper function to centralize the creation of a RichResult.
|
|
||||||
Accepts any keyword arguments and passes them to RichResult.
|
|
||||||
"""
|
|
||||||
if conf.get('use_commit_number', False):
|
|
||||||
kwargs['version'] += f"+r{str(commits)}"
|
|
||||||
if conf.get('use_commit_hash', False):
|
|
||||||
kwargs['version'] += f"+g{sha[:9]}"
|
|
||||||
return RichResult(**kwargs)
|
|
||||||
|
|
||||||
async def execute_github_query(host: str, owner: str, reponame: str, token: str) -> dict:
|
|
||||||
"""
|
|
||||||
Execute GraphQL query against GitHub API and return the response data.
|
|
||||||
Centralizes error handling and query execution.
|
|
||||||
"""
|
|
||||||
client = await get_http_client()
|
|
||||||
|
|
||||||
headers = {
|
|
||||||
'Authorization': f'bearer {token}',
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
}
|
|
||||||
query_vars = QUERY_GITHUB.replace("$owner", owner).replace("$name", reponame)
|
|
||||||
client = await get_http_client()
|
|
||||||
|
|
||||||
try:
|
|
||||||
async with client.post(
|
|
||||||
GITHUB_GRAPHQL_URL % host,
|
|
||||||
headers=headers,
|
|
||||||
json={'query': query_vars}
|
|
||||||
) as response:
|
|
||||||
# Check response status
|
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
# Parse JSON response
|
|
||||||
data = await response.json()
|
|
||||||
|
|
||||||
# Handle rate limiting headers
|
|
||||||
remaining = response.headers.get('X-RateLimit-Remaining')
|
|
||||||
if remaining and int(remaining) == 0:
|
|
||||||
reset_time = int(response.headers.get('X-RateLimit-Reset', 0))
|
|
||||||
logger.warning(
|
|
||||||
"GitHub API rate limit reached",
|
|
||||||
reset_time=time.ctime(reset_time)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check for GraphQL errors
|
|
||||||
if 'errors' in data:
|
|
||||||
raise GetVersionError(f"GitHub API error: {data['errors']}")
|
|
||||||
await client.close()
|
|
||||||
return data['data']['repository']
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("GitHub API request failed", error=str(e))
|
|
||||||
raise
|
|
||||||
|
|
||||||
def get_github_token(conf: Entry, host: str, keymanager: KeyManager) -> Optional[str]:
|
|
||||||
"""Get GitHub token from config, keymanager, or environment."""
|
|
||||||
token = conf.get('token')
|
|
||||||
if token is None:
|
|
||||||
token = keymanager.get_key(host.lower(), 'github')
|
|
||||||
if token is None:
|
|
||||||
token = os.environ.get('GITHUB_TOKEN')
|
|
||||||
return token
|
|
||||||
|
|
||||||
async def get_version(name, conf, **kwargs):
|
|
||||||
global RATE_LIMITED_ERROR
|
|
||||||
|
|
||||||
if RATE_LIMITED_ERROR:
|
|
||||||
raise RuntimeError('rate limited')
|
|
||||||
|
|
||||||
try:
|
|
||||||
return await _get_version_with_retry(name, conf, **kwargs)
|
|
||||||
finally:
|
|
||||||
await _cleanup_http_client()
|
|
||||||
|
|
||||||
async def _get_version_with_retry(name, conf, **kwargs):
|
|
||||||
global ALLOW_REQUEST
|
|
||||||
if ALLOW_REQUEST is None:
|
|
||||||
ALLOW_REQUEST = asyncio.Event()
|
|
||||||
ALLOW_REQUEST.set()
|
|
||||||
|
|
||||||
for _ in range(2): # retry once
|
|
||||||
try:
|
|
||||||
await ALLOW_REQUEST.wait()
|
|
||||||
return await get_version_real(name, conf, **kwargs)
|
|
||||||
except HTTPError as e:
|
|
||||||
if e.code in [403, 429]:
|
|
||||||
if n := check_ratelimit(e, name):
|
|
||||||
ALLOW_REQUEST.clear()
|
|
||||||
await asyncio.sleep(n+1)
|
|
||||||
ALLOW_REQUEST.set()
|
|
||||||
continue
|
|
||||||
RATE_LIMITED_ERROR = True
|
|
||||||
raise
|
|
||||||
|
|
||||||
async def _cleanup_http_client():
|
|
||||||
"""Clean up the global HTTP client if it exists."""
|
|
||||||
global _http_client
|
|
||||||
if _http_client is not None:
|
|
||||||
await _http_client.close()
|
|
||||||
_http_client = None
|
|
||||||
|
|
||||||
QUERY_GITHUB = """
|
|
||||||
query {
|
|
||||||
rateLimit {
|
|
||||||
limit
|
|
||||||
remaining
|
|
||||||
resetAt
|
|
||||||
}
|
|
||||||
repository(owner: "$owner", name: "$name") {
|
|
||||||
# Default branch commits
|
|
||||||
defaultBranchRef {
|
|
||||||
target {
|
|
||||||
... on Commit {
|
|
||||||
history(first: 1) {
|
|
||||||
totalCount
|
|
||||||
edges {
|
|
||||||
node {
|
|
||||||
oid
|
|
||||||
committedDate
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# All tags
|
|
||||||
refs(refPrefix: "refs/tags/", first: 1, orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {
|
|
||||||
edges {
|
|
||||||
node {
|
|
||||||
name
|
|
||||||
target {
|
|
||||||
oid
|
|
||||||
... on Commit {
|
|
||||||
url
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# All releases (can't filter server-side)
|
|
||||||
releases(first: 100, orderBy: { field: CREATED_AT, direction: DESC }) {
|
|
||||||
totalCount
|
|
||||||
edges {
|
|
||||||
node {
|
|
||||||
name
|
|
||||||
url
|
|
||||||
tagName
|
|
||||||
isPrerelease
|
|
||||||
isLatest
|
|
||||||
createdAt
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pageInfo {
|
|
||||||
hasNextPage
|
|
||||||
endCursor
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
async def get_latest_tag(conf: Entry, key: Tuple[str, str, str, str]) -> RichResult:
|
|
||||||
host, repo, query, token = key
|
|
||||||
owner, reponame = repo.split('/')
|
|
||||||
if not token:
|
|
||||||
raise GetVersionError('token is required for latest tag query')
|
|
||||||
|
|
||||||
repo_data = await execute_github_query(host, owner, reponame, token)
|
|
||||||
refs = repo_data['refs']['edges']
|
|
||||||
version = refs[0]['node']['name']
|
|
||||||
revision = refs[0]['node']['target']['oid']
|
|
||||||
commits = repo_data["defaultBranchRef"]["target"]["history"]["totalCount"]
|
|
||||||
sha = repo_data["defaultBranchRef"]["target"]["history"]["edges"][0]["node"]["oid"]
|
|
||||||
return create_rich_result(
|
|
||||||
conf=conf,
|
|
||||||
commits=commits,
|
|
||||||
sha=sha,
|
|
||||||
version=version,
|
|
||||||
gitref=f"refs/tags/{version}",
|
|
||||||
revision=revision,
|
|
||||||
url=f'https://github.com/{repo}/releases/tag/{version}',
|
|
||||||
)
|
|
||||||
|
|
||||||
async def get_version_real(
|
|
||||||
name: str, conf: Entry, *,
|
|
||||||
cache: AsyncCache, keymanager: KeyManager,
|
|
||||||
**kwargs,
|
|
||||||
) -> VersionResult:
|
|
||||||
repo = conf['github']
|
|
||||||
owner, reponame = repo.split('/')
|
|
||||||
host = conf.get('host', "github.com")
|
|
||||||
token = get_github_token(conf, host, keymanager)
|
|
||||||
|
|
||||||
use_latest_tag = conf.get('use_latest_tag', False)
|
|
||||||
if use_latest_tag:
|
|
||||||
if not token:
|
|
||||||
raise GetVersionError('token not given but it is required')
|
|
||||||
|
|
||||||
query = conf.get('query', '')
|
|
||||||
return await cache.get((host, repo, query, token), get_latest_tag(conf))
|
|
||||||
|
|
||||||
repo_data = await execute_github_query(host, owner, reponame, token)
|
|
||||||
commits = repo_data["defaultBranchRef"]["target"]["history"]["totalCount"]
|
|
||||||
sha = repo_data["defaultBranchRef"]["target"]["history"]["edges"][0]["node"]["oid"]
|
|
||||||
|
|
||||||
use_max_tag = conf.get('use_max_tag', False)
|
|
||||||
if use_max_tag:
|
|
||||||
refs = repo_data['refs']['edges']
|
|
||||||
tags: List[Union[str, RichResult]] = [
|
|
||||||
create_rich_result(
|
|
||||||
conf=conf,
|
|
||||||
commits=commits,
|
|
||||||
sha=sha,
|
|
||||||
version=ref['node']['name'],
|
|
||||||
gitref=f"refs/tags/{ref['node']['name']}",
|
|
||||||
revision=ref['node']['target']['oid'],
|
|
||||||
url=f'https://github.com/{repo}/releases/tag/{ref["node"]["name"]}',
|
|
||||||
) for ref in refs
|
|
||||||
]
|
|
||||||
if not tags:
|
|
||||||
raise GetVersionError('No tag found in upstream repository.')
|
|
||||||
return tags
|
|
||||||
use_latest_release = conf.get('use_latest_release', False)
|
|
||||||
use_newest_release = conf.get('use_newest_release', False)
|
|
||||||
include_prereleases = conf.get('use_prereleases', False)
|
|
||||||
if use_latest_release or use_newest_release:
|
|
||||||
if use_latest_release:
|
|
||||||
releases = repo_data['releases']['edges']
|
|
||||||
if not releases:
|
|
||||||
raise GetVersionError('No release found in upstream repository.')
|
|
||||||
if include_prereleases:
|
|
||||||
latest_release = next(
|
|
||||||
(release['node'] for release in releases if release['node']['isLatest'] or release['node']['isPrerelease]']),
|
|
||||||
None
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
latest_release = next(
|
|
||||||
(release['node'] for release in releases if release['node']['isLatest'] and not release['node']['isPrerelease']),
|
|
||||||
None
|
|
||||||
)
|
|
||||||
|
|
||||||
elif use_newest_release:
|
|
||||||
releases = repo_data['releases']['edges']
|
|
||||||
latest_release = next(
|
|
||||||
(release['node'] for release in releases),
|
|
||||||
None
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
use_release_name = conf.get('use_release_name', False)
|
|
||||||
version = latest_release['name'] if use_release_name else latest_release['tagName']
|
|
||||||
|
|
||||||
return create_rich_result(
|
|
||||||
conf=conf,
|
|
||||||
commits=commits,
|
|
||||||
sha=sha,
|
|
||||||
version=version,
|
|
||||||
gitref=f"refs/tags/{latest_release['tagName']}",
|
|
||||||
url=latest_release['url'],
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
commit = repo_data['defaultBranchRef']['target']['history']['edges'][0]['node']
|
|
||||||
return create_rich_result(
|
|
||||||
conf=conf,
|
|
||||||
commits=commits,
|
|
||||||
sha=sha,
|
|
||||||
version=commit['committedDate'].rstrip('Z').replace('-', '').replace(':', '').replace('T', '.'),
|
|
||||||
revision=commit['oid'],
|
|
||||||
url=f'https://github.com/{repo}/commit/{commit["oid"]}',
|
|
||||||
)
|
|
||||||
|
|
||||||
def check_ratelimit(exc: HTTPError, name: str) -> Optional[int]:
|
|
||||||
res = exc.response
|
|
||||||
if not res:
|
|
||||||
raise exc
|
|
||||||
|
|
||||||
if v := res.headers.get('retry-after'):
|
|
||||||
n = int(v)
|
|
||||||
logger.warning('retry-after', n=n)
|
|
||||||
return n
|
|
||||||
|
|
||||||
# default -1 is used to re-raise the exception
|
|
||||||
n = int(res.headers.get('X-RateLimit-Remaining', -1))
|
|
||||||
if n == 0:
|
|
||||||
reset = int(res.headers.get('X-RateLimit-Reset'))
|
|
||||||
logger.error(f'rate limited, resetting at {time.ctime(reset)}. '
|
|
||||||
'Or get an API token to increase the allowance if not yet',
|
|
||||||
name = name,
|
|
||||||
reset = reset)
|
|
||||||
return None
|
|
||||||
|
|
||||||
raise exc
|
|
||||||
Loading…
Add table
Reference in a new issue