From dbc35aa9015c2eb77567ea0cb0cbcf0950299ea4 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sun, 3 Nov 2013 18:21:50 +0800 Subject: [PATCH] refactor get_version.py --- nvchecker/get_version.py | 163 ++--------------------------------- nvchecker/main.py | 4 +- nvchecker/source/__init__.py | 1 + nvchecker/source/aur.py | 16 ++++ nvchecker/source/base.py | 7 ++ nvchecker/source/cmd.py | 41 +++++++++ nvchecker/source/gcode_hg.py | 29 +++++++ nvchecker/source/github.py | 17 ++++ nvchecker/source/pacman.py | 7 ++ nvchecker/source/pypi.py | 17 ++++ nvchecker/source/regex.py | 46 ++++++++++ 11 files changed, 189 insertions(+), 159 deletions(-) create mode 100644 nvchecker/source/__init__.py create mode 100644 nvchecker/source/aur.py create mode 100644 nvchecker/source/base.py create mode 100644 nvchecker/source/cmd.py create mode 100644 nvchecker/source/gcode_hg.py create mode 100644 nvchecker/source/github.py create mode 100644 nvchecker/source/pacman.py create mode 100644 nvchecker/source/pypi.py create mode 100644 nvchecker/source/regex.py diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index bfb065f..e6697c2 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -1,167 +1,18 @@ -import re -import sre_constants import logging -from functools import partial -import queue -import json -import urllib.parse -import time - -from pkg_resources import parse_version -from tornado.httpclient import AsyncHTTPClient -import tornado.process -from tornado.ioloop import IOLoop +from importlib import import_module logger = logging.getLogger(__name__) -handler_precedence = ('github', 'aur', 'pypi', 'pacman', - 'cmd', 'gcode_hg', 'regex') - -try: - import pycurl - AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") -except ImportError: - pycurl = None +handler_precedence = ( + 'github', 'aur', 'pypi', 'pacman', + 'cmd', 'gcode_hg', 'regex', +) def get_version(name, conf, callback): - g = globals() for key in handler_precedence: if key in conf: - funcname = 'get_version_by_' + key - g[funcname](name, conf, callback) + func = import_module('.source.' + key, __package__).get_version + func(name, conf, callback) break else: logger.error('%s: no idea to get version info.', name) callback(name, None) - -def get_version_by_regex(name, conf, callback): - try: - r = re.compile(conf['regex']) - except sre_constants.error: - logger.warn('%s: bad regex, skipped.', name, exc_info=True) - callback(name, None) - return - - encoding = conf.get('encoding', 'latin1') - httpclient = AsyncHTTPClient() - - kwargs = {} - if conf.get('proxy'): - if pycurl: - host, port = urllib.parse.splitport(conf['proxy']) - kwargs['proxy_host'] = host - kwargs['proxy_port'] = int(port) - else: - logger.warn('%s: proxy set but not used because pycurl is unavailable.', name) - - httpclient.fetch(conf['url'], partial( - _get_version_by_regex, name, r, encoding, callback - ), **kwargs) - -def _get_version_by_regex(name, regex, encoding, callback, res): - body = res.body.decode(encoding) - try: - version = max(regex.findall(body), key=parse_version) - except ValueError: - logger.error('%s: version string not found.', name) - callback(name, None) - else: - callback(name, version) - -AUR_URL = 'https://aur.archlinux.org/rpc.php?type=info&arg=' - -def get_version_by_aur(name, conf, callback): - aurname = conf.get('aur') or name - url = AUR_URL + aurname - AsyncHTTPClient().fetch(url, partial(_aur_done, name, callback)) - -def _aur_done(name, callback, res): - data = json.loads(res.body.decode('utf-8')) - version = data['results']['Version'] - callback(name, version) - -GITHUB_URL = 'https://api.github.com/repos/%s/commits' - -def get_version_by_github(name, conf, callback): - repo = conf.get('github') - url = GITHUB_URL % repo - AsyncHTTPClient().fetch(url, user_agent='lilydjwg/nvchecker', - callback=partial(_github_done, name, callback)) - -def _github_done(name, callback, res): - data = json.loads(res.body.decode('utf-8')) - version = data[0]['commit']['committer']['date'].split('T', 1)[0].replace('-', '') - callback(name, version) - -cmd_q = queue.Queue() -cmd_q.running = False - -def get_version_by_cmd(name, conf, callback): - cmd = conf['cmd'] - cmd_q.put((name, cmd, callback)) - if not cmd_q.running: - _run_command() - -def _run_command(): - cmd_q.running = True - try: - name, cmd, callback = cmd_q.get_nowait() - except queue.Empty: - cmd_q.running = False - return - - p = tornado.process.Subprocess(cmd, shell=True, io_loop=IOLoop.instance(), - stdout=tornado.process.Subprocess.STREAM) - p.set_exit_callback(partial(_command_done, name, callback, p)) - -def _command_done(name, callback, process, status): - if status != 0: - logger.error('%s: command exited with %d.', name, status) - callback(name, None) - else: - process.stdout.read_until_close(partial(_got_version_from_cmd, callback, name)) - _run_command() - -def _got_version_from_cmd(callback, name, output): - output = output.strip().decode('latin1') - callback(name, output) - -PYPI_URL = 'https://pypi.python.org/pypi/%s/json' - -def get_version_by_pypi(name, conf, callback): - repo = conf.get('pypi') or name - url = PYPI_URL % repo - AsyncHTTPClient().fetch(url, user_agent='lilydjwg/nvchecker', - callback=partial(_pypi_done, name, callback)) - -def _pypi_done(name, callback, res): - data = json.loads(res.body.decode('utf-8')) - version = data['info']['version'] - callback(name, version) - -GCODE_URL = 'https://code.google.com/p/%s/source/list' -GCODE_HG_RE = re.compile( - r'([^<]+)') - -def get_version_by_gcode_hg(name, conf, callback): - repo = conf.get('gcode_hg') or name - url = GCODE_URL % repo - AsyncHTTPClient().fetch(url, user_agent='lilydjwg/nvchecker', - callback=partial(_gcodehg_done, name, callback)) - -def _gcodehg_done(name, callback, res): - data = res.body.decode('utf-8') - m = GCODE_HG_RE.search(data) - if m: - t = time.strptime('Aug 15, 2013', '%b %d, %Y') - version = time.strftime('%Y%m%d', t) - else: - logger.error('%s: version not found.', name) - version = None - callback(name, version) - - -def get_version_by_pacman(name, conf, callback): - referree = conf['pacman'] - cmd = "LANG=C pacman -Si %s | grep -F Version | awk '{print $3}'" % referree - conf['cmd'] = cmd - get_version_by_cmd(name, conf, callback) diff --git a/nvchecker/main.py b/nvchecker/main.py index 42e9ef1..02632aa 100755 --- a/nvchecker/main.py +++ b/nvchecker/main.py @@ -1,11 +1,8 @@ #!/usr/bin/env python3 -import os -import sys import configparser import logging import argparse -from functools import partial from pkg_resources import parse_version from tornado.ioloop import IOLoop @@ -20,6 +17,7 @@ notifications = [] g_counter = 0 g_oldver = {} g_curver = {} +args = None def task_inc(): global g_counter diff --git a/nvchecker/source/__init__.py b/nvchecker/source/__init__.py new file mode 100644 index 0000000..9b5ed21 --- /dev/null +++ b/nvchecker/source/__init__.py @@ -0,0 +1 @@ +from .base import * diff --git a/nvchecker/source/aur.py b/nvchecker/source/aur.py new file mode 100644 index 0000000..d29baf5 --- /dev/null +++ b/nvchecker/source/aur.py @@ -0,0 +1,16 @@ +from functools import partial +import json + +from tornado.httpclient import AsyncHTTPClient + +AUR_URL = 'https://aur.archlinux.org/rpc.php?type=info&arg=' + +def get_version(name, conf, callback): + aurname = conf.get('aur') or name + url = AUR_URL + aurname + AsyncHTTPClient().fetch(url, partial(_aur_done, name, callback)) + +def _aur_done(name, callback, res): + data = json.loads(res.body.decode('utf-8')) + version = data['results']['Version'] + callback(name, version) diff --git a/nvchecker/source/base.py b/nvchecker/source/base.py new file mode 100644 index 0000000..69041c8 --- /dev/null +++ b/nvchecker/source/base.py @@ -0,0 +1,7 @@ +from tornado.httpclient import AsyncHTTPClient +try: + import pycurl + AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") +except ImportError: + pycurl = None + diff --git a/nvchecker/source/cmd.py b/nvchecker/source/cmd.py new file mode 100644 index 0000000..1cb1fc3 --- /dev/null +++ b/nvchecker/source/cmd.py @@ -0,0 +1,41 @@ +import queue +import logging +from functools import partial + +import tornado.process +from tornado.ioloop import IOLoop + +logger = logging.getLogger(__name__) +cmd_q = queue.Queue() +cmd_q.running = False + +def get_version(name, conf, callback): + cmd = conf['cmd'] + cmd_q.put((name, cmd, callback)) + if not cmd_q.running: + _run_command() + +def _run_command(): + cmd_q.running = True + try: + name, cmd, callback = cmd_q.get_nowait() + except queue.Empty: + cmd_q.running = False + return + + p = tornado.process.Subprocess(cmd, shell=True, io_loop=IOLoop.instance(), + stdout=tornado.process.Subprocess.STREAM) + p.set_exit_callback(partial(_command_done, name, callback, p)) + +def _command_done(name, callback, process, status): + if status != 0: + logger.error('%s: command exited with %d.', name, status) + callback(name, None) + else: + process.stdout.read_until_close(partial(_got_version_from_cmd, callback, name)) + _run_command() + +def _got_version_from_cmd(callback, name, output): + output = output.strip().decode('latin1') + callback(name, output) + diff --git a/nvchecker/source/gcode_hg.py b/nvchecker/source/gcode_hg.py new file mode 100644 index 0000000..0b91a4a --- /dev/null +++ b/nvchecker/source/gcode_hg.py @@ -0,0 +1,29 @@ +import re +import time +import logging +from functools import partial + +from tornado.httpclient import AsyncHTTPClient + +logger = logging.getLogger(__name__) + +GCODE_URL = 'https://code.google.com/p/%s/source/list' +GCODE_HG_RE = re.compile( + r'([^<]+)') + +def get_version(name, conf, callback): + repo = conf.get('gcode_hg') or name + url = GCODE_URL % repo + AsyncHTTPClient().fetch(url, user_agent='lilydjwg/nvchecker', + callback=partial(_gcodehg_done, name, callback)) + +def _gcodehg_done(name, callback, res): + data = res.body.decode('utf-8') + m = GCODE_HG_RE.search(data) + if m: + t = time.strptime('Aug 15, 2013', '%b %d, %Y') + version = time.strftime('%Y%m%d', t) + else: + logger.error('%s: version not found.', name) + version = None + callback(name, version) diff --git a/nvchecker/source/github.py b/nvchecker/source/github.py new file mode 100644 index 0000000..814afb4 --- /dev/null +++ b/nvchecker/source/github.py @@ -0,0 +1,17 @@ +import json +from functools import partial + +from tornado.httpclient import AsyncHTTPClient + +GITHUB_URL = 'https://api.github.com/repos/%s/commits' + +def get_version(name, conf, callback): + repo = conf.get('github') + url = GITHUB_URL % repo + AsyncHTTPClient().fetch(url, user_agent='lilydjwg/nvchecker', + callback=partial(_github_done, name, callback)) + +def _github_done(name, callback, res): + data = json.loads(res.body.decode('utf-8')) + version = data[0]['commit']['committer']['date'].split('T', 1)[0].replace('-', '') + callback(name, version) diff --git a/nvchecker/source/pacman.py b/nvchecker/source/pacman.py new file mode 100644 index 0000000..dc4557c --- /dev/null +++ b/nvchecker/source/pacman.py @@ -0,0 +1,7 @@ +from . import cmd + +def get_version(name, conf, callback): + referree = conf['pacman'] + c = "LANG=C pacman -Si %s | grep -F Version | awk '{print $3}'" % referree + conf['cmd'] = c + cmd.get_version(name, conf, callback) diff --git a/nvchecker/source/pypi.py b/nvchecker/source/pypi.py new file mode 100644 index 0000000..51f47e9 --- /dev/null +++ b/nvchecker/source/pypi.py @@ -0,0 +1,17 @@ +import json +from functools import partial + +from tornado.httpclient import AsyncHTTPClient + +PYPI_URL = 'https://pypi.python.org/pypi/%s/json' + +def get_version(name, conf, callback): + repo = conf.get('pypi') or name + url = PYPI_URL % repo + AsyncHTTPClient().fetch(url, user_agent='lilydjwg/nvchecker', + callback=partial(_pypi_done, name, callback)) + +def _pypi_done(name, callback, res): + data = json.loads(res.body.decode('utf-8')) + version = data['info']['version'] + callback(name, version) diff --git a/nvchecker/source/regex.py b/nvchecker/source/regex.py new file mode 100644 index 0000000..e4b81db --- /dev/null +++ b/nvchecker/source/regex.py @@ -0,0 +1,46 @@ +import re +import sre_constants +import logging +import urllib.parse +from functools import partial + +from pkg_resources import parse_version +from tornado.httpclient import AsyncHTTPClient + +from .base import pycurl + +logger = logging.getLogger(__name__) + +def get_version(name, conf, callback): + try: + r = re.compile(conf['regex']) + except sre_constants.error: + logger.warn('%s: bad regex, skipped.', name, exc_info=True) + callback(name, None) + return + + encoding = conf.get('encoding', 'latin1') + httpclient = AsyncHTTPClient() + + kwargs = {} + if conf.get('proxy'): + if pycurl: + host, port = urllib.parse.splitport(conf['proxy']) + kwargs['proxy_host'] = host + kwargs['proxy_port'] = int(port) + else: + logger.warn('%s: proxy set but not used because pycurl is unavailable.', name) + + httpclient.fetch(conf['url'], partial( + _got_version, name, r, encoding, callback + ), **kwargs) + +def _got_version(name, regex, encoding, callback, res): + body = res.body.decode(encoding) + try: + version = max(regex.findall(body), key=parse_version) + except ValueError: + logger.error('%s: version string not found.', name) + callback(name, None) + else: + callback(name, version)