diff --git a/updater/dbmanager.py b/updater/dbmanager.py index ce7dca7..daa82aa 100755 --- a/updater/dbmanager.py +++ b/updater/dbmanager.py @@ -12,6 +12,39 @@ import datetime from dateutil.parser import parse as parsedate import re +EXCLUDED_PKGS = { + "base", + "boot", + "class", + "cluster", + "codetools", + "compiler", + "datasets", + "foreign", + "graphics", + "grDevices", + "grid", + "KernSmooth", + "lattice", + "MASS", + "Matrix", + "methods", + "mgcv", + "nlme", + "nnet", + "parallel", + "rpart", + "spatial", + "splines", + "stats", + "stats4", + "survival", + "tcltk", + "tools", + "utils", + "R" +} + Base = declarative_base() @@ -114,6 +147,7 @@ def remove_all_cran_pkg(engine): session.query(PkgMeta).filter_by(repo='CRAN').delete() session.commit() + def update_DB(engine, min_ver=None, first_run=False, mtime=None, bioc_mirror="https://bioconductor.org", cran_mirror="https://cran.r-project.org"): ''' @@ -169,7 +203,7 @@ def update_DB(engine, min_ver=None, first_run=False, mtime=None, # insert or skip for pkgmeta in pkgmetas: # we already deleted all CRAN packages, so we can just add them. - session.add(pkgmeta) + add_or_update(session, PkgMeta, pkgmeta) def add_or_skip(session, table, pkgmeta): diff --git a/updater/depends_updater.py b/updater/depends_updater.py index 6c7cc40..0df5971 100755 --- a/updater/depends_updater.py +++ b/updater/depends_updater.py @@ -8,45 +8,15 @@ import logging from lilac2 import api as lilac import argparse import os +import shutil +import tempfile +import filecmp import yaml from typing import Optional import sqlite3 -from dbmanager import get_bioc_versions +from dbmanager import get_bioc_versions, EXCLUDED_PKGS from pkg_archiver import archive_pkg_yaml, archive_pkg_pkgbuild -EXCLUDED_PKGS = { - "base", - "boot", - "class", - "cluster", - "codetools", - "compiler", - "datasets", - "foreign", - "graphics", - "grDevices", - "grid", - "KernSmooth", - "lattice", - "MASS", - "Matrix", - "methods", - "mgcv", - "nlme", - "nnet", - "parallel", - "rpart", - "spatial", - "splines", - "stats", - "stats4", - "survival", - "tcltk", - "tools", - "utils", - "R" -} - class PkgInfo: def __init__(self, pkgname=None, depends=None, optdepends=None, @@ -316,6 +286,12 @@ class PkgInfo: yaml.dump(docs, f, sort_keys=False) +def create_temporary_copy(path): + tmp = tempfile.NamedTemporaryFile(delete=False) + shutil.copy2(path, tmp.name) + return tmp.name + + def update_depends_by_file(file, bioarch_path="BioArchLinux", db="sqlite.db", auto_archive=False, bioc_min_ver="3.0", bioc_meta_mirror="https://bioconductor.org", output_file="added_depends.txt"): @@ -343,6 +319,8 @@ def update_depends_by_file(file, bioarch_path="BioArchLinux", db="sqlite.db", pkgname = "r-"+pkgname.lower() logging.info(f"Updating {pkgname}") os.chdir(f"{bioarch_path}/{pkgname}") + temp_pkgbuild = create_temporary_copy("PKGBUILD") + temp_lilac = create_temporary_copy("lilac.yaml") pkginfo = PkgInfo(bioc_min_version=bioc_min_ver, bioc_meta_mirror=bioc_meta_mirror, bioc_versions=bioc_versions) pkginfo.build_body(cursor) @@ -351,7 +329,15 @@ def update_depends_by_file(file, bioarch_path="BioArchLinux", db="sqlite.db", if auto_archive and pkginfo.is_archived(): archive_pkg_yaml(bioconductor_version=pkginfo.bioc_ver) archive_pkg_pkgbuild(bioconductor_version=pkginfo.bioc_ver) - lilac.update_pkgrel() + # if PKGBUILD changed, bump pkgrel + if not filecmp.cmp(temp_pkgbuild, "PKGBUILD"): + lilac.update_pkgrel() + else: + # else revert changes to lilac.yaml + shutil.copy2(temp_lilac, "lilac.yaml") + os.remove(temp_pkgbuild) + os.remove(temp_lilac) + if pkginfo.added_depends: added_deps += pkginfo.added_depends os.chdir(current_dir) diff --git a/updater/pkg_archiver.py b/updater/pkg_archiver.py index 2c05872..5f02fd6 100644 --- a/updater/pkg_archiver.py +++ b/updater/pkg_archiver.py @@ -7,6 +7,7 @@ import os import yaml import argparse from lilac2.api import update_pkgrel +import re def archive_pkg_by_file_list(file, bioarch_path="BioArchLinux", biconductor_version=3.15, step=1): @@ -40,21 +41,25 @@ def archive_pkg_yaml(bioconductor_version=3.15, yaml_file="lilac.yaml"): with open(yaml_file, "r") as f: docs = yaml.load(f, Loader=yaml.FullLoader) url_idx = -1 + url = None for i in range(len(docs['update_on'])): if "url" in docs['update_on'][i].keys(): url = docs['update_on'][i]['url'] url_idx = i break + if not url: + return pkg = url.rstrip('/').split('/')[-1] + archive_url = None # CRAN ARCHIVE if 'cran.r-project.org' in url: archive_url = f"https://cran.r-project.org/src/contrib/Archive/{pkg}" # Bioconductor ARCHIVE elif 'bioconductor.org' in url: - archive_url = f"https://bioconductor.org/packages/{bioconductor_version}/{pkg}" - - docs['update_on'][url_idx]['url'] = archive_url + archive_url = url.replace('release', f"{bioconductor_version}") + if archive_url: + docs['update_on'][url_idx]['url'] = archive_url with open(yaml_file, 'w') as f: yaml.dump(docs, f, sort_keys=False) @@ -70,7 +75,7 @@ def archive_pkg_pkgbuild(bioconductor_version=3.15, _pkgname="_pkgname"): flag = False for i in range(len(lines)): - if lines[i].startswith("url=") and '//bioconductor.org' in lines[i]: + if lines[i].startswith("url=") and '//bioconductor.org' in lines[i] and not re.search("packages/[\d.]+", lines[i]): lines[i] = lines[i].replace( "packages/", f"packages/{bioconductor_version}/") changed = True