fix bugs

updater/dbmanager.py

@@ -12,6 +12,39 @@ import datetime
 from dateutil.parser import parse as parsedate
 import re
 
+EXCLUDED_PKGS = {
+    "base",
+    "boot",
+    "class",
+    "cluster",
+    "codetools",
+    "compiler",
+    "datasets",
+    "foreign",
+    "graphics",
+    "grDevices",
+    "grid",
+    "KernSmooth",
+    "lattice",
+    "MASS",
+    "Matrix",
+    "methods",
+    "mgcv",
+    "nlme",
+    "nnet",
+    "parallel",
+    "rpart",
+    "spatial",
+    "splines",
+    "stats",
+    "stats4",
+    "survival",
+    "tcltk",
+    "tools",
+    "utils",
+    "R"
+}
+
 Base = declarative_base()
 
 
@@ -114,6 +147,7 @@ def remove_all_cran_pkg(engine):
     session.query(PkgMeta).filter_by(repo='CRAN').delete()
     session.commit()
 
+
 def update_DB(engine, min_ver=None, first_run=False, mtime=None,
               bioc_mirror="https://bioconductor.org", cran_mirror="https://cran.r-project.org"):
     '''
@@ -169,7 +203,7 @@ def update_DB(engine, min_ver=None, first_run=False, mtime=None,
             # insert or skip
             for pkgmeta in pkgmetas:
                 # we already deleted all CRAN packages, so we can just add them.
-                session.add(pkgmeta)
+                add_or_update(session, PkgMeta, pkgmeta)
 
 
 def add_or_skip(session, table, pkgmeta):

updater/depends_updater.py

@@ -8,45 +8,15 @@ import logging
 from lilac2 import api as lilac
 import argparse
 import os
+import shutil
+import tempfile
+import filecmp
 import yaml
 from typing import Optional
 import sqlite3
-from dbmanager import get_bioc_versions
+from dbmanager import get_bioc_versions, EXCLUDED_PKGS
 from pkg_archiver import archive_pkg_yaml, archive_pkg_pkgbuild
 
-EXCLUDED_PKGS = {
-    "base",
-    "boot",
-    "class",
-    "cluster",
-    "codetools",
-    "compiler",
-    "datasets",
-    "foreign",
-    "graphics",
-    "grDevices",
-    "grid",
-    "KernSmooth",
-    "lattice",
-    "MASS",
-    "Matrix",
-    "methods",
-    "mgcv",
-    "nlme",
-    "nnet",
-    "parallel",
-    "rpart",
-    "spatial",
-    "splines",
-    "stats",
-    "stats4",
-    "survival",
-    "tcltk",
-    "tools",
-    "utils",
-    "R"
-}
-
 
 class PkgInfo:
     def __init__(self, pkgname=None, depends=None, optdepends=None,
@@ -316,6 +286,12 @@ class PkgInfo:
             yaml.dump(docs, f, sort_keys=False)
 
 
+def create_temporary_copy(path):
+    tmp = tempfile.NamedTemporaryFile(delete=False)
+    shutil.copy2(path, tmp.name)
+    return tmp.name
+
+
 def update_depends_by_file(file, bioarch_path="BioArchLinux", db="sqlite.db",
                            auto_archive=False,
                            bioc_min_ver="3.0", bioc_meta_mirror="https://bioconductor.org", output_file="added_depends.txt"):
@@ -343,6 +319,8 @@ def update_depends_by_file(file, bioarch_path="BioArchLinux", db="sqlite.db",
                 pkgname = "r-"+pkgname.lower()
             logging.info(f"Updating {pkgname}")
             os.chdir(f"{bioarch_path}/{pkgname}")
+            temp_pkgbuild = create_temporary_copy("PKGBUILD")
+            temp_lilac = create_temporary_copy("lilac.yaml")
             pkginfo = PkgInfo(bioc_min_version=bioc_min_ver,
                               bioc_meta_mirror=bioc_meta_mirror, bioc_versions=bioc_versions)
             pkginfo.build_body(cursor)
@@ -351,7 +329,15 @@ def update_depends_by_file(file, bioarch_path="BioArchLinux", db="sqlite.db",
             if auto_archive and pkginfo.is_archived():
                 archive_pkg_yaml(bioconductor_version=pkginfo.bioc_ver)
                 archive_pkg_pkgbuild(bioconductor_version=pkginfo.bioc_ver)
-            lilac.update_pkgrel()
+            # if PKGBUILD changed, bump pkgrel
+            if not filecmp.cmp(temp_pkgbuild, "PKGBUILD"):
+                lilac.update_pkgrel()
+            else:
+                # else revert changes to lilac.yaml
+                shutil.copy2(temp_lilac, "lilac.yaml")
+            os.remove(temp_pkgbuild)
+            os.remove(temp_lilac)
+
             if pkginfo.added_depends:
                 added_deps += pkginfo.added_depends
             os.chdir(current_dir)

updater/pkg_archiver.py

@@ -7,6 +7,7 @@ import os
 import yaml
 import argparse
 from lilac2.api import update_pkgrel
+import re
 
 
 def archive_pkg_by_file_list(file, bioarch_path="BioArchLinux", biconductor_version=3.15, step=1):
@@ -40,21 +41,25 @@ def archive_pkg_yaml(bioconductor_version=3.15, yaml_file="lilac.yaml"):
     with open(yaml_file, "r") as f:
         docs = yaml.load(f, Loader=yaml.FullLoader)
     url_idx = -1
+    url = None
     for i in range(len(docs['update_on'])):
         if "url" in docs['update_on'][i].keys():
             url = docs['update_on'][i]['url']
             url_idx = i
             break
 
+    if not url:
+        return
     pkg = url.rstrip('/').split('/')[-1]
+    archive_url = None
     # CRAN ARCHIVE
     if 'cran.r-project.org' in url:
         archive_url = f"https://cran.r-project.org/src/contrib/Archive/{pkg}"
     # Bioconductor ARCHIVE
     elif 'bioconductor.org' in url:
-        archive_url = f"https://bioconductor.org/packages/{bioconductor_version}/{pkg}"
-
-    docs['update_on'][url_idx]['url'] = archive_url
+        archive_url = url.replace('release', f"{bioconductor_version}")
+    if archive_url:
+        docs['update_on'][url_idx]['url'] = archive_url
     with open(yaml_file, 'w') as f:
         yaml.dump(docs, f, sort_keys=False)
 
@@ -70,7 +75,7 @@ def archive_pkg_pkgbuild(bioconductor_version=3.15, _pkgname="_pkgname"):
     flag = False
     for i in range(len(lines)):
 
-        if lines[i].startswith("url=") and '//bioconductor.org' in lines[i]:
+        if lines[i].startswith("url=") and '//bioconductor.org' in lines[i] and not re.search("packages/[\d.]+", lines[i]):
             lines[i] = lines[i].replace(
                 "packages/", f"packages/{bioconductor_version}/")
             changed = True