This is the mail archive of the
cygwin-apps-cvs
mailing list for the cygwin-apps project.
[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-101-g6aa625a
- From: jturney at sourceware dot org
- To: cygwin-apps-cvs at sourceware dot org
- Date: 8 Oct 2017 13:42:56 -0000
- Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-101-g6aa625a
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=6aa625a6a83ae615a859f61993696a1665340690
commit 6aa625a6a83ae615a859f61993696a1665340690
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu Oct 5 12:23:19 2017 +0100
Upgrade some checks from a warning to an error
Make "curr: version isn't the most recent non-test: version" and "empty but
not in _obsolete category" errors.
Fix tests by fixing up package timestamps
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4a10c12553a645bd719c056fdc9b463f1065702f
commit 4a10c12553a645bd719c056fdc9b463f1065702f
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu Oct 5 14:25:56 2017 +0100
Update past_mistakes
Clean out things no longer needed
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=900929448bf1c7852598ebe9ea632faa2a475276
commit 900929448bf1c7852598ebe9ea632faa2a475276
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu Oct 5 13:09:40 2017 +0100
Add strict handling for some more warnings in read_package
In particular, the hyphen-in-version warning is now treated as an error in
strict (upload) mode.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=52f0a5f18d2173c710eb603ee0675a17f91b9457
commit 52f0a5f18d2173c710eb603ee0675a17f91b9457
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Wed Oct 4 16:07:22 2017 +0100
Also use irker to notify of important events
Also, don't fail if irkerd isn't running.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3673949b6259839db7526584efbe8de432d41db4
commit 3673949b6259839db7526584efbe8de432d41db4
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Wed Oct 4 15:41:41 2017 +0100
Add more version comparision tests
Test data taken from RPM
Diff:
---
TODO | 1 -
calm/calm.py | 9 ++++-
calm/irk.py | 13 ++++--
calm/package.py | 26 ++++++++++---
calm/past_mistakes.py | 47 +----------------------
test/test_calm.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 138 insertions(+), 57 deletions(-)
diff --git a/TODO b/TODO
index e82b165..07510d5 100644
--- a/TODO
+++ b/TODO
@@ -1,5 +1,4 @@
* more than 2 versions possible
-* use irkerd to report when calm failed due to an error?
* upload a hash at the same time as package, and pass that through to setup.ini
* mksetupini should have an okmissing option for override.hint which names non-existent versions
* use ./setup.hint inside the tar file, avoiding all the hint/tar coherence problems
diff --git a/calm/calm.py b/calm/calm.py
index 72325f7..14d4317 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -66,6 +66,7 @@ import time
from .abeyance_handler import AbeyanceHandler
from .buffering_smtp_handler import BufferingSMTPHandler
from . import common_constants
+from . import irk
from . import maintainers
from . import package
from . import pkg2html
@@ -246,7 +247,9 @@ def process_uploads(args, state):
for arch in common_constants.ARCHES:
# use merged package list
state.packages[arch] = merged_packages[arch]
- logging.debug("added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name))
+ msg = "added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name)
+ logging.debug(msg)
+ irk.irk(msg)
# record updated reminder times for maintainers
maintainers.Maintainer.update_reminder_times(mlist)
@@ -488,6 +491,7 @@ def do_daemon(args, state):
with context:
logging_setup(args)
logging.info("calm daemon started, pid %d" % (os.getpid()))
+ irk.irk("calm daemon started")
state.packages = {}
@@ -531,6 +535,9 @@ def do_daemon(args, state):
except Exception as e:
with mail_logs(args.email, toaddrs=args.email, subject='calm stopping due to unhandled exception', thresholdLevel=logging.ERROR) as leads_email:
logging.error("exception %s" % (type(e).__name__), exc_info=True)
+ irk.irk("calm daemon stopped due to unhandled exception")
+ else:
+ irk.irk("calm daemon stopped")
logging.info("calm daemon stopped")
diff --git a/calm/irk.py b/calm/irk.py
index b9f8957..47cb112 100755
--- a/calm/irk.py
+++ b/calm/irk.py
@@ -26,13 +26,16 @@ def send(s, target, message):
def irk(message, target=DEFAULT_TARGET, server=DEFAULT_SERVER):
- s = connect(server)
- if "irc:" not in target and "ircs:" not in target:
- target = "irc://chat.freenode.net/{0}".format(target)
+ try:
+ s = connect(server)
+ if "irc:" not in target and "ircs:" not in target:
+ target = "irc://chat.freenode.net/{0}".format(target)
- send(s, target, message)
+ send(s, target, message)
- s.close()
+ s.close()
+ except ConnectionRefusedError:
+ pass
def main():
diff --git a/calm/package.py b/calm/package.py
index cf9d242..ed4cf00 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -259,11 +259,16 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
# we already know P to split unambiguously), but this is a bad
# idea.
if '-' in v:
- lvl = logging.WARNING if p not in past_mistakes.hyphen_in_version else logging.INFO
+ if p not in past_mistakes.hyphen_in_version:
+ lvl = strict_lvl
+ warnings = True
+ else:
+ lvl = logging.INFO
logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p))
if not v[0].isdigit():
- logging.warning("file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
+ logging.log(strict_lvl, "file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
+ warnings = True
# if not there already, add to version-release list
vr_list.add('%s-%s' % (v, r))
@@ -329,7 +334,8 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
packages[p].skip = any(['skip' in version_hints[vr] for vr in vr_list])
elif (len(files) > 0) and (relpath.count(os.path.sep) > 1):
- logging.warning("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
+ logging.log(strict_lvl, "no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
+ warnings = True
if strict:
return warnings
@@ -553,13 +559,17 @@ def validate_packages(args, packages):
elif 'curr' not in packages[p].stability and 'curr' not in getattr(args, 'okmissing', []):
logging.warning("package '%s' doesn't have a curr version" % (p))
- # warn if the curr: version isn't the most recent non-test: version
+ # error if the curr: version isn't the most recent non-test: version
for v in sorted(packages[p].vermap.keys(), key=lambda v: packages[p].vermap[v]['mtime'], reverse=True):
if 'test' in packages[p].version_hints[v]:
continue
if packages[p].stability['curr'] != v:
- lvl = logging.WARNING if p not in past_mistakes.mtime_anomalies else logging.DEBUG
+ if p in past_mistakes.mtime_anomalies:
+ lvl = logging.DEBUG
+ else:
+ lvl = logging.ERROR
+ error = True
logging.log(lvl, "package '%s' version '%s' is most recent non-test version, but version '%s' is curr:" % (p, v, packages[p].stability['curr']))
break
@@ -584,7 +594,11 @@ def validate_packages(args, packages):
if 'source' not in packages[p].vermap[vr]:
if 'install' in packages[p].vermap[vr]:
if packages[p].tars[packages[p].vermap[vr]['install']].is_empty:
- lvl = logging.WARNING if p not in past_mistakes.empty_but_not_obsolete else logging.DEBUG
+ if p in past_mistakes.empty_but_not_obsolete:
+ lvl = logging.DEBUG
+ else:
+ lvl = logging.ERROR
+ error = True
logging.log(lvl, "package '%s' version '%s' has empty install tar file and no source, but it's not in the _obsolete category" % (p, vr))
# make another pass to verify a source tarfile exists for every install
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index ab925a1..de99439 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -114,19 +114,15 @@ nonunique_versions = [
'gnome-panel-doc',
'gtk2.0-engines-svg',
'guile-doc',
- 'guile-gv', # dropped pending guile-2
- 'info', # something went wrong with package build?
'kdepasswd', # dropped from split kde-baseapps
'kdialog', # split out from kde-baseapps
'keditbookmarks', # split out from kde-baseapps
'kexi', # split out from calligra
'kfind', # split out from kde-baseapps
'kfilereplace', # split out from kdewebdev
- 'khelpcenter', # split out from kde-runtime
'kimagemapeditor', # split out from kdewebdev
'klinkstatus', # split out from kdewebdev
'konqueror', # split out from kde-baseapps
- 'konqueror-devel', # split out from kde-baseapps
'libatomic_ops-devel', # separated out from libgc
'libcaca-doc', # dropped pending fix for current doxygen
'libfltk-doc',
@@ -139,68 +135,31 @@ nonunique_versions = [
'minizip',
'mutter-doc',
'ocaml-camlp4', # ocaml-camlp4 removed from ocaml distribution after 4.01.0
- 'ocaml-gv', # dropped pending ocaml cleanup
'okular4-part', # changed to okular5-part in 17.04
- 'python-clang', # split out from clang
- 'python3-clang', # split out from clang
'python-spiceclientgtk', # gtk2 dropped from spice-gtk
'sng-debuginfo',
'socat-debuginfo', # debuginfo for test version when curr has no debuginfo
'sqlite3-zlib', # sqlite3-zlib removed in 3.8.10, use sqlite3-compress instead
- 'texinfo-debuginfo', # something went wrong with package build?
'w3m-img',
]
# packages with an empty install file, no source, but aren't obsolete
empty_but_not_obsolete = [
- 'gcc-java', # gcc-java will be obsolete in test version 6.3
- 'gdal-debuginfo', # version 2.1.2-1 has empty debuginfo
- 'libgcj-common',
- 'libgcj16',
- 'libpopt0', # version 1.16-1 was empty (x8_64)
- 'libpopt-devel', #
- 'mutt-debuginfo', # version 1.7.2-1 has empty debuginfo
+ 'libpopt0', # version 1.16-1 was empty
+ 'libpopt-devel', # version 1.16-1 was empty (x86_64)
]
# packages with timestamp anomalies
mtime_anomalies = [
- 'gcc-ada',
- 'gcc-cilkplus',
- 'gcc-core',
- 'gcc-debuginfo',
- 'gcc-fortran',
- 'gcc-g++',
'gcc-java',
- 'gcc-objc',
- 'gcc-objc++',
'gcc-tools-epoch2-autoconf',
'glproto',
'gv-debuginfo',
- 'info',
- 'libatomic1',
- 'libcilkrts5',
- 'libgcc1',
'libgcj-common',
'libgcj16',
- 'libgfortran3',
- 'libgomp1',
- 'libobjc4',
- 'libquadmath0',
- 'libssp0',
- 'libstdc++6',
- 'opencv', # 2.4.11-1 was created when changed from source-only to install
'python-gtk2.0',
'python-gtk2.0-demo',
'python-gtk2.0-devel',
- 'subversion',
- 'subversion-debuginfo',
- 'subversion-devel',
- 'subversion-gnome',
- 'subversion-httpd',
- 'subversion-perl',
- 'subversion-python',
- 'subversion-ruby',
- 'subversion-tools',
- 'texinfo-debuginfo',
+ 'python-wx', # timestamps reset when split out from wxWidgets
'xextproto',
]
diff --git a/test/test_calm.py b/test/test_calm.py
index d1cf3af..0f30e85 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -172,6 +172,68 @@ class CalmTest(unittest.TestCase):
["2a", "2.0", -1],
["1.0", "1.fc4", 1],
["3.0.0_fc", "3.0.0.fc", 0],
+ # from RPM tests
+ ["1.0", "1.0", 0],
+ ["1.0", "2.0", -1],
+ ["2.0", "1.0", 1],
+ ["2.0.1", "2.0.1", 0],
+ ["2.0", "2.0.1", -1],
+ ["2.0.1", "2.0", 1],
+ ["2.0.1a", "2.0.1a", 0],
+ ["2.0.1a", "2.0.1", 1],
+ ["2.0.1", "2.0.1a", -1],
+ ["5.5p1", "5.5p1", 0],
+ ["5.5p1", "5.5p2", -1],
+ ["5.5p2", "5.5p1", 1],
+ ["5.5p10", "5.5p10", 0],
+ ["5.5p1", "5.5p10", -1],
+ ["5.5p10", "5.5p1", 1],
+ ["10xyz", "10.1xyz", -1],
+ ["10.1xyz", "10xyz", 1],
+ ["xyz10", "xyz10", 0],
+ ["xyz10", "xyz10.1", -1],
+ ["xyz10.1", "xyz10", 1],
+ ["xyz.4", "xyz.4", 0],
+ ["xyz.4", "8", -1],
+ ["8", "xyz.4", 1],
+ ["xyz.4", "2", -1],
+ ["2", "xyz.4", 1],
+ ["5.5p2", "5.6p1", -1],
+ ["5.6p1", "5.5p2", 1],
+ ["5.6p1", "6.5p1", -1],
+ ["6.5p1", "5.6p1", 1],
+ ["6.0.rc1", "6.0", 1],
+ ["6.0", "6.0.rc1", -1],
+ ["10b2", "10a1", 1],
+ ["10a2", "10b2", -1],
+ ["1.0aa", "1.0aa", 0],
+ ["1.0a", "1.0aa", -1],
+ ["1.0aa", "1.0a", 1],
+ ["10.0001", "10.0001", 0],
+ ["10.0001", "10.1", 0],
+ ["10.1", "10.0001", 0],
+ ["10.0001", "10.0039", -1],
+ ["10.0039", "10.0001", 1],
+ ["4.999.9", "5.0", -1],
+ ["5.0", "4.999.9", 1],
+ ["20101121", "20101121", 0],
+ ["20101121", "20101122", -1],
+ ["20101122", "20101121", 1],
+ ["2_0", "2_0", 0],
+ ["2.0", "2_0", 0],
+ ["2_0", "2.0", 0],
+ ["a", "a", 0],
+ ["a+", "a+", 0],
+ ["a+", "a_", 0],
+ ["a_", "a+", 0],
+ ["+a", "+a", 0],
+ ["+a", "_a", 0],
+ ["_a", "+a", 0],
+ ["+_", "+_", 0],
+ ["_+", "+_", 0],
+ ["_+", "_+", 0],
+ ["+", "_", 0],
+ ["_", "+", 0],
]
for d in test_data:
@@ -314,6 +376,43 @@ class CalmTest(unittest.TestCase):
# remove !ready files
os.system("find testdata/homes -name !ready -exec rm {} \;")
+ # fix up package timestamps so highest version is also latest
+ # (git doesn't store timestamps, so they will all be dated the time of checkout)
+ relarea_x86 = os.path.join('testdata', 'relarea', 'x86', 'release')
+ relarea_noarch = os.path.join('testdata', 'relarea', 'noarch', 'release')
+ touches = [(os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1.tar.xz'), '2016-11-01'),
+ (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1-src.tar.xz'), '2016-11-01'),
+ (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.1-1.tar.xz'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.1-1-src.tar.xz'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'cygwin', 'cygwin-debuginfo', 'cygwin-debuginfo-2.2.0-1.tar.xz'), '2016-11-01'),
+ (os.path.join(relarea_x86, 'cygwin', 'cygwin-debuginfo', 'cygwin-debuginfo-2.2.1-1.tar.xz'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'cygwin', 'cygwin-devel', 'cygwin-devel-2.2.0-1.tar.xz'), '2016-11-01'),
+ (os.path.join(relarea_x86, 'cygwin', 'cygwin-devel', 'cygwin-devel-2.2.1-1.tar.xz'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'base-cygwin', 'base-cygwin-3.6-1.tar.xz'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'per-version', 'per-version-4.0-1.tar.xz'), '2017-04-09'),
+ (os.path.join(relarea_x86, 'per-version', 'per-version-4.0-1-src.tar.xz'), '2017-04-09'),
+ (os.path.join(relarea_x86, 'rpm-doc', 'rpm-doc-4.1-2.tar.bz2'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'rpm-doc', 'rpm-doc-4.1-2-src.tar.bz2'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-240-1.tar.xz'), '2017-04-07'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-240-1-src.tar.xz'), '2017-04-07'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-242-0.tar.xz'), '2017-04-08'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-242-0-src.tar.xz'), '2017-04-08'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-243-0.tar.xz'), '2017-04-09'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-243-0-src.tar.xz'), '2017-04-09'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-250-0.tar.xz'), '2017-04-10'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-250-0-src.tar.xz'), '2017-04-10'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-251-0.tar.xz'), '2017-04-09'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-251-0-src.tar.xz'), '2017-04-09'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-260-0.tar.xz'), '2017-04-12'),
+ (os.path.join(relarea_x86, 'staleversion', 'staleversion-260-0-src.tar.xz'), '2017-04-12'),
+ (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1.tar.bz2'), '2016-11-02'),
+ (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1-src.tar.bz2'), '2016-11-02'),
+ (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1.tar.xz'), '2016-11-01'),
+ (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1-src.tar.xz'), '2016-11-01'),
+ ]
+ for (f, t) in touches:
+ os.system('touch %s -d %s' % (f, t))
+
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)