This is the mail archive of the cygwin-apps-cvs mailing list for the cygwin-apps project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[calm - Cygwin server-side packaging maintenance script] branch master, updated. b26ae5b83bad5b1c12ef7d59377278059f0ede5d




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=b26ae5b83bad5b1c12ef7d59377278059f0ede5d

commit b26ae5b83bad5b1c12ef7d59377278059f0ede5d
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed Mar 23 10:26:55 2016 +0000

    Ignore packages marked as OBSOLETE in cygwin-pkg-maint
    
    Update tests appropriately

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=872274f5b91ad4cf309918d25b9c02ec7e5b08fc

commit 872274f5b91ad4cf309918d25b9c02ec7e5b08fc
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Mar 21 20:37:40 2016 +0000

    Remove shell script, do everything in python
    
    Have I mentioned lately how much I hate shell scripts...
    
    v2:
    Backup existing .ini file, rather than overwriting it
    Add an option to control where we look for setup executable


Diff:
---
 TODO                              |    1 -
 calm-cron-script                  |   36 ---------------
 calm.py                           |   86 ++++++++++++++++++++++++++++++++++---
 extract-setup-version             |   66 ----------------------------
 maintainers.py                    |   11 +++-
 setup_exe.py                      |   73 +++++++++++++++++++++++++++++++
 testdata/pkglist/cygwin-pkg-maint |    2 +-
 testdata/pkglist/expected         |    2 +-
 tests.py                          |    2 +-
 9 files changed, 164 insertions(+), 115 deletions(-)

diff --git a/TODO b/TODO
index d782f11..246aac2 100644
--- a/TODO
+++ b/TODO
@@ -1,4 +1,3 @@
-* cronscript should be in python
 * per-package-version hint files (e.g. named <pvr>.hint, or /.setup.hint inside tar file)
 * more than 2 versions possible (and automatically vault 'old' versions, where we have some mechanism to explicity say what is old)
 * enable .xz compressed setup.ini (setup.exe checks for it and presumably can handle it)
diff --git a/calm-cron-script b/calm-cron-script
deleted file mode 100755
index 554c868..0000000
--- a/calm-cron-script
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-
-# a unicode locale is required
-export LANG=en_US.UTF-8
-
-FTP=/var/ftp/pub/cygwin
-SRCDIR=`/usr/bin/dirname "$0"`
-
-for arch in x86_64 x86 ; do
-    TMPFILE=`/bin/mktemp -t setup.ini.${arch}XXX`
-    INIFILE=${FTP}/${arch}/setup.ini
-
-    SETUP_VERSION=`${SRCDIR}/extract-setup-version /www/sourceware/htdocs/cygwin/setup-${arch}.exe`
-
-    ${SRCDIR}/calm --inifile ${TMPFILE} --setup-version ${SETUP_VERSION} --arch=${arch} --email
-    if [ $? -eq 0 ]
-    then
-        # if changed in more than timestamp
-        /usr/bin/diff -I^setup-timestamp -w -B -q ${INIFILE} ${TMPFILE} >/dev/null
-        if [ $? -eq 1 ]
-        then
-            # update setup.ini
-            /bin/mv ${TMPFILE} ${INIFILE}
-            # compress
-            /usr/bin/bzip2 <${INIFILE} >${INIFILE/%.ini/.bz2}
-            # re-sign
-            /bin/rm -f ${INIFILE}.sig ${INIFILE/%.ini/.bz2}.sig
-            /usr/bin/gpg --batch --yes -b ${INIFILE}
-            /usr/bin/gpg --batch --yes -b ${INIFILE/%.ini/.bz2}
-            # arrange for checksums to be recomputed
-            /bin/rm -f ${FTP}/${arch}/md5.sum ${FTP}/${arch}/sha512.sum
-        fi
-    fi
-    /bin/rm -f ${TMPFILE}
-done
diff --git a/calm.py b/calm.py
index d4568d4..6cbc690 100755
--- a/calm.py
+++ b/calm.py
@@ -48,13 +48,16 @@
 import argparse
 import logging
 import os
+import shutil
 import sys
+import tempfile
 
 from buffering_smtp_handler import mail_logs
 import common_constants
 import maintainers
 import package
 import pkg2html
+import setup_exe
 import uploads
 
 
@@ -62,7 +65,7 @@ import uploads
 #
 #
 
-def main(args):
+def process_arch(args):
     details = '%s%s' % (args.arch, ',dry-run' if args.dryrun else '')
 
     # send one email per run to leads
@@ -76,7 +79,7 @@ def main(args):
         # validate the package set
         if not package.validate_packages(args, packages):
             logging.error("existing package set has errors, not processing uploads or writing setup.ini")
-            return 1
+            return False
 
         # read maintainer list
         mlist = maintainers.Maintainer.read(args)
@@ -133,7 +136,79 @@ def main(args):
         # XXX: perhaps we need a --[no]listing command line option to disable this from being run?
         pkg2html.update_package_listings(args, packages)
 
-        return 0
+        return True
+
+
+#
+#
+#
+
+def main(args):
+    # for each arch
+    for arch in common_constants.ARCHES:
+        logging.info("processing arch %s" % (arch))
+
+        args.arch = arch
+        args.setup_version = setup_exe.extract_version(os.path.join(args.setupdir, 'setup-' + args.arch + '.exe'))
+        logging.info("setup version is '%s'" % (args.setup_version))
+
+        basedir = os.path.join(args.rel_area, args.arch)
+        inifile = os.path.join(basedir, 'setup.ini')
+
+        # write setup.ini to a temporary file
+        with tempfile.NamedTemporaryFile(delete=False) as tmpfile:
+            args.inifile = tmpfile.name
+
+            changed = False
+            if not process_arch(args):
+                # generating setup.ini failed
+                pass
+            elif not os.path.exists(inifile):
+                # if the setup.ini file doesn't exist yet
+                logging.warning('no existing %s' % (inifile))
+                changed = True
+            else:
+                # or, if it's changed in more than timestamp
+                status = os.system('/usr/bin/diff -I^setup-timestamp -w -B -q %s %s >/dev/null' % (inifile, tmpfile.name))
+                logging.info('diff exit status %d' % (status))
+                if (status >> 8) == 1:
+                    changed = True
+
+            # then update setup.ini
+            if changed:
+                if args.dryrun:
+                    logging.warning("not moving %s to %s, due to --dry-run" % (tmpfile.name, inifile))
+                    os.remove(tmpfile.name)
+                else:
+                    # make a backup of the current setup.ini
+                    shutil.copy2(inifile, inifile + '.bak')
+
+                    # replace setup.ini
+                    logging.warning("moving %s to %s" % (tmpfile.name, inifile))
+                    os.rename(tmpfile.name, inifile)
+
+                    # compress and re-sign
+                    for ext in ['.ini', '.bz2']:
+                        try:
+                            os.remove(os.path.join(basedir, 'setup' + ext + '.sig'))
+                        except FileNotFoundError:
+                            pass
+
+                        if ext == '.bz2':
+                            os.system('/usr/bin/bzip2 <%s >%s' % (inifile, os.path.splitext(inifile)[0] + ext))
+
+                        os.system('/usr/bin/gpg --batch --yes -b ' + os.path.join(basedir, 'setup' + ext))
+
+                    # arrange for checksums to be recomputed
+                    for sumfile in ['md5.sum', 'sha512.sum']:
+                        try:
+                            os.remove(os.path.join(basedir, sumfile))
+                        except FileNotFoundError:
+                            pass
+            else:
+                logging.info("removing %s, unchanged %s" % (tmpfile.name, inifile))
+                os.remove(tmpfile.name)
+
 
 #
 #
@@ -145,22 +220,21 @@ if __name__ == "__main__":
     orphanmaint_default = common_constants.ORPHANMAINT
     pkglist_default = common_constants.PKGMAINT
     relarea_default = common_constants.FTP
+    setupdir_default = common_constants.HTDOCS
     vault_default = common_constants.VAULT
     logdir_default = '/sourceware/cygwin-staging/logs'
 
     parser = argparse.ArgumentParser(description='Upset replacement')
-    parser.add_argument('--arch', action='store', required=True, choices=common_constants.ARCHES)
     parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help='email output to maintainer and ADDRS (default: ' + common_constants.EMAILS + ')', metavar='ADDRS')
     parser.add_argument('--force', action='store_true', help="overwrite existing files")
     parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default)
     parser.add_argument('--htdocs', action='store', metavar='DIR', help="htdocs output directory (default: " + htdocs_default + ")", default=htdocs_default)
-    parser.add_argument('--inifile', '-u', action='store', help='output filename', required=True)
     parser.add_argument('--logdir', action='store', metavar='DIR', help="log directory (default: '" + logdir_default + "')", default=logdir_default)
     parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default)
     parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default)
     parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin')
     parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
-    parser.add_argument('--setup-version', action='store', metavar='VERSION', help='value for setup-version key')
+    parser.add_argument('--setupdir', action='store', metavar='DIR', help="setup executable directory (default: " + setupdir_default + ")", default=setupdir_default)
     parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun', help="don't do anything")
     parser.add_argument('--vault', action='store', metavar='DIR', help="vault directory (default: " + vault_default + ")", default=vault_default, dest='vault')
     parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output')
diff --git a/extract-setup-version b/extract-setup-version
deleted file mode 100755
index 23eac1c..0000000
--- a/extract-setup-version
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (c) 2015 Jon Turney
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-#
-# extract version from setup executable
-#
-
-import argparse
-import os
-import re
-
-
-#
-# the setup binary contains a '%%% setup-version ' string identifying the
-# version, but it's hard to read that when it's UPX packed.
-#
-# so instead, we expect the exe file to be a symlink to a file which encodes
-# the version in it's name
-#
-# XXX: possibly we could work around this by placing the version string into
-# a string resource and using upx's --keep-resource flag to keep that resource
-# uncompressed
-#
-
-def main(fn):
-    # canonicalize the pathname
-    fn = os.path.realpath(fn)
-
-    match = re.search(r'setup-([\d\.]+).x86', fn)
-    if match:
-        print(match.group(1))
-        return 0
-    else:
-        return 1
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Extract version from setup executable')
-    parser.add_argument('exe', action='store', nargs='?', metavar='filename', help='executable file')
-    (args) = parser.parse_args()
-
-    if args.exe:
-        exit(main(args.exe))
-
-    parser.print_help()
-
-# XXX:maybe this could make the filename from /www/sourceware/htdocs/cygwin and arch?
diff --git a/maintainers.py b/maintainers.py
index d0afb0f..8bc9ec7 100644
--- a/maintainers.py
+++ b/maintainers.py
@@ -101,13 +101,18 @@ class Maintainer(object):
                     pkg = match.group(1)
                     m = match.group(2)
 
+                    # if maintainer starts with a word in all caps, just use that
+                    m = re.sub(r'^([A-Z]+)\b.*$', r'\1', m)
+
+                    # ignore packages marked as 'OBSOLETE'
+                    if m == 'OBSOLETE':
+                        continue
+
                     # orphaned packages get the default maintainer if we have
                     # one, otherwise are assigned to 'ORPHANED'
-                    if m.startswith('ORPHANED'):
+                    if m == 'ORPHANED':
                         if orphanMaint is not None:
                             m = orphanMaint
-                        else:
-                            m = 'ORPHANED'
 
                     # joint maintainers are separated by '/'
                     for name in m.split('/'):
diff --git a/setup_exe.py b/setup_exe.py
new file mode 100755
index 0000000..d9d85be
--- /dev/null
+++ b/setup_exe.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2015 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+#
+# extract version from setup executable
+#
+
+import argparse
+import os
+import re
+
+
+#
+# the setup binary contains a '%%% setup-version ' string identifying the
+# version, but it's hard to read that when it's UPX packed.
+#
+# so instead, we expect the exe file to be a symlink to a file which encodes
+# the version in it's name
+#
+# XXX: possibly we could work around this by placing the version string into
+# a string resource and using upx's --keep-resource flag to keep that resource
+# uncompressed
+#
+
+def extract_version(fn):
+    # check the file exists
+    if not os.path.exists(fn):
+        raise FileNotFoundError
+
+    # canonicalize the pathname
+    fn = os.path.realpath(fn)
+
+    match = re.search(r'setup-([\d\.]+).x86', fn)
+    if match:
+        return match.group(1)
+    else:
+        return None
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Extract version from setup executable')
+    parser.add_argument('exe', action='store', nargs='?', metavar='filename', help='executable file')
+    (args) = parser.parse_args()
+
+    if args.exe:
+        v = extract_version(args.exe)
+        if v:
+            print(v)
+            exit(0)
+        exit(1)
+
+    parser.print_help()
+
+# XXX:maybe this could make the filename from /www/sourceware/htdocs/cygwin and arch?
diff --git a/testdata/pkglist/cygwin-pkg-maint b/testdata/pkglist/cygwin-pkg-maint
index d2375af..41b4175 100644
--- a/testdata/pkglist/cygwin-pkg-maint
+++ b/testdata/pkglist/cygwin-pkg-maint
@@ -1744,7 +1744,7 @@ presentproto                                 Yaakov Selkowitz
 prison                                       Yaakov Selkowitz
 pristine-tar                                 Jari Aalto
 procmail                                     Daniel Boland
-procps                                       Corinna Vinschen
+procps                                       OBSOLETE (Corinna Vinschen)
 proj                                         Marco Atzeri
 protobuf                                     ORPHANED (Reini Urban)
 pscan                                        Jari Aalto
diff --git a/testdata/pkglist/expected b/testdata/pkglist/expected
index 4e8db92..b340aa2 100644
--- a/testdata/pkglist/expected
+++ b/testdata/pkglist/expected
@@ -9,7 +9,7 @@
  'Chris LeBlanc': maintainers.Maintainer('Chris LeBlanc', [], ['python-h5py', 'python3-h5py']),
  'Christian Franke': maintainers.Maintainer('Christian Franke', [], ['busybox', 'cdrkit', 'ddrescue', 'hostname', 'isomaster', 'ncdu', 'postfix', 'smartmontools']),
  'Christian Kellermann': maintainers.Maintainer('Christian Kellermann', [], ['chicken']),
- 'Corinna Vinschen': maintainers.Maintainer('Corinna Vinschen', [], ['attr', 'base-cygwin', 'catgets', 'cpio', 'crypt', 'csih', 'cygrunsrv', 'cygwin', 'cygwin-devel', 'editrights', 'email', 'eventlog', 'fetchmail', 'file', 'gawk', 'gdb', 'getent', 'ipc-utils', 'libedit', 'login', 'lynx', 'mt', 'nc', 'nc6', 'openssh', 'openssl', 'patch', 'pl', 'procps', 'psmisc', 'rebase', 'recode', 'rlwrap', 'rng-tools', 'robots', 'sed', 'ssmtp', 'syslog-ng', 'tcsh', 'tin', 'tzcode', 'windows-default-manifest', 'xinetd']),
+ 'Corinna Vinschen': maintainers.Maintainer('Corinna Vinschen', [], ['attr', 'base-cygwin', 'catgets', 'cpio', 'crypt', 'csih', 'cygrunsrv', 'cygwin', 'cygwin-devel', 'editrights', 'email', 'eventlog', 'fetchmail', 'file', 'gawk', 'gdb', 'getent', 'ipc-utils', 'libedit', 'login', 'lynx', 'mt', 'nc', 'nc6', 'openssh', 'openssl', 'patch', 'pl', 'psmisc', 'rebase', 'recode', 'rlwrap', 'rng-tools', 'robots', 'sed', 'ssmtp', 'syslog-ng', 'tcsh', 'tin', 'tzcode', 'windows-default-manifest', 'xinetd']),
  'Damien Doligez': maintainers.Maintainer('Damien Doligez', [], ['ocaml']),
  'Daniel Boland': maintainers.Maintainer('Daniel Boland', [], ['libfakesu', 'procmail', 'sendmail']),
  'Dave Kilroy': maintainers.Maintainer('Dave Kilroy', [], ['chere']),
diff --git a/tests.py b/tests.py
index 6f0ff19..3252ac6 100755
--- a/tests.py
+++ b/tests.py
@@ -262,7 +262,7 @@ class TestMain(unittest.TestCase):
         for (f, t) in ready_fns:
             os.system('touch %s "%s"' % (t, f))
 
-        self.assertEqual(calm.main(args), 0)
+        self.assertEqual(calm.process_arch(args), True)
 
         for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
             with self.subTest(directory=d):


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]