This is the mail archive of the cygwin-apps-cvs mailing list for the cygwin-apps project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 1d2e982935cafa1dacb323f0238d892087446746




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1d2e982935cafa1dacb323f0238d892087446746

commit 1d2e982935cafa1dacb323f0238d892087446746
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sun May 8 12:37:08 2016 +0000

    Add noarch support
    
    - Unfold process() as we need to validate both x86 and x86_64 before we can
    move uploaded noarch packages to relarea
    - Add arch as an argument to upload.scan(), package.write_setup_ini(),
    pkg2html.update_package_listing() etc. rather than passing it around in args
    - Use paths relative to relarea, rather than relarea/arch
    - Generalize package merge to merge more than 2 sets of packages
    - Remove uploads-scan script used in working-up
    - Move the !reminder-timestamp file up to maintainer home dir
    - Add a noarch package to testdata
    - Update tests


Diff:
---
 calm.py                                            |  171 +++++++++++++-------
 mksetupini                                         |    2 +-
 package.py                                         |   81 +++++-----
 pkg2html.py                                        |   17 ++-
 queue.py                                           |    4 +-
 .../noarch/release/perl-Net-SMTP-SSL/expected      |    5 +
 .../perl-Net-SMTP-SSL-1.03-2-src.tar.xz            |  Bin 0 -> 2752 bytes
 .../perl-Net-SMTP-SSL-1.03-2.tar.xz                |  Bin 0 -> 3180 bytes
 .../noarch/release/perl-Net-SMTP-SSL/setup.hint    |    5 +
 testdata/htdocs.expected/x86/packages.inc          |    1 +
 .../x86/perl-Net-SMTP-SSL/.htaccess                |    3 +
 .../x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1 |   21 +++
 .../perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src |    8 +
 testdata/inifile/setup.ini.expected                |   11 ++
 testdata/pkglist/cygwin-pkg-maint                  |    2 +-
 testdata/pkglist/expected                          |    2 +-
 testdata/process_arch/homedir.expected             |    3 +
 testdata/process_arch/htdocs.expected              |    5 +
 testdata/process_arch/rel_area.expected            |    8 +
 .../perl-Net-SMTP-SSL-1.03-1-src.tar.xz            |  Bin 0 -> 2752 bytes
 .../perl-Net-SMTP-SSL-1.03-1.tar.xz                |  Bin 0 -> 3180 bytes
 .../noarch/release/perl-Net-SMTP-SSL/setup.hint    |    5 +
 testdata/uploads/move.expected                     |    6 +-
 testdata/uploads/pkglist.expected                  |    6 +-
 tests.py                                           |   23 ++-
 upload-scan                                        |  109 -------------
 uploads.py                                         |   16 +-
 27 files changed, 271 insertions(+), 243 deletions(-)

diff --git a/calm.py b/calm.py
index ae48916..4393597 100755
--- a/calm.py
+++ b/calm.py
@@ -26,22 +26,27 @@
 #
 
 #
-# read packages from release area
+# for each arch
+# - read and validate packages from release area
+# - stop if there are errors
+# otherwise,
 # for each maintainer
 # - read and validate any package uploads
 # - build a list of files to move and remove
-# - merge package sets
-# - remove from the package set files which are to be removed
-# - validate merged package set
-# - process remove list
+# - for each arch
+# -- merge package sets
+# -- remove from the package set files which are to be removed
+# -- validate merged package set
+# -- process remove list
 # - on failure
 # -- mail maintainer with errors
 # -- empty move list
-# -- discard merged package set
+# -- discard merged package sets
 # - on success
 # -- process move list
 # -- mail maintainer with movelist
-# -- continue with merged package set
+# -- continue with merged package sets
+# write package listings
 # write setup.ini file
 #
 
@@ -68,22 +73,26 @@ import uploads
 #
 #
 
-def process_arch(args):
-    subject = 'calm: cygwin package upload report from %s' % (os.uname()[1])
-    details = '%s%s' % (args.arch, ',dry-run' if args.dryrun else '')
+def process(args):
+    subject = 'calm%s: cygwin package upload report from %s' % (' [dry-run]' if args.dryrun else '', os.uname()[1])
 
     # send one email per run to leads, if any errors occurred
-    with mail_logs(args.email, toaddrs=args.email, subject='%s [%s]' % (subject, details), thresholdLevel=logging.ERROR) as leads_email:
+    with mail_logs(args.email, toaddrs=args.email, subject='%s' % (subject), thresholdLevel=logging.ERROR) as leads_email:
         if args.dryrun:
             logging.warning("--dry-run is in effect, nothing will really be done")
 
-        # build package list
-        packages = package.read_packages(args.rel_area, args.arch)
+        # for each arch
+        packages = {}
+        for arch in common_constants.ARCHES:
+            logging.debug("reading existing packages for arch %s" % (arch))
 
-        # validate the package set
-        if not package.validate_packages(args, packages):
-            logging.error("existing package set has errors, not processing uploads or writing setup.ini")
-            return False
+            # build package list
+            packages[arch] = package.read_packages(args.rel_area, arch)
+
+            # validate the package set
+            if not package.validate_packages(args, packages[arch]):
+                logging.error("existing %s package set has errors", arch)
+                return None
 
         # read maintainer list
         mlist = maintainers.Maintainer.read(args)
@@ -96,57 +105,83 @@ def process_arch(args):
             m = mlist[name]
 
             # also send a mail to each maintainer about their packages
-            with mail_logs(args.email, toaddrs=m.email, subject='%s for %s [%s]' % (subject, name, details), thresholdLevel=logging.INFO) as maint_email:
+            with mail_logs(args.email, toaddrs=m.email, subject='%s for %s' % (subject, name), thresholdLevel=logging.INFO) as maint_email:
 
-                scan_result = uploads.scan(m, all_packages, args)
+                # for each arch and noarch
+                scan_result = {}
+                skip_maintainer = False
+                for arch in common_constants.ARCHES + ['noarch']:
+                    logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name))
 
-                uploads.remove(args, scan_result.remove_always)
+                    # read uploads
+                    scan_result[arch] = uploads.scan(m, all_packages, arch, args)
 
-                if scan_result.error:
-                    logging.error("error while reading uploaded packages for %s" % (name))
-                    continue
+                    # remove triggers
+                    uploads.remove(args, scan_result[arch].remove_always)
+
+                    if scan_result[arch].error:
+                        logging.error("error while reading uploaded arch %s packages from maintainer %s" % (arch, name))
+                        skip_maintainer = True
+                        continue
+
+                    # queue for source package validator
+                    queue.add(args, scan_result[arch].to_relarea, os.path.join(m.homedir()))
 
                 # if there are no uploaded or removed packages for this
                 # maintainer, we don't have anything to do
-                if not scan_result.packages and not scan_result.to_vault:
+                if not any([scan_result[a].packages or scan_result[a].to_vault for a in scan_result]):
                     logging.debug("nothing to do for maintainer %s" % (name))
+                    skip_maintainer = True
+
+                if skip_maintainer:
                     continue
 
-                # queue for source package validator
-                queue.add(args, scan_result.to_relarea, os.path.join(m.homedir(), args.arch))
-
-                # merge package set
-                merged_packages = package.merge(packages, scan_result.packages)
-
-                # remove file which are to be removed
-                #
-                # XXX: this doesn't properly account for removing setup.hint
-                # files
-                for p in scan_result.to_vault:
-                    for f in scan_result.to_vault[p]:
-                        package.delete(merged_packages, p, f)
-
-                # validate the package set
-                if package.validate_packages(args, merged_packages):
-                    # process the move list
-                    uploads.move_to_vault(args, scan_result.to_vault)
-                    uploads.remove(args, scan_result.remove_success)
-                    uploads.move_to_relarea(m, args, scan_result.to_relarea)
-                    # use merged package list
-                    packages = merged_packages
-                    logging.debug("added %d packages from maintainer %s" % (len(scan_result.packages), name))
-                else:
-                    # otherwise we discard move list and merged_packages
-                    logging.error("error while merging uploaded packages for %s" % (name))
+                # for each arch
+                merged_packages = {}
+                valid = True
+                for arch in common_constants.ARCHES:
+                    logging.debug("merging %s package set with uploads from maintainer %s" % (arch, name))
+
+                    # merge package sets
+                    merged_packages[arch] = package.merge(packages[arch], scan_result[arch].packages, scan_result['noarch'].packages)
+                    if not merged_packages[arch]:
+                        valid = False
+                        break
+
+                    # remove files which are to be removed
+                    #
+                    # XXX: this doesn't properly account for removing setup.hint
+                    # files
+                    for p in scan_result[arch].to_vault:
+                        for f in scan_result[arch].to_vault[p]:
+                            package.delete(merged_packages[arch], p, f)
+
+                    # validate the package set
+                    logging.debug("validating merged %s package set for maintainer %s" % (arch, name))
+                    if not package.validate_packages(args, merged_packages[arch]):
+                        valid = False
+
+                if not valid:
+                    # discard move list and merged_packages
+                    logging.error("error while merging uploaded %s packages for %s" % (arch, name))
+                    continue
 
-        # write setup.ini
-        package.write_setup_ini(args, packages)
+                # for each arch and noarch
+                for arch in common_constants.ARCHES + ['noarch']:
+                    logging.debug("moving %s packages for maintainer %s" % (arch, name))
 
-        # update packages listings
-        # XXX: perhaps we need a --[no]listing command line option to disable this from being run?
-        pkg2html.update_package_listings(args, packages)
+                    # process the move lists
+                    uploads.move_to_vault(args, scan_result[arch].to_vault)
+                    uploads.remove(args, scan_result[arch].remove_success)
+                    uploads.move_to_relarea(m, args, scan_result[arch].to_relarea)
 
-        return True
+                # for each arch
+                for arch in common_constants.ARCHES:
+                    # use merged package list
+                    packages[arch] = merged_packages[arch]
+                    logging.debug("added %d + %d packages from maintainer %s" % (len(scan_result[arch].packages), len(scan_result['noarch'].packages), name))
+
+    return packages
 
 
 #
@@ -154,9 +189,22 @@ def process_arch(args):
 #
 
 def main(args):
+    # read package set and process uploads
+    packages = process(args)
+
+    if not packages:
+        logging.error("not processing uploads or writing setup.ini")
+        return
+
+    # for each arch
+    for arch in common_constants.ARCHES:
+        # update packages listings
+        # XXX: perhaps we need a --[no]listing command line option to disable this from being run?
+        pkg2html.update_package_listings(args, packages[arch], arch)
+
     # for each arch
     for arch in common_constants.ARCHES:
-        logging.debug("processing arch %s" % (arch))
+        logging.debug("writing setup.ini for arch %s" % (arch))
 
         args.arch = arch
         args.setup_version = setup_exe.extract_version(os.path.join(args.setupdir, 'setup-' + args.arch + '.exe'))
@@ -170,10 +218,11 @@ def main(args):
             args.inifile = tmpfile.name
 
             changed = False
-            if not process_arch(args):
-                # generating setup.ini failed
-                pass
-            elif not os.path.exists(inifile):
+
+            # write setup.ini
+            package.write_setup_ini(args, packages[arch], arch)
+
+            if not os.path.exists(inifile):
                 # if the setup.ini file doesn't exist yet
                 logging.warning('no existing %s' % (inifile))
                 changed = True
diff --git a/mksetupini b/mksetupini
index 4c3a3d5..f87a53c 100755
--- a/mksetupini
+++ b/mksetupini
@@ -63,7 +63,7 @@ def main(args):
         return
 
     # write setup.ini
-    package.write_setup_ini(args, packages)
+    package.write_setup_ini(args, packages, args.arch)
 
     if args.stats:
         stats(packages)
diff --git a/package.py b/package.py
index 0c5fbeb..b515c2e 100755
--- a/package.py
+++ b/package.py
@@ -74,13 +74,15 @@ class Tar(object):
 def read_packages(rel_area, arch):
     packages = defaultdict(Package)
 
-    releasedir = os.path.join(rel_area, arch)
-    logging.debug('reading packages from %s' % releasedir)
+    # both noarch/ and <arch>/ directories are considered
+    for root in ['noarch', arch]:
+        releasedir = os.path.join(rel_area, root)
+        logging.debug('reading packages from %s' % releasedir)
 
-    for (dirpath, subdirs, files) in os.walk(releasedir):
-        read_package(packages, releasedir, dirpath, files)
+        for (dirpath, subdirs, files) in os.walk(releasedir):
+            read_package(packages, rel_area, dirpath, files)
 
-    logging.debug("%d packages read" % len(packages))
+        logging.debug("%d packages read" % len(packages))
 
     return packages
 
@@ -223,7 +225,7 @@ def read_package(packages, basedir, dirpath, files, strict=False):
                     logging.log(strict_lvl, "package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2))))
                     warnings = True
 
-    elif (len(files) > 0) and (relpath.count(os.path.sep) > 0):
+    elif (len(files) > 0) and (relpath.count(os.path.sep) > 1):
         logging.warning("no setup.hint in %s but has files: %s" % (dirpath, ', '.join(files)))
 
     if strict:
@@ -494,7 +496,7 @@ def validate_package_maintainers(args, packages):
 #
 # write setup.ini
 #
-def write_setup_ini(args, packages):
+def write_setup_ini(args, packages, arch):
 
     logging.debug('writing %s' % (args.inifile))
 
@@ -510,7 +512,7 @@ def write_setup_ini(args, packages):
 
         if args.release:
             print("release: %s" % args.release, file=f)
-        print("arch: %s" % args.arch, file=f)
+        print("arch: %s" % arch, file=f)
         print("setup-timestamp: %d" % time.time(), file=f)
         if args.setup_version:
             print("setup-version: %s" % args.setup_version, file=f)
@@ -550,18 +552,18 @@ def write_setup_ini(args, packages):
 
                     if 'install' in packages[p].vermap[version]:
                         t = packages[p].vermap[version]['install']
-                        tar_line('install', args.arch, packages[p], t, f)
+                        tar_line('install', packages[p], t, f)
 
                     # look for corresponding source in this package first
                     if 'source' in packages[p].vermap[version]:
                         t = packages[p].vermap[version]['source']
-                        tar_line('source', args.arch, packages[p], t, f)
+                        tar_line('source', packages[p], t, f)
                     # if that doesn't exist, follow external-source
                     elif 'external-source' in packages[p].hints:
                         s = packages[p].hints['external-source']
                         if 'source' in packages[s].vermap[version]:
                             t = packages[s].vermap[version]['source']
-                            tar_line('source', args.arch, packages[s], t, f)
+                            tar_line('source', packages[s], t, f)
                         else:
                             logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s))
 
@@ -570,8 +572,8 @@ def write_setup_ini(args, packages):
 
 
 # helper function to output details for a particular tar file
-def tar_line(category, arch, p, t, f):
-    fn = os.path.join(arch, p.path, t)
+def tar_line(category, p, t, f):
+    fn = os.path.join(p.path, t)
     sha512 = p.tars[t].sha512
     size = p.tars[t].size
     print("%s: %s %d %s" % (category, fn, size, sha512), file=f)
@@ -584,42 +586,45 @@ def upper_first_character(s):
 
 
 #
-# merge two sets of packages
+# merge sets of packages
 #
 # for each package which exist in both a and b:
-# - they must exist at the same relative path, or the package from a is used
-# - we combine the list of tarfiles, duplicates are not expected
+# - they must exist at the same relative path
+# - we combine the list of tarfiles, duplicates are not permitted
 # - we use the hints from b, and warn if they are different to the hints for a
 #
-def merge(a, b):
+def merge(a, *l):
     # start with a copy of a
     c = copy.deepcopy(a)
 
-    for p in b:
-        # if the package is in b but not in a, add it to the copy
-        if p not in a:
-            c[p] = b[p]
-        # else, if the package is both in a and b, we have to do a merge
-        else:
-            # package must exist at same relative path
-            if a[p].path != b[p].path:
-                logging.error("package '%s' is at paths %s and %s" % (p, a[p].path, b[p].path))
+    for b in l:
+        for p in b:
+            # if the package is in b but not in a, add it to the copy
+            if p not in a:
+                c[p] = b[p]
+            # else, if the package is both in a and b, we have to do a merge
             else:
-                for t in b[p].tars:
-                    if t in c[p].tars:
-                        logging.error("package '%s' has duplicate tarfile %s" % (p, t))
-                    else:
-                        c[p].tars[t] = b[p].tars[t]
+                # package must exist at same relative path
+                if a[p].path != b[p].path:
+                    logging.error("package '%s' is at paths %s and %s" % (p, a[p].path, b[p].path))
+                    return None
+                else:
+                    for t in b[p].tars:
+                        if t in c[p].tars:
+                            logging.error("package '%s' has duplicate tarfile %s" % (p, t))
+                            return None
+                        else:
+                            c[p].tars[t] = b[p].tars[t]
 
-                # use hints from b, but warn if they have changed
-                if a[p].hints != b[p].hints:
-                    c[p].hints = b[p].hints
+                    # use hints from b, but warn if they have changed
+                    if a[p].hints != b[p].hints:
+                        c[p].hints = b[p].hints
 
-                    diff = '\n'.join(difflib.ndiff(
-                        pprint.pformat(a[p].hints).splitlines(),
-                        pprint.pformat(b[p].hints).splitlines()))
+                        diff = '\n'.join(difflib.ndiff(
+                            pprint.pformat(a[p].hints).splitlines(),
+                            pprint.pformat(b[p].hints).splitlines()))
 
-                    logging.warning("package '%s' hints changed\n%s" % (p, diff))
+                        logging.warning("package '%s' hints changed\n%s" % (p, diff))
 
     return c
 
diff --git a/pkg2html.py b/pkg2html.py
index 171d618..2e16076 100755
--- a/pkg2html.py
+++ b/pkg2html.py
@@ -34,6 +34,9 @@
 # - remove any listing files for which there was no package
 # - remove any empty directories (TBD)
 #
+# note that the directory hierarchy of (noarch|arch)/package/subpackages is
+# flattened in the package listing to just the package name
+#
 
 from collections import defaultdict
 import argparse
@@ -54,8 +57,8 @@ import package
 #
 #
 
-def update_package_listings(args, packages):
-    base = os.path.join(args.htdocs, args.arch)
+def update_package_listings(args, packages, arch):
+    base = os.path.join(args.htdocs, arch)
     if not args.dryrun:
         try:
             os.makedirs(base, exist_ok=True)
@@ -77,7 +80,7 @@ def update_package_listings(args, packages):
         if not args.dryrun:
             with open(htaccess, 'w') as f:
 
-                print('Redirect temp /packages/%s/index.html https://cygwin.com/packages/package_list.html' % (args.arch),
+                print('Redirect temp /packages/%s/index.html https://cygwin.com/packages/package_list.html' % (arch),
                       file=f)
 
     toremove = glob.glob(os.path.join(base, '*', '*'))
@@ -145,7 +148,7 @@ def update_package_listings(args, packages):
                                                  <h1>%s</h1>
                                                  <tt><pre>''' % (header)), file=f)
 
-                        tf = os.path.join(args.rel_area, args.arch, packages[p].path, t)
+                        tf = os.path.join(args.rel_area, packages[p].path, t)
                         if not os.path.exists(tf):
                             # XXX: this shouldn't happen with a full mirror...
                             print('tarfile %s not found' % tf, file=f)
@@ -190,7 +193,7 @@ def update_package_listings(args, packages):
                                      <b class="rbottom"><b class="r4"></b><b class="r3"></b><b class="r2"></b><b class="r1"></b></b>
                                      </div>
                                      <br>
-                                     <table class="pkglist">''') % (args.arch, args.arch), file=index)
+                                     <table class="pkglist">''') % (arch, arch), file=index)
 
             for p in sorted(packages.keys(), key=package.sort_key):
                 # don't write anything if 'skip'
@@ -199,7 +202,7 @@ def update_package_listings(args, packages):
 
                 header = packages[p].hints['sdesc'].replace('"', '')
 
-                print('<tr><td><a href="' + args.arch + '/' + p + '">' + p + '</a></td><td>' + header + '</td></tr>', file=index)
+                print('<tr><td><a href="' + arch + '/' + p + '">' + p + '</a></td><td>' + header + '</td></tr>', file=index)
 
             print(textwrap.dedent('''\
                                      </table>
@@ -233,4 +236,4 @@ if __name__ == "__main__":
     logging.basicConfig(format=os.path.basename(sys.argv[0])+': %(message)s')
 
     packages = package.read_packages(args.rel_area, args.arch)
-    update_package_listings(args, packages)
+    update_package_listings(args, packages, args.arch)
diff --git a/queue.py b/queue.py
index ed4f89a..1d352e2 100644
--- a/queue.py
+++ b/queue.py
@@ -53,7 +53,7 @@ def add(args, movelist, fromdir):
     for p in movelist:
         for f in movelist[p]:
             if re.search(r'-src.tar.(bz2|gz|lzma|xz)$', f):
-                srcpkgs.append(os.path.join(args.arch, p, f))
+                srcpkgs.append(os.path.join(p, f))
 
     # if so...
     #
@@ -61,7 +61,7 @@ def add(args, movelist, fromdir):
     # source file
     if len(srcpkgs) >= 1:
         # keep all the files for comparison
-        uploads.copy(args, movelist, fromdir, os.path.join(upload_root, args.arch))
+        uploads.copy(args, movelist, fromdir, upload_root)
 
         # queue any srcpkgs
         for p in srcpkgs:
diff --git a/testdata/hints/noarch/release/perl-Net-SMTP-SSL/expected b/testdata/hints/noarch/release/perl-Net-SMTP-SSL/expected
new file mode 100644
index 0000000..9516e05
--- /dev/null
+++ b/testdata/hints/noarch/release/perl-Net-SMTP-SSL/expected
@@ -0,0 +1,5 @@
+{'category': 'Perl',
+ 'requires': '',
+ 'sdesc': '"Perl distribution Net-SMTP-SSL"',
+ 'ldesc': '"Implements the same API as Net::SMTP, but uses IO::Socket::SSL for\n'
+          'its network operations in order to support encrypted connections."'}
diff --git a/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2-src.tar.xz b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2-src.tar.xz
new file mode 100644
index 0000000..9e7f93d
Binary files /dev/null and b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2-src.tar.xz differ
diff --git a/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2.tar.xz b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2.tar.xz
new file mode 100644
index 0000000..d986060
Binary files /dev/null and b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2.tar.xz differ
diff --git a/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/setup.hint b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/setup.hint
new file mode 100644
index 0000000..f91a0e1
--- /dev/null
+++ b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/setup.hint	
@@ -0,0 +1,5 @@
+category: Perl
+requires:
+sdesc: "Perl distribution Net-SMTP-SSL"
+ldesc: "Implements the same API as Net::SMTP, but uses IO::Socket::SSL for
+its network operations in order to support encrypted connections."
diff --git a/testdata/htdocs.expected/x86/packages.inc b/testdata/htdocs.expected/x86/packages.inc
index 8f95160..289a07e 100755
--- a/testdata/htdocs.expected/x86/packages.inc
+++ b/testdata/htdocs.expected/x86/packages.inc
@@ -16,6 +16,7 @@
 <tr><td><a href="x86/libdns_sd1">libdns_sd1</a></td><td>Bonjour Zeroconf implementation</td></tr>
 <tr><td><a href="x86/mDNSResponder">mDNSResponder</a></td><td>Bonjour Zeroconf implementation</td></tr>
 <tr><td><a href="x86/openssh">openssh</a></td><td>The OpenSSH server and client programs</td></tr>
+<tr><td><a href="x86/perl-Net-SMTP-SSL">perl-Net-SMTP-SSL</a></td><td>Perl distribution Net-SMTP-SSL</td></tr>
 <tr><td><a href="x86/rpm-doc">rpm-doc</a></td><td>Obsolete package for RPM package management system manual pages</td></tr>
 <tr><td><a href="x86/testpackage">testpackage</a></td><td>A test package</td></tr>
 </table>
diff --git a/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/.htaccess b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/.htaccess
new file mode 100644
index 0000000..3196d64
--- /dev/null
+++ b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/.htaccess
@@ -0,0 +1,3 @@
+Options Indexes
+IndexOptions -FancyIndexing
+AddType text/html 1 2 3 4 5 6 7 8 9
diff --git a/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1 b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1
new file mode 100644
index 0000000..0f6212c
--- /dev/null
+++ b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1
@@ -0,0 +1,21 @@
+<html>
+<h1>perl-Net-SMTP-SSL: Perl distribution Net-SMTP-SSL (installed binaries and support files)</h1>
+<tt><pre>
+    2015-06-26 19:10           0 usr/
+    2015-06-26 19:10           0 usr/lib/
+    2015-06-26 19:10           0 usr/lib/perl5/
+    2015-06-26 19:10           0 usr/lib/perl5/vendor_perl/
+    2015-06-26 19:10           0 usr/lib/perl5/vendor_perl/5.22/
+    2015-06-26 19:10           0 usr/lib/perl5/vendor_perl/5.22/Net/
+    2015-06-26 19:10           0 usr/lib/perl5/vendor_perl/5.22/Net/SMTP/
+    2015-06-21 01:58        1462 usr/lib/perl5/vendor_perl/5.22/Net/SMTP/SSL.pm
+    2015-06-26 19:10           0 usr/share/
+    2015-06-26 19:10           0 usr/share/doc/
+    2015-06-26 19:10           0 usr/share/doc/perl-Net-SMTP-SSL/
+    2015-06-26 19:10         199 usr/share/doc/perl-Net-SMTP-SSL/Changes
+    2015-06-26 19:10         867 usr/share/doc/perl-Net-SMTP-SSL/README
+    2015-06-26 19:10           0 usr/share/man/
+    2015-06-26 19:10           0 usr/share/man/man3/
+    2015-06-26 19:10        1619 usr/share/man/man3/Net.SMTP.SSL.3pm.gz
+</pre></tt>
+</html>
diff --git a/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src
new file mode 100644
index 0000000..76e9a50
--- /dev/null
+++ b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src
@@ -0,0 +1,8 @@
+<html>
+<h1>perl-Net-SMTP-SSL: Perl distribution Net-SMTP-SSL (source code)</h1>
+<tt><pre>
+    2015-06-26 19:10           0 perl-Net-SMTP-SSL-1.03-1.src/
+    2015-06-26 19:10        2271 perl-Net-SMTP-SSL-1.03-1.src/Net-SMTP-SSL-1.03.tar.gz
+    2015-06-26 19:10         296 perl-Net-SMTP-SSL-1.03-1.src/perl-Net-SMTP-SSL.cygport
+</pre></tt>
+</html>
diff --git a/testdata/inifile/setup.ini.expected b/testdata/inifile/setup.ini.expected
index 6c09d84..c41865a 100644
--- a/testdata/inifile/setup.ini.expected
+++ b/testdata/inifile/setup.ini.expected
@@ -182,6 +182,17 @@
  'source: x86/release/openssh/openssh-7.2p2-1-src.tar.xz 228 '
  'e675b0ac4bc2c3e1c4971bc56d77b0cd53a9bdf5632873a235d7582e29dfd3e8a7bb04b28f6cdee3e6b3d14c25ed39392538e3f628a9bfda6c905646ebc3c225\n'
  '\n'
+ '@ perl-Net-SMTP-SSL\n'
+ 'sdesc: "Perl distribution Net-SMTP-SSL"\n'
+ 'ldesc: "Implements the same API as Net::SMTP, but uses IO::Socket::SSL for\n'
+ 'its network operations in order to support encrypted connections."\n'
+ 'category: Perl\n'
+ 'version: 1.03-1\n'
+ 'install: noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz 3180 '
+ 'aac6428f56fed431da1430242a327f36f1ec1ca6106366acb6752dd87f1b9adb87767709be9279b2f9435d16cee003f119a5ed4519c7365c8411404555618e66\n'
+ 'source: noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz 2752 '
+ '4de528554acb0b63f2a964bc8a0421118f96ce05cb4c9d111983aaea1a4a93025b31627fd66875fa09c2dfaecc11a89f2a294bdd3c77055bfaa93a27fdfc1f07\n'
+ '\n'
  '@ rpm-doc\n'
  'sdesc: "Obsolete package for RPM package management system manual pages"\n'
  'category: _obsolete\n'
diff --git a/testdata/pkglist/cygwin-pkg-maint b/testdata/pkglist/cygwin-pkg-maint
index 41b4175..c157e17 100644
--- a/testdata/pkglist/cygwin-pkg-maint
+++ b/testdata/pkglist/cygwin-pkg-maint
@@ -1606,7 +1606,7 @@ perl-Net-DNS                                 Achim Gratz
 perl-Net-DNS-SEC                             Achim Gratz
 perl-Net-HTTP                                Achim Gratz/Yaakov Selkowitz
 perl-Net-IP                                  Achim Gratz
-perl-Net-SMTP-SSL                            Yaakov Selkowitz
+perl-Net-SMTP-SSL                            Yaakov Selkowitz/Blooey McFooey
 perl-Net-SSLeay                              Achim Gratz/Yaakov Selkowitz
 perl-Number-Compare                          Achim Gratz/Ken Brown
 perl-OpenGL                                  Yaakov Selkowitz
diff --git a/testdata/pkglist/expected b/testdata/pkglist/expected
index b340aa2..c9cd92c 100644
--- a/testdata/pkglist/expected
+++ b/testdata/pkglist/expected
@@ -3,7 +3,7 @@
  'Adam Dinwoodie': maintainers.Maintainer('Adam Dinwoodie', [], ['git']),
  'Alexey Sokolov': maintainers.Maintainer('Alexey Sokolov', [], ['znc']),
  'Andrew Schulman': maintainers.Maintainer('Andrew Schulman', [], ['atool', 'autossh', 'bc', 'discus', 'fish', 'lftp', 'libargp', 'nosleep', 'orpie', 'pinfo', 'ploticus', 'ploticus-doc', 'screen', 'sitecopy', 'sng', 'socat', 'stow', 'stunnel', 'time', 'unison2.27', 'unison2.32', 'unison2.40', 'unison2.45', 'unison2.48']),
- 'Blooey McFooey': maintainers.Maintainer('Blooey McFooey', [], ['testpackage']),
+ 'Blooey McFooey': maintainers.Maintainer('Blooey McFooey', [], ['perl-Net-SMTP-SSL', 'testpackage']),
  'Bob Heckel': maintainers.Maintainer('Bob Heckel', [], ['libgc', 'w3m']),
  'Chris J. Breisch': maintainers.Maintainer('Chris J. Breisch', [], ['man-db']),
  'Chris LeBlanc': maintainers.Maintainer('Chris LeBlanc', [], ['python-h5py', 'python3-h5py']),
diff --git a/testdata/process_arch/homedir.expected b/testdata/process_arch/homedir.expected
index 8dcf175..1c52158 100644
--- a/testdata/process_arch/homedir.expected
+++ b/testdata/process_arch/homedir.expected
@@ -1,5 +1,8 @@
 {'.': ['an_unexpected_file'],
  'Blooey McFooey': [],
+ 'Blooey McFooey/noarch': [],
+ 'Blooey McFooey/noarch/release': [],
+ 'Blooey McFooey/noarch/release/perl-Net-SMTP-SSL': [],
  'Blooey McFooey/x86': [],
  'Blooey McFooey/x86/release': [],
  'Blooey McFooey/x86/release/after-ready': ['after-ready-1.0-1.tar.bz2', 'setup.hint'],
diff --git a/testdata/process_arch/htdocs.expected b/testdata/process_arch/htdocs.expected
index 6554900..4670972 100644
--- a/testdata/process_arch/htdocs.expected
+++ b/testdata/process_arch/htdocs.expected
@@ -19,6 +19,11 @@
  'x86/libdns_sd1': ['.htaccess', 'libdns_sd1-379.32.1-1'],
  'x86/mDNSResponder': ['.htaccess', 'mDNSResponder-379.32.1-1', 'mDNSResponder-379.32.1-1-src'],
  'x86/openssh': ['.htaccess', 'openssh-7.2p2-1', 'openssh-7.2p2-1-src'],
+ 'x86/perl-Net-SMTP-SSL': ['.htaccess',
+                           'perl-Net-SMTP-SSL-1.03-1',
+                           'perl-Net-SMTP-SSL-1.03-1-src',
+                           'perl-Net-SMTP-SSL-1.03-2',
+                           'perl-Net-SMTP-SSL-1.03-2-src'],
  'x86/rpm-doc': ['.htaccess', 'rpm-doc-4.1-2', 'rpm-doc-4.1-2-src', 'rpm-doc-999-1'],
  'x86/testpackage': ['.htaccess', 'testpackage-1.0-1', 'testpackage-1.0-1-src'],
  'x86/testpackage-subpackage': ['.htaccess', 'testpackage-subpackage-1.0-1']}
diff --git a/testdata/process_arch/rel_area.expected b/testdata/process_arch/rel_area.expected
index f081c02..44161bf 100644
--- a/testdata/process_arch/rel_area.expected
+++ b/testdata/process_arch/rel_area.expected
@@ -1,4 +1,12 @@
 {'.': ['setup.ini'],
+ 'noarch': ['sha512.sum'],
+ 'noarch/release': ['sha512.sum'],
+ 'noarch/release/perl-Net-SMTP-SSL': ['perl-Net-SMTP-SSL-1.03-1-src.tar.xz',
+                                      'perl-Net-SMTP-SSL-1.03-1.tar.xz',
+                                      'perl-Net-SMTP-SSL-1.03-2-src.tar.xz',
+                                      'perl-Net-SMTP-SSL-1.03-2.tar.xz',
+                                      'setup.hint',
+                                      'sha512.sum'],
  'x86': ['sha512.sum'],
  'x86/release': ['sha512.sum'],
  'x86/release/arc': ['arc-4.32.7-10-src.tar.bz2', 'arc-4.32.7-10.tar.bz2', 'setup.hint', 'sha512.sum'],
diff --git a/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz
new file mode 100644
index 0000000..9e7f93d
Binary files /dev/null and b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz differ
diff --git a/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz
new file mode 100644
index 0000000..d986060
Binary files /dev/null and b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz differ
diff --git a/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/setup.hint b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/setup.hint
new file mode 100644
index 0000000..f91a0e1
--- /dev/null
+++ b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/setup.hint
@@ -0,0 +1,5 @@
+category: Perl
+requires:
+sdesc: "Perl distribution Net-SMTP-SSL"
+ldesc: "Implements the same API as Net::SMTP, but uses IO::Socket::SSL for
+its network operations in order to support encrypted connections."
diff --git a/testdata/uploads/move.expected b/testdata/uploads/move.expected
index b0caaf3..47cb0af 100644
--- a/testdata/uploads/move.expected
+++ b/testdata/uploads/move.expected
@@ -1,3 +1,3 @@
-{'release/testpackage': ['setup.hint', 'testpackage-1.0-1-src.tar.bz2', 'testpackage-1.0-1.tar.bz2'],
- 'release/testpackage/testpackage-subpackage': ['setup.hint', 'testpackage-subpackage-1.0-1.tar.bz2'],
- 'release/testpackage2/testpackage2-subpackage': ['setup.hint', 'testpackage2-subpackage-1.0-1.tar.bz2']}
+{'x86/release/testpackage': ['setup.hint', 'testpackage-1.0-1-src.tar.bz2', 'testpackage-1.0-1.tar.bz2'],
+ 'x86/release/testpackage/testpackage-subpackage': ['setup.hint', 'testpackage-subpackage-1.0-1.tar.bz2'],
+ 'x86/release/testpackage2/testpackage2-subpackage': ['setup.hint', 'testpackage2-subpackage-1.0-1.tar.bz2']}
diff --git a/testdata/uploads/pkglist.expected b/testdata/uploads/pkglist.expected
index f98adb3..bbe52b0 100644
--- a/testdata/uploads/pkglist.expected
+++ b/testdata/uploads/pkglist.expected
@@ -1,14 +1,14 @@
-{'testpackage': Package('release/testpackage', {'testpackage-1.0-1-src.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False),
+{'testpackage': Package('x86/release/testpackage', {'testpackage-1.0-1-src.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False),
  'testpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test package"',
  'ldesc': '"A test package\n'
           "It's description might contains some unicode gibberish\n"
           'Like itâ??s youâ??re Markup Languageâ?¢ Nokogiriâ??s toolâ??that Bézier."',
  'category': 'Devel',
  'requires': 'cygwin'}),
- 'testpackage-subpackage': Package('release/testpackage/testpackage-subpackage', {'testpackage-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage"',
+ 'testpackage-subpackage': Package('x86/release/testpackage/testpackage-subpackage', {'testpackage-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage"',
  'ldesc': '"A test subpackage"',
  'category': 'Devel',
  'external-source': 'testpackage'}),
- 'testpackage2-subpackage': Package('release/testpackage2/testpackage2-subpackage', {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage 2"',
+ 'testpackage2-subpackage': Package('x86/release/testpackage2/testpackage2-subpackage', {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage 2"',
  'ldesc': '"A test subpackage 2"',
  'category': 'Devel'})}
diff --git a/tests.py b/tests.py
index 8f3350d..0881ab8 100755
--- a/tests.py
+++ b/tests.py
@@ -123,7 +123,7 @@ class TestMain(unittest.TestCase):
 
         packages = package.read_packages(args.rel_area, args.arch)
         package.validate_packages(args, packages)
-        pkg2html.update_package_listings(args, packages)
+        pkg2html.update_package_listings(args, packages, args.arch)
 
         # compare the output files with expected
         for (dirpath, subdirs, files) in os.walk(htdocs):
@@ -200,10 +200,11 @@ class TestMain(unittest.TestCase):
         for (f, t) in ready_fns:
             os.system('touch %s "%s"' % (t, f))
 
-        scan_result = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args)
+        scan_result = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args.arch, args)
 
         self.assertEqual(scan_result.error, False)
         compare_with_expected_file(self, 'testdata/uploads', scan_result.to_relarea, 'move')
+        self.assertCountEqual(scan_result.to_vault, {'x86/release/testpackage': ['x86/release/testpackage/testpackage-0.1-1.tar.bz2']})
         self.assertCountEqual(scan_result.remove_always, [f for (f, t) in ready_fns])
         self.assertEqual(scan_result.remove_success, ['testdata/homes/Blooey McFooey/x86/release/testpackage/-testpackage-0.1-1.tar.bz2'])
         compare_with_expected_file(self, 'testdata/uploads', scan_result.packages, 'pkglist')
@@ -222,9 +223,9 @@ class TestMain(unittest.TestCase):
         setattr(args, 'setup_version', '4.321')
 
         packages = package.read_packages(args.rel_area, args.arch)
-        package.delete(packages, 'release/nonexistent', 'nosuchfile-1.0.0.tar.xz')
+        package.delete(packages, 'x86/release/nonexistent', 'nosuchfile-1.0.0.tar.xz')
         self.assertEqual(package.validate_packages(args, packages), True)
-        package.write_setup_ini(args, packages)
+        package.write_setup_ini(args, packages, args.arch)
         with open(args.inifile) as inifile:
             results = inifile.read()
             # fix the timestamp to match expected
@@ -233,7 +234,7 @@ class TestMain(unittest.TestCase):
 
         # XXX: delete a needed package, and check validate fails
 
-    def test_process_arch(self):
+    def test_process(self):
         self.maxDiff = None
 
         args = types.SimpleNamespace()
@@ -242,7 +243,6 @@ class TestMain(unittest.TestCase):
             setattr(args, d, tempfile.mktemp())
             logging.info('%s = %s', d, getattr(args, d))
 
-        setattr(args, 'arch', 'x86')
         setattr(args, 'dryrun', False)
         setattr(args, 'email', None)
         setattr(args, 'force', False)
@@ -258,11 +258,16 @@ class TestMain(unittest.TestCase):
         m_homedir = os.path.join(getattr(args, 'homedir'), 'Blooey McFooey')
         ready_fns = [(os.path.join(m_homedir, 'x86', 'release', 'testpackage', '!ready'), ''),
                      (os.path.join(m_homedir, 'x86', 'release', 'testpackage2', 'testpackage2-subpackage', '!ready'), ''),
-                     (os.path.join(m_homedir, 'x86', 'release', 'after-ready', '!ready'), '-t 198709011700')]
+                     (os.path.join(m_homedir, 'x86', 'release', 'after-ready', '!ready'), '-t 198709011700'),
+                     (os.path.join(m_homedir, 'noarch', 'release', 'perl-Net-SMTP-SSL', '!ready'), '')]
         for (f, t) in ready_fns:
             os.system('touch %s "%s"' % (t, f))
 
-        self.assertEqual(calm.process_arch(args), True)
+        packages = calm.process(args)
+        self.assertTrue(packages)
+
+        pkg2html.update_package_listings(args, packages['x86'], 'x86')
+        package.write_setup_ini(args, packages['x86'], 'x86')
 
         for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
             with self.subTest(directory=d):
@@ -272,7 +277,7 @@ class TestMain(unittest.TestCase):
 
 if __name__ == '__main__':
     # ensure sha512.sum files exist
-    os.system("find testdata/relarea/x86 -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;")
+    os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;")
     # should remove a sha512.sum file so that we test functioning when it's absent
     os.unlink('testdata/relarea/x86/release/naim/sha512.sum')
     # remove !ready files
diff --git a/upload-scan b/upload-scan
deleted file mode 100755
index 08ab677..0000000
--- a/upload-scan
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (c) 2015 Jon Turney
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-#
-# check maintainer upload directories for files to move to release area
-#
-# - only process packages for which we are listed as a maintainer
-# - only move things with an mtime earlier than the mtime of the !ready file
-# - validate the syntax of the setup.ini file before moving anything
-# - note if package file already exists in release area with identical contents
-# - error if package file already exist in release area with different contents
-# - remove !ready file
-# - send a report on what we did to the maintainer and project leads
-#
-
-import argparse
-import logging
-import os
-import sys
-
-from buffering_smtp_handler import mail_logs
-import common_constants
-import maintainers
-import uploads
-
-
-#
-#
-#
-
-def main(args):
-    # create maintainer list
-    mlist = maintainers.Maintainer.read(args)
-
-    # make the list of all packages
-    all_packages = maintainers.Maintainer.all_packages(mlist)
-
-    for arch in common_constants.ARCHES:
-        args.arch = arch
-
-        for name in sorted(mlist.keys()):
-            m = mlist[name]
-
-            with mail_logs(args.email, toaddrs=(args.email or []) + m.email, subject='upset messages') as maint_email:
-
-                # search for and validate uploaded packages
-                (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args)
-
-                # always remove all the !ready files
-                uploads.remove(args, remove_always)
-
-                # but only move something if there were no errors
-                if not error:
-                    uploads.move_to_vault(args, to_vault)
-                    uploads.remove(args, remove_success)
-                    uploads.move_to_relarea(m, args, to_relarea)
-
-
-#
-#
-#
-
-if __name__ == "__main__":
-    homedir_default = common_constants.HOMEDIR
-    orphanmaint_default = common_constants.ORPHANMAINT
-    pkglist_default = common_constants.PKGMAINT
-    relarea_default = common_constants.FTP
-    vault_default = common_constants.VAULT
-
-    parser = argparse.ArgumentParser(description='Move files from maintainer upload directories to release area')
-    parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help='email output to maintainer and ADDRS (default: ' + common_constants.EMAILS + ')', metavar='ADDRS')
-    parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default)
-    parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default)
-    parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default)
-    parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
-    parser.add_argument('--vault', action='store', metavar='DIR', help="vault directory (default: " + vault_default + ")", default=vault_default, dest='vault')
-    parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun', help="don't do anything")
-    parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output')
-    (args) = parser.parse_args()
-
-    if args.verbose:
-        logging.getLogger().setLevel(logging.INFO)
-
-    logging.basicConfig(format=os.path.basename(sys.argv[0])+': %(message)s')
-
-    if args.email:
-        args.email = args.email.split(',')
-
-    main(args)
diff --git a/uploads.py b/uploads.py
index b921ff1..137b159 100644
--- a/uploads.py
+++ b/uploads.py
@@ -46,9 +46,9 @@ ScanResult = namedtuple('ScanResult', 'error,packages,to_relarea,to_vault,remove
 #
 #
 
-def scan(m, all_packages, args):
-    basedir = os.path.join(m.homedir(), args.arch)
-    releasedir = os.path.join(args.rel_area, args.arch)
+def scan(m, all_packages, arch, args):
+    basedir = os.path.join(m.homedir(), arch)
+    releasedir = os.path.join(args.rel_area, arch)
 
     packages = defaultdict(package.Package)
     move = defaultdict(list)
@@ -70,7 +70,7 @@ def scan(m, all_packages, args):
 
     # the mtime of this file indicates when 'ignoring as there is no !ready'
     # warnings were last emitted
-    reminder_file = os.path.join(basedir, '!reminder-timestamp')
+    reminder_file = os.path.join(m.homedir(), '!reminder-timestamp')
     if os.path.exists(reminder_file):
         reminder_time = os.path.getmtime(reminder_file)
     else:
@@ -80,7 +80,7 @@ def scan(m, all_packages, args):
 
     # scan package directories
     for (dirpath, subdirs, files) in os.walk(os.path.join(basedir, 'release')):
-        relpath = os.path.relpath(dirpath, basedir)
+        relpath = os.path.relpath(dirpath, m.homedir())
 
         # skip uninteresting directories
         if (not files) or (relpath == 'release'):
@@ -190,7 +190,7 @@ def scan(m, all_packages, args):
         # read and validate package
         if files:
             # strict means we consider warnings as fatal for upload
-            if package.read_package(packages, basedir, dirpath, files, strict=True):
+            if package.read_package(packages, m.homedir(), dirpath, files, strict=True):
                 error = True
 
     # if we didn't need to check the reminder timestamp, it can be reset
@@ -246,14 +246,14 @@ def move(args, movelist, fromdir, todir):
 
 
 def move_to_relarea(m, args, movelist):
-    move(args, movelist, os.path.join(m.homedir(), args.arch), os.path.join(args.rel_area, args.arch))
+    move(args, movelist, m.homedir(), args.rel_area)
     # XXX: Note that there seems to be a separate process, not run from
     # cygwin-admin's crontab, which changes the ownership of files in the
     # release area to cyguser:cygwin
 
 
 def move_to_vault(args, movelist):
-    move(args, movelist, os.path.join(args.rel_area, args.arch), os.path.join(args.vault, args.arch))
+    move(args, movelist, args.rel_area, args.vault)
 
 
 #


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]