[arvados-dev] updated: 3d3039cd0ad13789c16acee19a2f9e5fae89f4a5

[git repository hosting]

Summary of changes:
 jenkins/run_upload_packages.py | 51 +++++++++++++-----------------------------
 1 file changed, 15 insertions(+), 36 deletions(-)

       via  3d3039cd0ad13789c16acee19a2f9e5fae89f4a5 (commit)
       via  bc684e32733a5a027aeb8c4255ed559f4a8449d7 (commit)
       via  f535659da4796d1c502337886a0f0f390a71e816 (commit)
      from  ecdb11a532dd37a4fee0d93831ca67fffcad88c0 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 3d3039cd0ad13789c16acee19a2f9e5fae89f4a5
Merge: ecdb11a bc684e3
Author: Brett Smith <brett.smith at curii.com>
Date:   Tue May 28 12:05:40 2024 -0400

    Merge branch '21840-twine-upload'
    
    Refs #21840.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>


commit bc684e32733a5a027aeb8c4255ed559f4a8449d7
Author: Brett Smith <brett.smith at curii.com>
Date:   Tue May 28 12:00:47 2024 -0400

    21840: Use twine for Python uploads
    
    In addition to getting us away from a deprecated setup.py invocation,
    this also lets the Python upload work a lot more like the Ruby upload:
    you just upload distributions that Jenkins built previously.
    
    Note uploading wheels will require a mirror change in the Jenkins build
    script.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/jenkins/run_upload_packages.py b/jenkins/run_upload_packages.py
index 991d821..a299c2a 100755
--- a/jenkins/run_upload_packages.py
+++ b/jenkins/run_upload_packages.py
@@ -117,40 +117,15 @@ class PackageSuite:
 
 class PythonPackageSuite(PackageSuite):
     LOGGER_PART = 'python'
-    REUPLOAD_REGEXPS = [
-        re.compile(
-            r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b'),
-        re.compile(
-            r'^error: Upload failed \(400\): File already exists\b'),
-        re.compile(
-            r'^error: Upload failed \(400\): Only one sdist may be uploaded per release\b'),
-    ]
-
-    def __init__(self, glob_root, rel_globs):
-        super().__init__(glob_root, rel_globs)
-        self.seen_packages = set()
 
     def upload_file(self, path):
-        src_dir = os.path.dirname(os.path.dirname(path))
-        if src_dir in self.seen_packages:
-            return
-        self.seen_packages.add(src_dir)
-        # We also must run `sdist` before `upload`: `upload` uploads any
-        # distributions previously generated in the command.  It doesn't
-        # know how to upload distributions already on disk.  We write the
-        # result to a dedicated directory to avoid interfering with our
-        # timestamp tracking.
-        cmd = ['python3', 'setup.py']
-        if not self.logger.isEnabledFor(logging.INFO):
-            cmd.append('--quiet')
-        cmd.extend(['bdist_wheel', '--dist-dir', '.upload_dist'])
-        cmd.extend(['sdist', '--dist-dir', '.upload_dist'])
-        cmd.extend(['upload'])
-        upload_returncode, repushed = run_and_grep(
-            cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
-        if (upload_returncode != 0) and not repushed:
-            raise subprocess.CalledProcessError(upload_returncode, cmd)
-        shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
+        subprocess.run([
+            'twine', 'upload',
+            '--disable-progress-bar',
+            '--non-interactive',
+            '--skip-existing',
+            path,
+        ], stdin=subprocess.DEVNULL, check=True)
 
 
 class GemPackageSuite(PackageSuite):
@@ -271,10 +246,14 @@ def _define_suite(suite_class, *rel_globs, **kwargs):
 
 PACKAGE_SUITES = {
     'python': _define_suite(PythonPackageSuite,
-                            'sdk/python/dist/*.tar.gz',
                             'sdk/cwl/dist/*.tar.gz',
+                            'sdk/cwl/dist/*.whl',
+                            'sdk/python/dist/*.tar.gz',
+                            'sdk/python/dist/*.whl',
                             'services/fuse/dist/*.tar.gz',
+                            'services/fuse/dist/*.whl',
                             'tools/crunchstat-summary/dist/*.tar.gz',
+                            'tools/crunchstat-summary/dist/*.whl',
                         ),
     'gems': _define_suite(GemPackageSuite,
                           'sdk/ruby/*.gem',

commit f535659da4796d1c502337886a0f0f390a71e816
Author: Brett Smith <brett.smith at curii.com>
Date:   Tue May 28 11:40:30 2024 -0400

    21840: Migrate from pipes.quote to shlex.quote
    
    To avoid the pipes deprecation in modern Python.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/jenkins/run_upload_packages.py b/jenkins/run_upload_packages.py
index 216b56b..991d821 100755
--- a/jenkins/run_upload_packages.py
+++ b/jenkins/run_upload_packages.py
@@ -11,8 +11,8 @@ import glob
 import locale
 import logging
 import os
-import pipes
 import re
+import shlex
 import shutil
 import subprocess
 import sys
@@ -191,8 +191,8 @@ class DistroPackageSuite(PackageSuite):
         # self.__class__.__name__ provides $0 for the script, which makes a
         # nicer message if there's an error.
         subprocess.check_call(self._build_cmd(
-                'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script),
-                self.__class__.__name__, *(pipes.quote(s) for s in args)))
+                'ssh', self.ssh_host, 'bash', '-ec', shlex.quote(script),
+                self.__class__.__name__, *(shlex.quote(s) for s in args)))
 
     def upload_files(self, paths):
         dest_dir = os.path.join(self.REMOTE_DEST_DIR, self.target)

-----------------------------------------------------------------------


hooks/post-receive
--