[arvados-dev] created: 1b436c2a6d2dfd78a28ff184d35238eb3e515722

[git repository hosting]

        at  1b436c2a6d2dfd78a28ff184d35238eb3e515722 (commit)


commit 1b436c2a6d2dfd78a28ff184d35238eb3e515722
Author: Brett Smith <brett.smith at curii.com>
Date:   Tue May 28 11:14:01 2024 -0400

    21840: Upload Python packages with twine
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/jenkins/run_upload_packages.py b/jenkins/run_upload_packages.py
index 1b91d04..3d336a8 100755
--- a/jenkins/run_upload_packages.py
+++ b/jenkins/run_upload_packages.py
@@ -19,6 +19,8 @@ import sys
 import tempfile
 import time
 
+from pathlib import Path
+
 def run_and_grep(cmd, read_output, *regexps,
                  encoding=locale.getpreferredencoding(), **popen_kwargs):
     """Run a subprocess and capture output lines matching regexps.
@@ -118,14 +120,6 @@ class PackageSuite:
 
 class PythonPackageSuite(PackageSuite):
     LOGGER_PART = 'python'
-    REUPLOAD_REGEXPS = [
-        re.compile(
-            r'^error: Upload failed \(400\): A file named "[^"]+" already exists\b'),
-        re.compile(
-            r'^error: Upload failed \(400\): File already exists\b'),
-        re.compile(
-            r'^error: Upload failed \(400\): Only one sdist may be uploaded per release\b'),
-    ]
 
     def __init__(self, glob_root, rel_globs):
         super().__init__(glob_root, rel_globs)
@@ -136,11 +130,6 @@ class PythonPackageSuite(PackageSuite):
         if src_dir in self.seen_packages:
             return
         self.seen_packages.add(src_dir)
-        # We also must run `sdist` before `upload`: `upload` uploads any
-        # distributions previously generated in the command.  It doesn't
-        # know how to upload distributions already on disk.  We write the
-        # result to a dedicated directory to avoid interfering with our
-        # timestamp tracking.
         with tempfile.TemporaryDirectory(prefix='pydist-') as dist_dir:
             cmd = ['python3', 'setup.py']
             if not self.logger.isEnabledFor(logging.INFO):
@@ -148,12 +137,15 @@ class PythonPackageSuite(PackageSuite):
             cmd.extend([
                 'bdist_wheel', '--dist-dir', dist_dir,
                 'sdist', '--dist-dir', dist_dir,
-                'upload',
             ])
-            upload_returncode, repushed = run_and_grep(
-                cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
-            if (upload_returncode != 0) and not repushed:
-                raise subprocess.CalledProcessError(upload_returncode, cmd)
+            subprocess.run(cmd, stdin=subprocess.DEVNULL, check=True)
+            subprocess.run([
+                'twine', 'upload',
+                '--disable-progress-bar',
+                '--non-interactive',
+                '--skip-existing',
+                *Path(dist_dir).iterdir(),
+            ], stdin=subprocess.DEVNULL, check=True)
 
 
 class GemPackageSuite(PackageSuite):

commit feba81647156733d43eaf35f7d88875a00e1df8a
Author: Brett Smith <brett.smith at curii.com>
Date:   Tue May 28 10:52:11 2024 -0400

    21840: Use an explicit temporary directory context manager
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/jenkins/run_upload_packages.py b/jenkins/run_upload_packages.py
index 216b56b..1b91d04 100755
--- a/jenkins/run_upload_packages.py
+++ b/jenkins/run_upload_packages.py
@@ -16,6 +16,7 @@ import re
 import shutil
 import subprocess
 import sys
+import tempfile
 import time
 
 def run_and_grep(cmd, read_output, *regexps,
@@ -140,17 +141,19 @@ class PythonPackageSuite(PackageSuite):
         # know how to upload distributions already on disk.  We write the
         # result to a dedicated directory to avoid interfering with our
         # timestamp tracking.
-        cmd = ['python3', 'setup.py']
-        if not self.logger.isEnabledFor(logging.INFO):
-            cmd.append('--quiet')
-        cmd.extend(['bdist_wheel', '--dist-dir', '.upload_dist'])
-        cmd.extend(['sdist', '--dist-dir', '.upload_dist'])
-        cmd.extend(['upload'])
-        upload_returncode, repushed = run_and_grep(
-            cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
-        if (upload_returncode != 0) and not repushed:
-            raise subprocess.CalledProcessError(upload_returncode, cmd)
-        shutil.rmtree(os.path.join(src_dir, '.upload_dist'))
+        with tempfile.TemporaryDirectory(prefix='pydist-') as dist_dir:
+            cmd = ['python3', 'setup.py']
+            if not self.logger.isEnabledFor(logging.INFO):
+                cmd.append('--quiet')
+            cmd.extend([
+                'bdist_wheel', '--dist-dir', dist_dir,
+                'sdist', '--dist-dir', dist_dir,
+                'upload',
+            ])
+            upload_returncode, repushed = run_and_grep(
+                cmd, 'stderr', *self.REUPLOAD_REGEXPS, cwd=src_dir)
+            if (upload_returncode != 0) and not repushed:
+                raise subprocess.CalledProcessError(upload_returncode, cmd)
 
 
 class GemPackageSuite(PackageSuite):

-----------------------------------------------------------------------


hooks/post-receive
--