[ARVADOS] created: 4264bbad60a87547cfca97e48848ff120e660372
Git user
git at public.curoverse.com
Fri Oct 21 19:55:35 EDT 2016
at 4264bbad60a87547cfca97e48848ff120e660372 (commit)
commit 4264bbad60a87547cfca97e48848ff120e660372
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Oct 21 16:35:09 2016 -0400
10194: Improve versioning of arvados/jobs image.
arvados-cwl-runner sets version to max(cwl-runner, python-sdk).
build/run-build-docker-jobs-image.sh determines package versions corresponding
to current commit and supplies explicit package versions as Docker build arg.
diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index 22f6f54..57db0db 100755
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -5,8 +5,8 @@ function usage {
echo >&2 "usage: $0 [options]"
echo >&2
echo >&2 "$0 options:"
- echo >&2 " -t, --tags [csv_tags] comma separated tags"
echo >&2 " -u, --upload Upload the images (docker push)"
+ echo >&2 " --no-cache Don't use build cache"
echo >&2 " -h, --help Display this help and exit"
echo >&2
echo >&2 " If no options are given, just builds the images."
@@ -16,7 +16,7 @@ upload=false
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
TEMP=`getopt -o hut: \
- --long help,upload,tags: \
+ --long help,upload,no-cache,tags: \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -30,6 +30,10 @@ do
upload=true
shift
;;
+ --no-cache)
+ NOCACHE=--no-cache
+ shift
+ ;;
-t | --tags)
case "$2" in
"")
@@ -38,7 +42,7 @@ do
exit 1
;;
*)
- tags=$2;
+ echo "WARNING: --tags is deprecated and doesn't do anything";
shift 2
;;
esac
@@ -66,14 +70,6 @@ COLUMNS=80
. $WORKSPACE/build/run-library.sh
docker_push () {
- if [[ ! -z "$tags" ]]
- then
- for tag in $( echo $tags|tr "," " " )
- do
- $DOCKER tag $1 $1:$tag
- done
- fi
-
# Sometimes docker push fails; retry it a few times if necessary.
for i in `seq 1 5`; do
$DOCKER push $*
@@ -118,13 +114,30 @@ timer_reset
# clean up the docker build environment
cd "$WORKSPACE"
-cd docker/jobs
-if [[ ! -z "$tags" ]]; then
- docker build --build-arg COMMIT=${tags/,*/} -t arvados/jobs .
+
+python_sdk_ts=$(cd sdk/python && timestamp_from_git)
+cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+
+python_sdk_version=$(cd sdk/python && nohash_version_from_git 1.0)-3
+cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)-3
+
+if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+ cwl_runner_version=$python_sdk_version
+ gittag=$(cd sdk/python && git log --first-parent --max-count=1 --format=format:%H)
else
- docker build -t arvados/jobs .
+ gittag=$(cd sdk/cwl && git log --first-parent --max-count=1 --format=format:%H)
fi
+echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+
+cd docker/jobs
+docker build $NOCACHE \
+ --build-arg python_sdk_version=$python_sdk_version \
+ --build-arg cwl_runner_version=$cwl_runner_version \
+ -t arvados/jobs:$gittag .
+
+docker tag arvados/jobs:$gittag arvados/jobs:latest
+
ECODE=$?
if [[ "$ECODE" != "0" ]]; then
@@ -146,7 +159,8 @@ else
## even though credentials are already in .dockercfg
docker login -u arvados
- docker_push arvados/jobs
+ docker_push arvados/jobs:gittag
+ docker_push arvados/jobs:latest
title "upload arvados images finished (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set (`timer`)"
diff --git a/build/run-library.sh b/build/run-library.sh
index 73a99da..f0b120f 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -35,13 +35,19 @@ format_last_commit_here() {
version_from_git() {
# Generates a version number from the git log for the current working
# directory, and writes it to stdout.
- local git_ts git_hash
+ local git_ts git_hash prefix
+ if [[ -n "$1" ]] ; then
+ prefix="$1"
+ else
+ prefix="0.1"
+ fi
+
declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+ echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
}
nohash_version_from_git() {
- version_from_git | cut -d. -f1-3
+ version_from_git $1 | cut -d. -f1-3
}
timestamp_from_git() {
diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index e1e7e87..9b1be1e 100644
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -8,10 +8,16 @@ ADD apt.arvados.org.list /etc/apt/sources.list.d/
RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3
-ARG COMMIT=latest
-RUN echo $COMMIT && apt-get update -q
+ARG python_sdk_version
+ARG cwl_runner_version
+RUN echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
-RUN apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs python-arvados-cwl-runner
+RUN apt-get update -q
+RUN apt-get install -yq --no-install-recommends \
+ git python-pip python-virtualenv \
+ python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs \
+ python-arvados-python-client=$python_sdk_version \
+ python-arvados-cwl-runner=$cwl_runner_version
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
deleted file mode 120000
index d59c02c..0000000
--- a/sdk/cwl/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../python/gittaggers.py
\ No newline at end of file
diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
new file mode 100644
index 0000000..2a292ee
--- /dev/null
+++ b/sdk/cwl/gittaggers.py
@@ -0,0 +1,37 @@
+from setuptools.command.egg_info import egg_info
+import subprocess
+import time
+import os
+
+SETUP_DIR = os.path.dirname(__file__) or '.'
+
+def choose_version_from():
+ sdk_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
+ cwl_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', SETUP_DIR]).strip()
+ if int(sdk_ts) > int(cwl_ts):
+ getver = os.path.join(SETUP_DIR, "../python")
+ else:
+ getver = SETUP_DIR
+ return getver
+
+class EggInfoFromGit(egg_info):
+ """Tag the build with git commit timestamp.
+
+ If a build tag has already been set (e.g., "egg_info -b", building
+ from source package), leave it alone.
+ """
+
+ def git_timestamp_tag(self):
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', choose_version_from()]).strip()
+ return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
+
+ def tags(self):
+ if self.tag_build is None:
+ self.tag_build = self.git_timestamp_tag()
+ return egg_info.tags(self)
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 4adcac4..d1c8f9b 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -20,7 +20,7 @@ versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py")
try:
gitinfo = subprocess.check_output(
['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%H', SETUP_DIR]).strip()
+ '--format=format:%H', gittaggers.choose_version_from()]).strip()
with open(versionfile, "w") as f:
f.write("__version__ = '%s'\n" % gitinfo)
except Exception as e:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list