[ARVADOS] updated: 8fc563f15301baa0a59bfae748e0c8cd2d5eefda
Git user
git at public.curoverse.com
Tue Jan 3 20:58:01 EST 2017
Summary of changes:
build/run-build-packages.sh | 8 +-
build/run-tests.sh | 235 +++++++++++++++++----------------
services/fuse/arvados_fuse/__init__.py | 56 +++++---
services/fuse/setup.py | 2 +-
4 files changed, 169 insertions(+), 132 deletions(-)
via 8fc563f15301baa0a59bfae748e0c8cd2d5eefda (commit)
via ce42f38437ccda92142eda2e49d77226f78bd82d (commit)
via 002042bd4a1eb6992313ef821053c4a38fd3b4a8 (commit)
via e9661d2d3e117fcb9b48848757b6dfe65210ba43 (commit)
from e7f29d3e4baa5c5d857a1a6f396b7c8047c45c82 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8fc563f15301baa0a59bfae748e0c8cd2d5eefda
Merge: e7f29d3 ce42f38
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jan 3 20:57:40 2017 -0500
Merge branch '8345-revert-llfuse-to-0.41.1'
refs #8345
commit ce42f38437ccda92142eda2e49d77226f78bd82d
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jan 3 15:05:20 2017 -0500
8345: Revert llfuse to 0.41.1.
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 29b7e8c..b50538f 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -106,7 +106,7 @@ case "$TARGET" in
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka six \
- ciso8601 pycrypto backports.ssl_match_hostname 'llfuse>=1.0' \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
cachecontrol 'pathlib2>=2.1.0')
@@ -117,7 +117,7 @@ case "$TARGET" in
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka six \
- ciso8601 pycrypto backports.ssl_match_hostname 'llfuse>=1.0' \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro isodate ruamel.ordereddict
cachecontrol requests 'pathlib2>=2.1.0')
@@ -125,7 +125,7 @@ case "$TARGET" in
;;
ubuntu1404)
FORMAT=deb
- PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 'llfuse>=1.0' ciso8601 \
+ PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
@@ -145,7 +145,7 @@ case "$TARGET" in
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka \
ciso8601 pycrypto 'pycurl<7.21.5' \
- python-daemon==2.1.1 'llfuse>=1.0' 'pbr<1.0' pyyaml contextlib2 \
+ python-daemon==2.1.1 llfuse==0.41.1 'pbr<1.0' pyyaml contextlib2 \
'rdflib>=4.2.0' shellescape mistune typing avro \
isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0')
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index 0e0ed4b..9e282ca 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -38,7 +38,7 @@ setup(name='arvados_fuse',
],
install_requires=[
'arvados-python-client >= 0.1.20151118035730',
- 'llfuse >= 1.0',
+ 'llfuse==0.41.1',
'python-daemon',
'ciso8601',
'setuptools'
commit 002042bd4a1eb6992313ef821053c4a38fd3b4a8
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jan 3 10:50:41 2017 -0500
8345: Compatibility with llfuse 0.41 and 1.x.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 289966f..1d25aa8 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -74,7 +74,12 @@ import Queue
# unlimited to avoid deadlocks, see https://arvados.org/issues/3198#note-43 for
# details.
-llfuse._notify_queue = Queue.Queue()
+if hasattr(llfuse, 'capi'):
+ # llfuse < 0.42
+ llfuse.capi._notify_queue = Queue.Queue()
+else:
+ # llfuse >= 0.42
+ llfuse._notify_queue = Queue.Queue()
from fusedir import sanitize_filename, Directory, CollectionDirectory, TmpCollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
from fusefile import StringFile, FuseArvadosFile
@@ -360,7 +365,13 @@ class Operations(llfuse.Operations):
self.events.close()
self.events = None
- self.inodes.clear()
+ if llfuse.lock.acquire():
+ # llfuse < 0.42
+ self.inodes.clear()
+ llfuse.lock.release()
+ else:
+ # llfuse >= 0.42
+ self.inodes.clear()
def access(self, inode, mode, ctx):
return True
@@ -431,14 +442,21 @@ class Operations(llfuse.Operations):
entry.st_blksize = 512
entry.st_blocks = (entry.st_size/512)+1
- entry.st_atime_ns = int(e.atime() * 1000000000)
- entry.st_mtime_ns = int(e.mtime() * 1000000000)
- entry.st_ctime_ns = int(e.mtime() * 1000000000)
+ if hasattr(entry, 'st_atime_ns'):
+ # llfuse >= 0.42
+ entry.st_atime_ns = int(e.atime() * 1000000000)
+ entry.st_mtime_ns = int(e.mtime() * 1000000000)
+ entry.st_ctime_ns = int(e.mtime() * 1000000000)
+ else:
+ # llfuse < 0.42
+ entry.st_atime = int(e.atime)
+ entry.st_mtime = int(e.mtime)
+ entry.st_ctime = int(e.mtime)
return entry
@catch_exceptions
- def setattr(self, inode, attr, fields, fh, ctx):
+ def setattr(self, inode, attr, fields=None, fh=None, ctx=None):
entry = self.getattr(inode)
if fh is not None and fh in self._filehandles:
@@ -447,7 +465,13 @@ class Operations(llfuse.Operations):
else:
e = self.inodes[inode]
- if fields.update_size and isinstance(e, FuseArvadosFile):
+ if fields is None:
+ # llfuse < 0.42
+ update_size = attr.st_size is not None
+ else:
+ # llfuse >= 0.42
+ update_size = fields.update_size
+ if update_size and isinstance(e, FuseArvadosFile):
with llfuse.lock_released:
e.arvfile.truncate(attr.st_size)
entry.st_size = e.arvfile.size()
@@ -455,7 +479,7 @@ class Operations(llfuse.Operations):
return entry
@catch_exceptions
- def lookup(self, parent_inode, name, ctx):
+ def lookup(self, parent_inode, name, ctx=None):
name = unicode(name, self.inodes.encoding)
inode = None
@@ -491,7 +515,7 @@ class Operations(llfuse.Operations):
self.inodes.del_entry(ent)
@catch_exceptions
- def open(self, inode, flags, ctx):
+ def open(self, inode, flags, ctx=None):
if inode in self.inodes:
p = self.inodes[inode]
else:
@@ -578,7 +602,7 @@ class Operations(llfuse.Operations):
self.release(fh)
@catch_exceptions
- def opendir(self, inode, ctx):
+ def opendir(self, inode, ctx=None):
_logger.debug("arv-mount opendir: inode %i", inode)
if inode in self.inodes:
@@ -617,7 +641,7 @@ class Operations(llfuse.Operations):
e += 1
@catch_exceptions
- def statfs(self, ctx):
+ def statfs(self, ctx=None):
st = llfuse.StatvfsData()
st.f_bsize = 128 * 1024
st.f_blocks = 0
@@ -650,7 +674,7 @@ class Operations(llfuse.Operations):
return p
@catch_exceptions
- def create(self, inode_parent, name, mode, flags, ctx):
+ def create(self, inode_parent, name, mode, flags, ctx=None):
_logger.debug("arv-mount create: parent_inode %i '%s' %o", inode_parent, name, mode)
p = self._check_writable(inode_parent)
@@ -666,7 +690,7 @@ class Operations(llfuse.Operations):
return (fh, self.getattr(f.inode))
@catch_exceptions
- def mkdir(self, inode_parent, name, mode, ctx):
+ def mkdir(self, inode_parent, name, mode, ctx=None):
_logger.debug("arv-mount mkdir: parent_inode %i '%s' %o", inode_parent, name, mode)
p = self._check_writable(inode_parent)
@@ -679,19 +703,19 @@ class Operations(llfuse.Operations):
return self.getattr(d.inode)
@catch_exceptions
- def unlink(self, inode_parent, name, ctx):
+ def unlink(self, inode_parent, name, ctx=None):
_logger.debug("arv-mount unlink: parent_inode %i '%s'", inode_parent, name)
p = self._check_writable(inode_parent)
p.unlink(name)
@catch_exceptions
- def rmdir(self, inode_parent, name, ctx):
+ def rmdir(self, inode_parent, name, ctx=None):
_logger.debug("arv-mount rmdir: parent_inode %i '%s'", inode_parent, name)
p = self._check_writable(inode_parent)
p.rmdir(name)
@catch_exceptions
- def rename(self, inode_parent_old, name_old, inode_parent_new, name_new, ctx):
+ def rename(self, inode_parent_old, name_old, inode_parent_new, name_new, ctx=None):
_logger.debug("arv-mount rename: old_parent_inode %i '%s' new_parent_inode %i '%s'", inode_parent_old, name_old, inode_parent_new, name_new)
src = self._check_writable(inode_parent_old)
dest = self._check_writable(inode_parent_new)
commit e9661d2d3e117fcb9b48848757b6dfe65210ba43
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Dec 31 20:38:02 2016 -0500
8345: Add run-tests.sh --repeat option.
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 2831cf9..b129d09 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -22,6 +22,8 @@ Options:
--leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
Instead, show the path to give as --temp to reuse them in
subsequent invocations.
+--repeat N Repeat each install/test step until it succeeds N times.
+--retry Prompt to retry if an install or test suite fails.
--skip-install Do not run any install steps. Just run tests.
You should provide GOPATH, GEMHOME, and VENVDIR options
from a previous invocation if you use this option.
@@ -219,14 +221,7 @@ do
exit 1
;;
--skip)
- skipwhat="$1"; shift
- if [[ "$skipwhat" == "apps/workbench" ]]; then
- skip["apps/workbench_units"]=1
- skip["apps/workbench_functionals"]=1
- skip["apps/workbench_integration"]=1
- else
- skip[$skipwhat]=1
- fi
+ skip[$1]=1
;;
--only)
only="$1"; skip[$1]=""; shift
@@ -248,6 +243,9 @@ do
--leave-temp)
temp_preserve=1
;;
+ --repeat)
+ repeat=$((${1}+0)); shift
+ ;;
--retry)
retry=1
;;
@@ -497,136 +495,151 @@ then
fi
retry() {
- while ! ${@} && [[ "$retry" == 1 ]]
+ remain="${repeat}"
+ while :
do
- read -p 'Try again? [Y/n] ' x
- if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
- then
+ if ${@}; then
+ if [[ "$remain" -gt 1 ]]; then
+ remain=$((${remain}-1))
+ title "Repeating ${remain} more times"
+ else
+ break
+ fi
+ elif [[ "$retry" == 1 ]]; then
+ read -p 'Try again? [Y/n] ' x
+ if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
+ then
+ break
+ fi
+ else
break
fi
done
}
do_test() {
- retry do_test_once ${@}
+ case "${1}" in
+ apps/workbench_units | apps/workbench_functional | apps/workbench_integration)
+ suite=apps/workbench
+ ;;
+ *)
+ suite="${1}"
+ ;;
+ esac
+ if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \
+ (-z "${only}" || "${only}" == "${suite}") ]]; then
+ retry do_test_once ${@}
+ else
+ title "Skipping ${1} tests"
+ fi
}
do_test_once() {
unset result
- to_test=$1
- if (( [[ "$only" == "apps/workbench" ]] ) &&
- ( [[ "$to_test" == "apps/workbench_units" ]] || [[ "$to_test" == "apps/workbench_functionals" ]] ||
- [[ "$to_test" == "apps/workbench_integration" ]])); then
- to_test="apps/workbench"
- fi
- if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$to_test" ]] )
+
+ title "Running $1 tests"
+ timer_reset
+ if [[ "$2" == "go" ]]
then
- title "Running $1 tests"
- timer_reset
- if [[ "$2" == "go" ]]
- then
- covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
- coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
- # We do "go get -t" here to catch compilation errors
- # before trying "go test". Otherwise, coverage-reporting
- # mode makes Go show the wrong line numbers when reporting
- # compilation errors.
- go get -t "git.curoverse.com/arvados.git/$1" || return 1
- cd "$WORKSPACE/$1" || return 1
- gofmt -e -d . | egrep . && result=1
- if [[ -n "${testargs[$1]}" ]]
- then
- # "go test -check.vv giturl" doesn't work, but this
- # does:
- cd "$WORKSPACE/$1" && go test ${short:+-short} ${testargs[$1]}
- else
- # The above form gets verbose even when testargs is
- # empty, so use this form in such cases:
- go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
- fi
- result=${result:-$?}
- if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
- then
- go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
- rm "$WORKSPACE/tmp/.$covername.tmp"
- fi
- elif [[ "$2" == "pip" ]]
- then
- tries=0
- cd "$WORKSPACE/$1" && while :
- do
- tries=$((${tries}+1))
- # $3 can name a path directory for us to use, including trailing
- # slash; e.g., the bin/ subdirectory of a virtualenv.
- "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
- result=$?
- if [[ ${tries} < 3 && ${result} == 137 ]]
- then
- printf '\n*****\n%s tests killed -- retrying\n*****\n\n' "$1"
- continue
- else
- break
- fi
- done
- elif [[ "$2" != "" ]]
+ covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
+ coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+ # We do "go get -t" here to catch compilation errors
+ # before trying "go test". Otherwise, coverage-reporting
+ # mode makes Go show the wrong line numbers when reporting
+ # compilation errors.
+ go get -t "git.curoverse.com/arvados.git/$1" || return 1
+ cd "$WORKSPACE/$1" || return 1
+ gofmt -e -d . | egrep . && result=1
+ if [[ -n "${testargs[$1]}" ]]
then
- "test_$2"
+ # "go test -check.vv giturl" doesn't work, but this
+ # does:
+ cd "$WORKSPACE/$1" && go test ${short:+-short} ${testargs[$1]}
else
- "test_$1"
+ # The above form gets verbose even when testargs is
+ # empty, so use this form in such cases:
+ go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
fi
result=${result:-$?}
- checkexit $result "$1 tests"
- title "End of $1 tests (`timer`)"
- return $result
+ if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
+ then
+ go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
+ rm "$WORKSPACE/tmp/.$covername.tmp"
+ fi
+ elif [[ "$2" == "pip" ]]
+ then
+ tries=0
+ cd "$WORKSPACE/$1" && while :
+ do
+ tries=$((${tries}+1))
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+ result=$?
+ if [[ ${tries} < 3 && ${result} == 137 ]]
+ then
+ printf '\n*****\n%s tests killed -- retrying\n*****\n\n' "$1"
+ continue
+ else
+ break
+ fi
+ done
+ elif [[ "$2" != "" ]]
+ then
+ "test_$2"
else
- title "Skipping $1 tests"
+ "test_$1"
fi
+ result=${result:-$?}
+ checkexit $result "$1 tests"
+ title "End of $1 tests (`timer`)"
+ return $result
}
do_install() {
- retry do_install_once ${@}
+ if [[ -z "${skip_install}" && \
+ (-z "${only_install}" || "${only_install}" == "${1}") ]]; then
+ retry do_install_once ${@}
+ else
+ title "Skipping $1 install"
+ fi
}
do_install_once() {
- if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
+ title "Running $1 install"
+ timer_reset
+ if [[ "$2" == "go" ]]
then
- title "Running $1 install"
- timer_reset
- if [[ "$2" == "go" ]]
- then
- go get -t "git.curoverse.com/arvados.git/$1"
- elif [[ "$2" == "pip" ]]
- then
- # $3 can name a path directory for us to use, including trailing
- # slash; e.g., the bin/ subdirectory of a virtualenv.
-
- # Need to change to a different directory after creating
- # the source dist package to avoid a pip bug.
- # see https://arvados.org/issues/5766 for details.
-
- # Also need to install twice, because if it believes the package is
- # already installed, pip it won't install it. So the first "pip
- # install" ensures that the dependencies are met, the second "pip
- # install" ensures that we've actually installed the local package
- # we just built.
- cd "$WORKSPACE/$1" \
- && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
- && cd "$WORKSPACE" \
- && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
- && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
- elif [[ "$2" != "" ]]
- then
- "install_$2"
- else
- "install_$1"
- fi
- result=$?
- checkexit $result "$1 install"
- title "End of $1 install (`timer`)"
- return $result
+ go get -t "git.curoverse.com/arvados.git/$1"
+ elif [[ "$2" == "pip" ]]
+ then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
+ # Need to change to a different directory after creating
+ # the source dist package to avoid a pip bug.
+ # see https://arvados.org/issues/5766 for details.
+
+ # Also need to install twice, because if it believes the package is
+ # already installed, pip it won't install it. So the first "pip
+ # install" ensures that the dependencies are met, the second "pip
+ # install" ensures that we've actually installed the local package
+ # we just built.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ && cd "$WORKSPACE" \
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+ elif [[ "$2" != "" ]]
+ then
+ "install_$2"
else
- title "Skipping $1 install"
+ "install_$1"
fi
+ result=$?
+ checkexit $result "$1 install"
+ title "End of $1 install (`timer`)"
+ return $result
}
bundle_install_trylocal() {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list