[ARVADOS] updated: a87ad3026e4f7e1cc2e22082a523fa7143388c88

git at public.curoverse.com git at public.curoverse.com
Thu May 8 16:10:18 EDT 2014


Summary of changes:
 .gitignore                                         |    2 +
 .../app/controllers/keep_disks_controller.rb       |   28 +-
 apps/workbench/app/models/pipeline_instance.rb     |    2 +-
 .../app/views/keep_disks/_content_layout.html.erb  |    2 +-
 apps/workbench/app/views/users/_tables.html.erb    |    2 +-
 apps/workbench/test/test_helper.rb                 |   20 +-
 doc/_config.yml                                    |    5 +-
 doc/api/methods.html.textile.liquid                |   39 ++-
 doc/api/methods/logs.html.textile.liquid           |   22 +-
 doc/api/schema/Job.html.textile.liquid             |    1 +
 doc/install/index.html.md.liquid                   |   18 -
 doc/install/index.html.textile.liquid              |   18 +
 doc/install/install-api-server.html.textile.liquid |    5 +-
 .../install-crunch-dispatch.html.textile.liquid    |   44 +-
 doc/install/install-sso.html.textile.liquid        |    3 +-
 .../install-workbench-app.html.textile.liquid      |   30 +-
 doc/sdk/index.html.textile.liquid                  |    3 +-
 doc/sdk/java/index.html.textile.liquid             |  140 +++++
 doc/sdk/perl/index.html.textile.liquid             |    2 +-
 doc/sdk/python/sdk-python.html.textile.liquid      |    9 +-
 docker/build_tools/Makefile                        |   10 +
 docker/jobs/Dockerfile                             |   20 +
 sdk/cli/bin/arv-run-pipeline-instance              |   44 ++-
 sdk/cli/bin/crunch-job                             |   69 ++-
 sdk/cli/test/test_arv-run-pipeline-instance.rb     |   33 ++
 sdk/java/.classpath                                |   21 +
 sdk/java/.project                                  |   14 +
 sdk/java/.settings/org.eclipse.jdt.core.prefs      |    5 +
 sdk/java/ArvadosSDKJavaExample.java                |   80 +++
 sdk/java/ArvadosSDKJavaExampleWithPrompt.java      |  123 ++++
 sdk/java/README                                    |    4 +
 sdk/java/pom.xml                                   |  106 ++++
 .../main/java/org/arvados/sdk/java/Arvados.java    |  366 ++++++++++++
 .../java/org/arvados/sdk/java/MethodDetails.java   |   22 +
 sdk/java/src/main/resources/log4j.properties       |   11 +
 .../java/org/arvados/sdk/java/ArvadosTest.java     |  387 +++++++++++++
 sdk/java/src/test/resources/first_pipeline.json    |   16 +
 sdk/python/.gitignore                              |    1 -
 sdk/python/arvados/events.py                       |   33 ++
 sdk/python/arvados/fuse.py                         |  317 -----------
 sdk/python/arvados/fuse/__init__.py                |  585 ++++++++++++++++++++
 sdk/python/bin/arv-mount                           |   51 ++-
 sdk/python/build.sh                                |   16 -
 sdk/python/requirements.txt                        |    3 +
 sdk/python/run_test_server.py                      |   86 +++
 sdk/python/{setup.py.src => setup.py}              |    9 +-
 sdk/python/setup.py.src                            |    8 +-
 sdk/python/setup_fuse.py                           |   21 +
 sdk/python/setup_header.py                         |   11 +
 sdk/python/test_mount.py                           |  246 +++++++--
 sdk/python/test_websockets.py                      |   32 ++
 .../controllers/arvados/v1/schema_controller.rb    |   15 +-
 services/api/app/models/pipeline_instance.rb       |   61 ++-
 services/api/lib/record_filters.rb                 |   13 +-
 services/api/script/import_commits.rb              |    9 -
 .../functional/arvados/v1/jobs_controller_test.rb  |   34 ++
 56 files changed, 2713 insertions(+), 564 deletions(-)
 delete mode 100644 doc/install/index.html.md.liquid
 create mode 100644 doc/install/index.html.textile.liquid
 create mode 100644 doc/sdk/java/index.html.textile.liquid
 create mode 100644 docker/jobs/Dockerfile
 create mode 100644 sdk/cli/test/test_arv-run-pipeline-instance.rb
 create mode 100644 sdk/java/.classpath
 create mode 100644 sdk/java/.project
 create mode 100644 sdk/java/.settings/org.eclipse.jdt.core.prefs
 create mode 100644 sdk/java/ArvadosSDKJavaExample.java
 create mode 100644 sdk/java/ArvadosSDKJavaExampleWithPrompt.java
 create mode 100644 sdk/java/README
 create mode 100644 sdk/java/pom.xml
 create mode 100644 sdk/java/src/main/java/org/arvados/sdk/java/Arvados.java
 create mode 100644 sdk/java/src/main/java/org/arvados/sdk/java/MethodDetails.java
 create mode 100644 sdk/java/src/main/resources/log4j.properties
 create mode 100644 sdk/java/src/test/java/org/arvados/sdk/java/ArvadosTest.java
 create mode 100644 sdk/java/src/test/resources/first_pipeline.json
 create mode 100644 sdk/python/arvados/events.py
 delete mode 100644 sdk/python/arvados/fuse.py
 create mode 100644 sdk/python/arvados/fuse/__init__.py
 delete mode 100755 sdk/python/build.sh
 create mode 100644 sdk/python/run_test_server.py
 copy sdk/python/{setup.py.src => setup.py} (82%)
 create mode 100644 sdk/python/setup_fuse.py
 create mode 100644 sdk/python/setup_header.py
 create mode 100644 sdk/python/test_websockets.py
 delete mode 100755 services/api/script/import_commits.rb

       via  a87ad3026e4f7e1cc2e22082a523fa7143388c88 (commit)
       via  b2a8e117cd2a02851c69136f76a1e63005386f1d (commit)
       via  97d8fa64264069fe6d8ae6c8131fd3cc2f347189 (commit)
       via  0c1af8806516569044c7354ff58f7371a510594d (commit)
       via  16af0b22def5b80a1861788907a2fc45b14e3273 (commit)
       via  15945ebab4fc71980b2c6ddceb149a3d7365c29e (commit)
       via  a9eaabb02fe7110f0e16bcd7f27d1a0cd5eab08b (commit)
       via  3aaf11c2dd372b5d34ba317a1a4b761d263f86d6 (commit)
       via  eed7af8f901adb263c870a2277ec166467a28d77 (commit)
       via  d97e3de81c48673b0e1d3927edaaf0e560a9a2ba (commit)
       via  14c312c452e0e010d617c02a33b94a178632ac26 (commit)
       via  5c3c04ad25cfabaa860bb0ea3f3dc078a9562008 (commit)
       via  5e968d7875c62a6145b7aefc39f070d14f4c97e8 (commit)
       via  eff3277fa6800bbc4ac81654f8883bbfd3a85d30 (commit)
       via  002f863605469cee7e112f4c16a78e3b6278672c (commit)
       via  6c2a704b7a2d721087976b2b8ec4f22cdaf44178 (commit)
       via  cdf7b162906cb37d4ecf88ff6c72433bcdd8fc84 (commit)
       via  722862465129f1f30cbb2432132140883d31b4ca (commit)
       via  27fa0d391eb95008acf41ba207bc13b8c4fe794c (commit)
       via  f7f61eca36d66a723b02ac34db888f5fbdc63fc8 (commit)
       via  2de827e57ae9ba1b0d322199d071ae900a5b7e0c (commit)
       via  d9aa5decc6edce30285df049277de2cf2c138c26 (commit)
       via  222ce386e36b3d146e718a5d2f64a95fb30996bb (commit)
       via  26d69329e87e343b84a132a4754068aca66ad132 (commit)
       via  036a701b45399e0114eed5b0e1ce6b478c614534 (commit)
       via  b4516edf0a7cd1b584442953a8c965fc65104906 (commit)
       via  a3cd0039e84373541d6648b76647457be5a7e098 (commit)
       via  8fcba3783d9a79bb18a00d339f9b3222f1eccd7f (commit)
       via  1c6bc4bcac86195a120e6be564703e7d2133dbe7 (commit)
       via  71143e41dbc8755d5ccb97990525bbb05aad69e9 (commit)
       via  31c711afc4ff8a1728aeb3e5f405a131047671c9 (commit)
       via  88590483c4d24904267b550cbb3af7d5a11d8953 (commit)
       via  5b252752cdfd8b129d10913f7e2ec7b47f544ead (commit)
       via  2376147b3d039acf08456a53015d59f04eabbf82 (commit)
       via  fde1e4f00e82c7a776892c1ab9c5edacf05dc1c3 (commit)
       via  97252f1f34486073c778c0a1c9de25f2f275cacd (commit)
       via  ff6a6fb89b42d17693786be6405c267bee445c4d (commit)
       via  45938e41f983ce41d8b55c3f980b7f31bc269124 (commit)
       via  f91c73aba940adf1e9fbd530d102931af45a63c5 (commit)
       via  d70ff3d064c9a6da8b8678b249abf7f4b93b6370 (commit)
       via  52a3016af3ff21b8fb5402bf70d0aab07f5ee486 (commit)
       via  e65b2e685833e1d850c8d671fd1df9d4e4fea4a3 (commit)
       via  fdbea9e3c225a3184f8f7c2fac52abde643603c6 (commit)
       via  a04d67ef0823d6c87e8515cfe6cf88793ef5e3bb (commit)
       via  df2ce01533175ff7190a5d13fdca4b61f22befd3 (commit)
       via  f493dd4dfab89c228fab05e2b46851211c7fc586 (commit)
       via  facf07110e28fe552342fbad623d621b43e22b99 (commit)
       via  6f3853aef989e8c2243159bbf0ea352948ae4878 (commit)
       via  733269172b6b9802c2a00a8f8d19d6a520546a67 (commit)
       via  a2104e385abe47470dd42836a66360bf62afa57b (commit)
       via  35fb8a1cd59d1dc0ad4be89c64e8b48c750faaeb (commit)
       via  0ab985ff1da223e4065273fd1a549284ff3454ee (commit)
       via  e6aa5b48be1209d74c4c69be3eeca549f02418f3 (commit)
       via  2e524eb008f8b70ca1263ccde460365109c66a90 (commit)
       via  3ac9fd0c91fe202a59e7c3611156bb1a9b8118fe (commit)
       via  01436e8b76300759ef4bfea30d7798445d50bf60 (commit)
       via  eee2c981d6a29eb7f15b8957570bbf8515d3d947 (commit)
       via  b5f193aa657134822d4df67cabc25c631926395b (commit)
       via  ff1e29f702d4f3036f5af014ad6e423b886ed0bb (commit)
       via  66f1ae7a9be590461592199c6d3646dbc4eba786 (commit)
       via  4bd14ccfa216fe1d115772508ac7d7c64ec59a48 (commit)
       via  1b8120689041941db937883c3132a9c6819b56c5 (commit)
       via  cbf0429a8a5edd2f8a8d9b01ce39334e07e56bf8 (commit)
       via  121fb6bdcdf3731903ce3d21c0a284c20e6a6fca (commit)
       via  6abcf1f6bdb82b176a9fbb48b6c7cefa64970a9d (commit)
       via  8624ad6e91ed73876bd4f12fe8df2387e3e6330e (commit)
       via  cbcb69fcbf73a1bde0ae78be8c0ad25f84c3f7c3 (commit)
       via  c6c3d3d23d1cca50381570d0558470d47e71fd95 (commit)
       via  2b03b05de753509c026b4c67857cd49a11f44dcf (commit)
       via  67fef1f4bec2d314c9d37bb05c55ff1067ba7b49 (commit)
       via  a998a520deb859216271567f1bf3623f310cf2d6 (commit)
       via  51928265eaf8ca6b6319dd231d74604ceb3af287 (commit)
       via  181fead894b660893e0e09274e609ec8ee4230a1 (commit)
       via  51af13e97bc4725e8cc010c72bbedb667b0a0e0f (commit)
       via  e9359f96365d629a688cbbc2f7d063fabf535a9f (commit)
       via  228f1ecbb98438fcd5bbe59465c830be637d38ce (commit)
       via  ee056b0ceb9ce569e0ee9a64b41ead728636772a (commit)
       via  3e237c89424489bad02cde9be33f715269e0c6b8 (commit)
       via  00ba956a67072f8b4a77fe71d3dc1e4dd8f70e98 (commit)
       via  98fd371d846022d8ccc25e8234d913e436ecce69 (commit)
       via  9c444153e80d3a872e11ca5898491c747a881a4d (commit)
       via  0f7fa46c1311ddd2aa6e7f0c280b98a6d69d4ec8 (commit)
       via  2e04dd8e9f872506922744dd449cccb99b55a847 (commit)
       via  752bd0d292c7fd6eb55b39cdc9d7373c7f35e0ec (commit)
       via  67c754c065dbe79e14a3c60640e6e604af32d0bd (commit)
       via  77c4def0b319e3720dba05a3c1fb0ab78323f227 (commit)
       via  65c155b7f691ce6a885125618bde3c9d68f6b47e (commit)
       via  d433ba0f548414219b3d976125e1ea530e1ba9ef (commit)
       via  d17ef1d036b4262f6c6c40324cc56d04d26581d3 (commit)
       via  c436da861845723ab18f56dba96052cf2569e539 (commit)
       via  22233bfa18060d000e303d7734ec3016f7e80c8e (commit)
       via  ebddfbe7fbca80ecf852bbb20af9c6b584394691 (commit)
       via  603bccd66ac868c2accf9dff9c3ca6b7feeed962 (commit)
       via  1f763a5b1ad5c65a3265b4284958ecc0fe336f50 (commit)
       via  9143c3ec039d6c89512055f70c2fcc6e1065f1d7 (commit)
       via  415347a3354031b48c22cae56c7bbfdcb76d607c (commit)
       via  a8918e636507200148758302dc6b9a081dc2f206 (commit)
       via  a74ae790b5a0682fa00423626103482c145e1b91 (commit)
       via  66d12ae6dc72d45cc3cb3fcc007cb41371371616 (commit)
       via  8ee3fe6685a716918a403f559c626c3b38e411f2 (commit)
       via  2e273679fbf3f73ac7b7cfcf9ef8380047e959a0 (commit)
       via  ab5ded7cdf302ad156c58d47f59ab07e3a14ec69 (commit)
       via  abcde4a0bed9e80fc0925e70d151c7893043bc82 (commit)
       via  c13b9d5300b7f778e2fc05f9d043db4122d15aa1 (commit)
       via  c7fed8bfbd8a0e6edf83142b50bed4fe53c10c5c (commit)
       via  d28bf64e878b8d45e1ace6409e94027a039d3909 (commit)
       via  72493a2fef971fc685ecef42c2b697338da69d2c (commit)
       via  4d1481ed38e238e2e39d37775cb8cd86150bd04e (commit)
       via  c4fcddbe2a01ebbc3a4898dfa8744eb6fc45227e (commit)
       via  84c471b1cfb20116b2d7d26215c2517cb57ba53f (commit)
       via  a5034fccf4a829a36d6e2670a9f62e4f7a545f2f (commit)
       via  72c312b352a88740a44fba9e274d72bbabb489a7 (commit)
       via  ba4693cda66632d3da1a711b6910441958ecee01 (commit)
       via  2e5fef753892523984f18c26fc16cfd599115f34 (commit)
      from  7fb83a3380e62721801a4980c48ba78208c7b2e2 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit a87ad3026e4f7e1cc2e22082a523fa7143388c88
Merge: b2a8e11 97d8fa6
Author: Misha Zatsman <misha-arvados at zatsman.com>
Date:   Thu May 8 20:10:04 2014 +0000

    Merge branch 'master' of git.curoverse.com:arvados into 2638-add-cache-age-disk-usage-histogram-to-workbench


commit b2a8e117cd2a02851c69136f76a1e63005386f1d
Author: Misha Zatsman <misha-arvados at zatsman.com>
Date:   Thu May 8 20:09:07 2014 +0000

    Added some documentation and renamed variables after Tim's review.

diff --git a/apps/workbench/app/controllers/keep_disks_controller.rb b/apps/workbench/app/controllers/keep_disks_controller.rb
index 345d17f..f57455b 100644
--- a/apps/workbench/app/controllers/keep_disks_controller.rb
+++ b/apps/workbench/app/controllers/keep_disks_controller.rb
@@ -6,9 +6,29 @@ class KeepDisksController < ApplicationController
   end
 
   def index
-    # Retrieve cache age histogram info
+    # Retrieve cache age histogram info from logs.
+
+    # In the logs we expect to find it in an ordered list with entries
+    # of the form (mtime, disk proportion free).
+
+    # An entry of the form (1388747781, 0.52) means that if we deleted
+    # the oldest non-presisted blocks until we had 52% of the disk
+    # free, then all blocks with an mtime greater than 1388747781
+    # would be preserved.
+
+    # The chart we want to produce, will tell us how much of the disk
+    # will be free if we use a cache age of x days. Therefore we will
+    # produce output specifying the age, cache and persisted. age is
+    # specified in milliseconds. cache is the size of the cache if we
+    # delete all blocks older than age. persistent is the size of the
+    # persisted blocks. It is constant regardless of age, but it lets
+    # us show a stacked graph.
+
+    # Finally each entry in cache_age_histogram is a dictionary,
+    # because that's what our charting package wats.
+
     @cache_age_histogram = []
-    @histogram_date = 0
+    @histogram_pretty_date = nil
     histogram_log = Log.
       filter([[:event_type, '=', 'block-age-free-space-histogram']]).
       order(:created_at => :desc).
@@ -18,12 +38,12 @@ class KeepDisksController < ApplicationController
       # specified limit(1)
       @cache_age_histogram = log_entry['properties'][:histogram]
       # Javascript wants dates in milliseconds.
-      @histogram_date = log_entry['event_at'].to_i * 1000
+      histogram_date_ms = log_entry['event_at'].to_i * 1000
       @histogram_pretty_date = log_entry['event_at'].strftime('%b %-d, %Y')
 
       total_free_cache = @cache_age_histogram[-1][1]
       persisted_storage = 1 - total_free_cache
-      @cache_age_histogram.map! { |x| {:age => @histogram_date - x[0]*1000,
+      @cache_age_histogram.map! { |x| {:age => histogram_date_ms - x[0]*1000,
           :cache => total_free_cache - x[1],
           :persisted => persisted_storage} }
     end
diff --git a/apps/workbench/app/views/keep_disks/_content_layout.html.erb b/apps/workbench/app/views/keep_disks/_content_layout.html.erb
index 38188a5..0f5cd7a 100644
--- a/apps/workbench/app/views/keep_disks/_content_layout.html.erb
+++ b/apps/workbench/app/views/keep_disks/_content_layout.html.erb
@@ -1,4 +1,4 @@
-<% if @histogram_date > 0 %>
+<% unless @histogram_pretty_date.nil? %>
   <% content_for :tab_panes do %>
   <%# We use protocol-relative paths here to avoid browsers refusing to load javascript over http in a page that was loaded over https. %>
   <%= javascript_include_tag '//cdnjs.cloudflare.com/ajax/libs/raphael/2.1.2/raphael-min.js' %>

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list