[ARVADOS] updated: 1.3.0-1468-gfec779666
Git user
git at public.curoverse.com
Fri Aug 9 15:02:52 UTC 2019
Summary of changes:
sdk/go/crunchrunner/crunchrunner.go | 439 -------------------
sdk/go/crunchrunner/crunchrunner_test.go | 478 ---------------------
sdk/go/crunchrunner/upload.go | 241 -----------
sdk/go/crunchrunner/upload_test.go | 152 -------
services/api/app/helpers/commits_helper.rb | 263 ++++++++++++
services/api/app/models/commit.rb | 272 ------------
services/api/app/models/job.rb | 61 ++-
...s.rb => 20190809135453_remove_commits_table.rb} | 4 +-
services/api/db/structure.sql | 58 +--
services/api/test/helpers/git_test_helper.rb | 8 +-
services/api/test/unit/commit_test.rb | 70 +--
services/api/test/unit/job_test.rb | 47 --
12 files changed, 332 insertions(+), 1761 deletions(-)
delete mode 100644 sdk/go/crunchrunner/crunchrunner.go
delete mode 100644 sdk/go/crunchrunner/crunchrunner_test.go
delete mode 100644 sdk/go/crunchrunner/upload.go
delete mode 100644 sdk/go/crunchrunner/upload_test.go
delete mode 100644 services/api/app/models/commit.rb
copy services/api/db/migrate/{20130315183626_add_log_to_jobs.rb => 20190809135453_remove_commits_table.rb} (58%)
via fec7796668a4f4a73bb52e16e13e4504f76649f5 (commit)
via e10aa8e0c8b9c45d69832e71480cfb3d6929834e (commit)
from a8aa509cf3913c8dc91640ddcad6b25e6609517a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fec7796668a4f4a73bb52e16e13e4504f76649f5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 9 10:42:34 2019 -0400
15133: Remove 'commits' table, move functions to CommitsHelper
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/app/helpers/commits_helper.rb b/services/api/app/helpers/commits_helper.rb
index d44719f92..b1c02b2a7 100644
--- a/services/api/app/helpers/commits_helper.rb
+++ b/services/api/app/helpers/commits_helper.rb
@@ -3,4 +3,267 @@
# SPDX-License-Identifier: AGPL-3.0
module CommitsHelper
+
+ class GitError < RequestError
+ def http_status
+ 422
+ end
+ end
+
+ def self.git_check_ref_format(e)
+ if !e or e.empty? or e[0] == '-' or e[0] == '$'
+ # definitely not valid
+ false
+ else
+ `git check-ref-format --allow-onelevel #{e.shellescape}`
+ $?.success?
+ end
+ end
+
+ # Return an array of commits (each a 40-char sha1) satisfying the
+ # given criteria.
+ #
+ # Return [] if the revisions given in minimum/maximum are invalid or
+ # don't exist in the given repository.
+ #
+ # Raise ArgumentError if the given repository is invalid, does not
+ # exist, or cannot be read for any reason. (Any transient error that
+ # prevents commit ranges from resolving must raise rather than
+ # returning an empty array.)
+ #
+ # repository can be the name of a locally hosted repository or a git
+ # URL (see git-fetch(1)). Currently http, https, and git schemes are
+ # supported.
+ def self.find_commit_range repository, minimum, maximum, exclude
+ if minimum and minimum.empty?
+ minimum = nil
+ end
+
+ if minimum and !git_check_ref_format(minimum)
+ Rails.logger.warn "find_commit_range called with invalid minimum revision: '#{minimum}'"
+ return []
+ end
+
+ if maximum and !git_check_ref_format(maximum)
+ Rails.logger.warn "find_commit_range called with invalid maximum revision: '#{maximum}'"
+ return []
+ end
+
+ if !maximum
+ maximum = "HEAD"
+ end
+
+ gitdir, is_remote = git_dir_for repository
+ fetch_remote_repository gitdir, repository if is_remote
+ ENV['GIT_DIR'] = gitdir
+
+ commits = []
+
+ # Get the commit hash for the upper bound
+ max_hash = nil
+ git_max_hash_cmd = "git rev-list --max-count=1 #{maximum.shellescape} --"
+ IO.foreach("|#{git_max_hash_cmd}") do |line|
+ max_hash = line.strip
+ end
+
+ # If not found, nothing else to do
+ if !max_hash
+ Rails.logger.warn "no refs found looking for max_hash: `GIT_DIR=#{gitdir} #{git_max_hash_cmd}` returned no output"
+ return []
+ end
+
+ # If string is invalid, nothing else to do
+ if !git_check_ref_format(max_hash)
+ Rails.logger.warn "ref returned by `GIT_DIR=#{gitdir} #{git_max_hash_cmd}` was invalid for max_hash: #{max_hash}"
+ return []
+ end
+
+ resolved_exclude = nil
+ if exclude
+ resolved_exclude = []
+ exclude.each do |e|
+ if git_check_ref_format(e)
+ IO.foreach("|git rev-list --max-count=1 #{e.shellescape} --") do |line|
+ resolved_exclude.push(line.strip)
+ end
+ else
+ Rails.logger.warn "find_commit_range called with invalid exclude invalid characters: '#{exclude}'"
+ return []
+ end
+ end
+ end
+
+ if minimum
+ # Get the commit hash for the lower bound
+ min_hash = nil
+ git_min_hash_cmd = "git rev-list --max-count=1 #{minimum.shellescape} --"
+ IO.foreach("|#{git_min_hash_cmd}") do |line|
+ min_hash = line.strip
+ end
+
+ # If not found, nothing else to do
+ if !min_hash
+ Rails.logger.warn "no refs found looking for min_hash: `GIT_DIR=#{gitdir} #{git_min_hash_cmd}` returned no output"
+ return []
+ end
+
+ # If string is invalid, nothing else to do
+ if !git_check_ref_format(min_hash)
+ Rails.logger.warn "ref returned by `GIT_DIR=#{gitdir} #{git_min_hash_cmd}` was invalid for min_hash: #{min_hash}"
+ return []
+ end
+
+ # Now find all commits between them
+ IO.foreach("|git rev-list #{min_hash.shellescape}..#{max_hash.shellescape} --") do |line|
+ hash = line.strip
+ commits.push(hash) if !resolved_exclude or !resolved_exclude.include? hash
+ end
+
+ commits.push(min_hash) if !resolved_exclude or !resolved_exclude.include? min_hash
+ else
+ commits.push(max_hash) if !resolved_exclude or !resolved_exclude.include? max_hash
+ end
+
+ commits
+ end
+
+ # Given a repository (url, or name of hosted repo) and commit sha1,
+ # copy the commit into the internal git repo (if necessary), and tag
+ # it with the given tag (typically a job UUID).
+ #
+ # The repo can be a remote url, but in this case sha1 must already
+ # be present in our local cache for that repo: e.g., sha1 was just
+ # returned by find_commit_range.
+ def self.tag_in_internal_repository repo_name, sha1, tag
+ unless git_check_ref_format tag
+ raise ArgumentError.new "invalid tag #{tag}"
+ end
+ unless /^[0-9a-f]{40}$/ =~ sha1
+ raise ArgumentError.new "invalid sha1 #{sha1}"
+ end
+ src_gitdir, _ = git_dir_for repo_name
+ unless src_gitdir
+ raise ArgumentError.new "no local repository for #{repo_name}"
+ end
+ dst_gitdir = Rails.configuration.Containers.JobsAPI.GitInternalDir
+
+ begin
+ commit_in_dst = must_git(dst_gitdir, "log -n1 --format=%H #{sha1.shellescape}^{commit}").strip
+ rescue GitError
+ commit_in_dst = false
+ end
+
+ tag_cmd = "tag --force #{tag.shellescape} #{sha1.shellescape}^{commit}"
+ if commit_in_dst == sha1
+ must_git(dst_gitdir, tag_cmd)
+ else
+ # git-fetch is faster than pack-objects|unpack-objects, but
+ # git-fetch can't fetch by sha1. So we first try to fetch a
+ # branch that has the desired commit, and if that fails (there
+ # is no such branch, or the branch we choose changes under us in
+ # race), we fall back to pack|unpack.
+ begin
+ branches = must_git(src_gitdir,
+ "branch --contains #{sha1.shellescape}")
+ m = branches.match(/^. (\w+)\n/)
+ if !m
+ raise GitError.new "commit is not on any branch"
+ end
+ branch = m[1]
+ must_git(dst_gitdir,
+ "fetch file://#{src_gitdir.shellescape} #{branch.shellescape}")
+ # Even if all of the above steps succeeded, we might still not
+ # have the right commit due to a race, in which case tag_cmd
+ # will fail, and we'll need to fall back to pack|unpack. So
+ # don't be tempted to condense this tag_cmd and the one in the
+ # rescue block into a single attempt.
+ must_git(dst_gitdir, tag_cmd)
+ rescue GitError
+ must_pipe("echo #{sha1.shellescape}",
+ "git --git-dir #{src_gitdir.shellescape} pack-objects -q --revs --stdout",
+ "git --git-dir #{dst_gitdir.shellescape} unpack-objects -q")
+ must_git(dst_gitdir, tag_cmd)
+ end
+ end
+ end
+
+ protected
+
+ def self.remote_url? repo_name
+ /^(https?|git):\/\// =~ repo_name
+ end
+
+ # Return [local_git_dir, is_remote]. If is_remote, caller must use
+ # fetch_remote_repository to ensure content is up-to-date.
+ #
+ # Raises an exception if the latest content could not be fetched for
+ # any reason.
+ def self.git_dir_for repo_name
+ if remote_url? repo_name
+ return [cache_dir_for(repo_name), true]
+ end
+ repos = Repository.readable_by(current_user).where(name: repo_name)
+ if repos.count == 0
+ raise ArgumentError.new "Repository not found: '#{repo_name}'"
+ elsif repos.count > 1
+ Rails.logger.error "Multiple repositories with name=='#{repo_name}'!"
+ raise ArgumentError.new "Name conflict"
+ else
+ return [repos.first.server_path, false]
+ end
+ end
+
+ def self.cache_dir_for git_url
+ File.join(cache_dir_base, Digest::SHA1.hexdigest(git_url) + ".git").to_s
+ end
+
+ def self.cache_dir_base
+ Rails.root.join 'tmp', 'git-cache'
+ end
+
+ def self.fetch_remote_repository gitdir, git_url
+ # Caller decides which protocols are worth using. This is just a
+ # safety check to ensure we never use urls like "--flag" or wander
+ # into git's hardlink features by using bare "/path/foo" instead
+ # of "file:///path/foo".
+ unless /^[a-z]+:\/\// =~ git_url
+ raise ArgumentError.new "invalid git url #{git_url}"
+ end
+ begin
+ must_git gitdir, "branch"
+ rescue GitError => e
+ raise unless /Not a git repository/i =~ e.to_s
+ # OK, this just means we need to create a blank cache repository
+ # before fetching.
+ FileUtils.mkdir_p gitdir
+ must_git gitdir, "init"
+ end
+ must_git(gitdir,
+ "fetch --no-progress --tags --prune --force --update-head-ok #{git_url.shellescape} 'refs/heads/*:refs/heads/*'")
+ end
+
+ def self.must_git gitdir, *cmds
+ # Clear token in case a git helper tries to use it as a password.
+ orig_token = ENV['ARVADOS_API_TOKEN']
+ ENV['ARVADOS_API_TOKEN'] = ''
+ last_output = ''
+ begin
+ git = "git --git-dir #{gitdir.shellescape}"
+ cmds.each do |cmd|
+ last_output = must_pipe git+" "+cmd
+ end
+ ensure
+ ENV['ARVADOS_API_TOKEN'] = orig_token
+ end
+ return last_output
+ end
+
+ def self.must_pipe *cmds
+ cmd = cmds.join(" 2>&1 |") + " 2>&1"
+ out = IO.read("| </dev/null #{cmd}")
+ if not $?.success?
+ raise GitError.new "#{cmd}: #{$?}: #{out}"
+ end
+ return out
+ end
end
diff --git a/services/api/app/models/commit.rb b/services/api/app/models/commit.rb
deleted file mode 100644
index 2f7e9cd93..000000000
--- a/services/api/app/models/commit.rb
+++ /dev/null
@@ -1,272 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'request_error'
-
-class Commit < ActiveRecord::Base
- extend CurrentApiClient
-
- class GitError < RequestError
- def http_status
- 422
- end
- end
-
- def self.git_check_ref_format(e)
- if !e or e.empty? or e[0] == '-' or e[0] == '$'
- # definitely not valid
- false
- else
- `git check-ref-format --allow-onelevel #{e.shellescape}`
- $?.success?
- end
- end
-
- # Return an array of commits (each a 40-char sha1) satisfying the
- # given criteria.
- #
- # Return [] if the revisions given in minimum/maximum are invalid or
- # don't exist in the given repository.
- #
- # Raise ArgumentError if the given repository is invalid, does not
- # exist, or cannot be read for any reason. (Any transient error that
- # prevents commit ranges from resolving must raise rather than
- # returning an empty array.)
- #
- # repository can be the name of a locally hosted repository or a git
- # URL (see git-fetch(1)). Currently http, https, and git schemes are
- # supported.
- def self.find_commit_range repository, minimum, maximum, exclude
- if minimum and minimum.empty?
- minimum = nil
- end
-
- if minimum and !git_check_ref_format(minimum)
- logger.warn "find_commit_range called with invalid minimum revision: '#{minimum}'"
- return []
- end
-
- if maximum and !git_check_ref_format(maximum)
- logger.warn "find_commit_range called with invalid maximum revision: '#{maximum}'"
- return []
- end
-
- if !maximum
- maximum = "HEAD"
- end
-
- gitdir, is_remote = git_dir_for repository
- fetch_remote_repository gitdir, repository if is_remote
- ENV['GIT_DIR'] = gitdir
-
- commits = []
-
- # Get the commit hash for the upper bound
- max_hash = nil
- git_max_hash_cmd = "git rev-list --max-count=1 #{maximum.shellescape} --"
- IO.foreach("|#{git_max_hash_cmd}") do |line|
- max_hash = line.strip
- end
-
- # If not found, nothing else to do
- if !max_hash
- logger.warn "no refs found looking for max_hash: `GIT_DIR=#{gitdir} #{git_max_hash_cmd}` returned no output"
- return []
- end
-
- # If string is invalid, nothing else to do
- if !git_check_ref_format(max_hash)
- logger.warn "ref returned by `GIT_DIR=#{gitdir} #{git_max_hash_cmd}` was invalid for max_hash: #{max_hash}"
- return []
- end
-
- resolved_exclude = nil
- if exclude
- resolved_exclude = []
- exclude.each do |e|
- if git_check_ref_format(e)
- IO.foreach("|git rev-list --max-count=1 #{e.shellescape} --") do |line|
- resolved_exclude.push(line.strip)
- end
- else
- logger.warn "find_commit_range called with invalid exclude invalid characters: '#{exclude}'"
- return []
- end
- end
- end
-
- if minimum
- # Get the commit hash for the lower bound
- min_hash = nil
- git_min_hash_cmd = "git rev-list --max-count=1 #{minimum.shellescape} --"
- IO.foreach("|#{git_min_hash_cmd}") do |line|
- min_hash = line.strip
- end
-
- # If not found, nothing else to do
- if !min_hash
- logger.warn "no refs found looking for min_hash: `GIT_DIR=#{gitdir} #{git_min_hash_cmd}` returned no output"
- return []
- end
-
- # If string is invalid, nothing else to do
- if !git_check_ref_format(min_hash)
- logger.warn "ref returned by `GIT_DIR=#{gitdir} #{git_min_hash_cmd}` was invalid for min_hash: #{min_hash}"
- return []
- end
-
- # Now find all commits between them
- IO.foreach("|git rev-list #{min_hash.shellescape}..#{max_hash.shellescape} --") do |line|
- hash = line.strip
- commits.push(hash) if !resolved_exclude or !resolved_exclude.include? hash
- end
-
- commits.push(min_hash) if !resolved_exclude or !resolved_exclude.include? min_hash
- else
- commits.push(max_hash) if !resolved_exclude or !resolved_exclude.include? max_hash
- end
-
- commits
- end
-
- # Given a repository (url, or name of hosted repo) and commit sha1,
- # copy the commit into the internal git repo (if necessary), and tag
- # it with the given tag (typically a job UUID).
- #
- # The repo can be a remote url, but in this case sha1 must already
- # be present in our local cache for that repo: e.g., sha1 was just
- # returned by find_commit_range.
- def self.tag_in_internal_repository repo_name, sha1, tag
- unless git_check_ref_format tag
- raise ArgumentError.new "invalid tag #{tag}"
- end
- unless /^[0-9a-f]{40}$/ =~ sha1
- raise ArgumentError.new "invalid sha1 #{sha1}"
- end
- src_gitdir, _ = git_dir_for repo_name
- unless src_gitdir
- raise ArgumentError.new "no local repository for #{repo_name}"
- end
- dst_gitdir = Rails.configuration.Containers.JobsAPI.GitInternalDir
-
- begin
- commit_in_dst = must_git(dst_gitdir, "log -n1 --format=%H #{sha1.shellescape}^{commit}").strip
- rescue GitError
- commit_in_dst = false
- end
-
- tag_cmd = "tag --force #{tag.shellescape} #{sha1.shellescape}^{commit}"
- if commit_in_dst == sha1
- must_git(dst_gitdir, tag_cmd)
- else
- # git-fetch is faster than pack-objects|unpack-objects, but
- # git-fetch can't fetch by sha1. So we first try to fetch a
- # branch that has the desired commit, and if that fails (there
- # is no such branch, or the branch we choose changes under us in
- # race), we fall back to pack|unpack.
- begin
- branches = must_git(src_gitdir,
- "branch --contains #{sha1.shellescape}")
- m = branches.match(/^. (\w+)\n/)
- if !m
- raise GitError.new "commit is not on any branch"
- end
- branch = m[1]
- must_git(dst_gitdir,
- "fetch file://#{src_gitdir.shellescape} #{branch.shellescape}")
- # Even if all of the above steps succeeded, we might still not
- # have the right commit due to a race, in which case tag_cmd
- # will fail, and we'll need to fall back to pack|unpack. So
- # don't be tempted to condense this tag_cmd and the one in the
- # rescue block into a single attempt.
- must_git(dst_gitdir, tag_cmd)
- rescue GitError
- must_pipe("echo #{sha1.shellescape}",
- "git --git-dir #{src_gitdir.shellescape} pack-objects -q --revs --stdout",
- "git --git-dir #{dst_gitdir.shellescape} unpack-objects -q")
- must_git(dst_gitdir, tag_cmd)
- end
- end
- end
-
- protected
-
- def self.remote_url? repo_name
- /^(https?|git):\/\// =~ repo_name
- end
-
- # Return [local_git_dir, is_remote]. If is_remote, caller must use
- # fetch_remote_repository to ensure content is up-to-date.
- #
- # Raises an exception if the latest content could not be fetched for
- # any reason.
- def self.git_dir_for repo_name
- if remote_url? repo_name
- return [cache_dir_for(repo_name), true]
- end
- repos = Repository.readable_by(current_user).where(name: repo_name)
- if repos.count == 0
- raise ArgumentError.new "Repository not found: '#{repo_name}'"
- elsif repos.count > 1
- logger.error "Multiple repositories with name=='#{repo_name}'!"
- raise ArgumentError.new "Name conflict"
- else
- return [repos.first.server_path, false]
- end
- end
-
- def self.cache_dir_for git_url
- File.join(cache_dir_base, Digest::SHA1.hexdigest(git_url) + ".git").to_s
- end
-
- def self.cache_dir_base
- Rails.root.join 'tmp', 'git-cache'
- end
-
- def self.fetch_remote_repository gitdir, git_url
- # Caller decides which protocols are worth using. This is just a
- # safety check to ensure we never use urls like "--flag" or wander
- # into git's hardlink features by using bare "/path/foo" instead
- # of "file:///path/foo".
- unless /^[a-z]+:\/\// =~ git_url
- raise ArgumentError.new "invalid git url #{git_url}"
- end
- begin
- must_git gitdir, "branch"
- rescue GitError => e
- raise unless /Not a git repository/i =~ e.to_s
- # OK, this just means we need to create a blank cache repository
- # before fetching.
- FileUtils.mkdir_p gitdir
- must_git gitdir, "init"
- end
- must_git(gitdir,
- "fetch --no-progress --tags --prune --force --update-head-ok #{git_url.shellescape} 'refs/heads/*:refs/heads/*'")
- end
-
- def self.must_git gitdir, *cmds
- # Clear token in case a git helper tries to use it as a password.
- orig_token = ENV['ARVADOS_API_TOKEN']
- ENV['ARVADOS_API_TOKEN'] = ''
- last_output = ''
- begin
- git = "git --git-dir #{gitdir.shellescape}"
- cmds.each do |cmd|
- last_output = must_pipe git+" "+cmd
- end
- ensure
- ENV['ARVADOS_API_TOKEN'] = orig_token
- end
- return last_output
- end
-
- def self.must_pipe *cmds
- cmd = cmds.join(" 2>&1 |") + " 2>&1"
- out = IO.read("| </dev/null #{cmd}")
- if not $?.success?
- raise GitError.new "#{cmd}: #{$?}: #{out}"
- end
- return out
- end
-end
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index c13d48bd8..30a0294a3 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -1,6 +1,26 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
+#
+#
+# Legacy jobs API aka crunch v1
+#
+# This is superceded by containers / container_requests (aka crunch v2)
+#
+# Arvados installations since the beginning of 2018 should have never
+# used jobs, and are unaffected by this change.
+#
+# So that older Arvados sites don't lose access to legacy records, the
+# API has been converted to read-only. Creating and updating jobs
+# (and related types job_task, pipeline_template and
+# pipeline_instance) is disabled and much of the business logic
+# related has been removed, along with the crunch-dispatch.rb and
+# various other code specific to the jobs API.
+#
+# If you need to resurrect any of this code, here is the last commit
+# on master before the branch removing jobs API support:
+#
+# Wed Aug 7 14:49:38 2019 -0400 07d92519438a592d531f2c7558cd51788da262ca
require 'log_reuse_info'
require 'safe_json'
@@ -189,7 +209,7 @@ class Job < ArvadosModel
else
raise ArgumentError.new("unknown attribute for git filter: #{attr}")
end
- revisions = Commit.find_commit_range(filter["repository"],
+ revisions = CommitsHelper::find_commit_range(filter["repository"],
filter["min_version"],
filter["max_version"],
filter["exclude_versions"])
@@ -209,7 +229,7 @@ class Job < ArvadosModel
# Add a filter to @filters for `attr_name` = the latest commit available
# in `repo_name` at `refspec`. No filter is added if refspec can't be
# resolved.
- commits = Commit.find_commit_range(repo_name, nil, refspec, nil)
+ commits = CommitsHelper::find_commit_range(repo_name, nil, refspec, nil)
if commit_hash = commits.first
[[attr_name, "=", commit_hash]]
else
@@ -218,36 +238,7 @@ class Job < ArvadosModel
end
def cancel(cascade: false, need_transaction: true)
- if need_transaction
- ActiveRecord::Base.transaction do
- cancel(cascade: cascade, need_transaction: false)
- end
- return
- end
-
- if self.state.in?([Queued, Running])
- self.state = Cancelled
- self.save!
- elsif self.state != Cancelled
- raise InvalidStateTransitionError
- end
-
- return if !cascade
-
- # cancel all children; they could be jobs or pipeline instances
- children = self.components.andand.collect{|_, u| u}.compact
-
- return if children.empty?
-
- # cancel any child jobs
- Job.where(uuid: children, state: [Queued, Running]).each do |job|
- job.cancel(cascade: cascade, need_transaction: false)
- end
-
- # cancel any child pipelines
- PipelineInstance.where(uuid: children, state: [PipelineInstance::RunningOnServer, PipelineInstance::RunningOnClient]).each do |pi|
- pi.cancel(cascade: cascade, need_transaction: false)
- end
+ raise "No longer supported"
end
protected
@@ -283,7 +274,7 @@ class Job < ArvadosModel
return true
end
if new_record? or repository_changed? or script_version_changed?
- sha1 = Commit.find_commit_range(repository,
+ sha1 = CommitsHelper::find_commit_range(repository,
nil, script_version, nil).first
if not sha1
errors.add :script_version, "#{script_version} does not resolve to a commit"
@@ -308,7 +299,7 @@ class Job < ArvadosModel
uuid_was = uuid
begin
assign_uuid
- Commit.tag_in_internal_repository repository, script_version, uuid
+ CommitsHelper::tag_in_internal_repository repository, script_version, uuid
rescue
self.uuid = uuid_was
raise
@@ -343,7 +334,7 @@ class Job < ArvadosModel
def find_arvados_sdk_version
resolve_runtime_constraint("arvados_sdk_version",
:arvados_sdk_version) do |git_search|
- commits = Commit.find_commit_range("arvados",
+ commits = CommitsHelper::find_commit_range("arvados",
nil, git_search, nil)
if commits.empty?
[false, "#{git_search} does not resolve to a commit"]
diff --git a/services/api/db/migrate/20190809135453_remove_commits_table.rb b/services/api/db/migrate/20190809135453_remove_commits_table.rb
new file mode 100644
index 000000000..a3f450e98
--- /dev/null
+++ b/services/api/db/migrate/20190809135453_remove_commits_table.rb
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class RemoveCommitsTable < ActiveRecord::Migration[5.0]
+ def change
+ drop_table :commits
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 080990a35..889ffa748 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -228,39 +228,6 @@ ALTER SEQUENCE public.collections_id_seq OWNED BY public.collections.id;
--
--- Name: commits; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.commits (
- id integer NOT NULL,
- repository_name character varying(255),
- sha1 character varying(255),
- message character varying(255),
- created_at timestamp without time zone NOT NULL,
- updated_at timestamp without time zone NOT NULL
-);
-
-
---
--- Name: commits_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.commits_id_seq
- START WITH 1
- INCREMENT BY 1
- NO MINVALUE
- NO MAXVALUE
- CACHE 1;
-
-
---
--- Name: commits_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.commits_id_seq OWNED BY public.commits.id;
-
-
---
-- Name: container_requests; Type: TABLE; Schema: public; Owner: -
--
@@ -1234,13 +1201,6 @@ ALTER TABLE ONLY public.collections ALTER COLUMN id SET DEFAULT nextval('public.
--
--- Name: commits id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.commits ALTER COLUMN id SET DEFAULT nextval('public.commits_id_seq'::regclass);
-
-
---
-- Name: container_requests id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -1421,14 +1381,6 @@ ALTER TABLE ONLY public.collections
--
--- Name: commits commits_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.commits
- ADD CONSTRAINT commits_pkey PRIMARY KEY (id);
-
-
---
-- Name: container_requests container_requests_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -1869,13 +1821,6 @@ CREATE UNIQUE INDEX index_collections_on_uuid ON public.collections USING btree
--
--- Name: index_commits_on_repository_name_and_sha1; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE UNIQUE INDEX index_commits_on_repository_name_and_sha1 ON public.commits USING btree (repository_name, sha1);
-
-
---
-- Name: index_container_requests_on_container_uuid; Type: INDEX; Schema: public; Owner: -
--
@@ -3070,6 +3015,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20190322174136'),
('20190422144631'),
('20190523180148'),
-('20190808145904');
+('20190808145904'),
+('20190809135453');
diff --git a/services/api/test/helpers/git_test_helper.rb b/services/api/test/helpers/git_test_helper.rb
index 03189bdfe..59debc576 100644
--- a/services/api/test/helpers/git_test_helper.rb
+++ b/services/api/test/helpers/git_test_helper.rb
@@ -30,7 +30,7 @@ module GitTestHelper
end
base.teardown do
- FileUtils.remove_entry Commit.cache_dir_base, true
+ FileUtils.remove_entry CommitsHelper.cache_dir_base, true
FileUtils.mkdir_p @tmpdir
system("tar", "-xC", @tmpdir.to_s, "-f", "test/test.git.tar")
end
@@ -48,10 +48,10 @@ module GitTestHelper
if fakeurl.is_a? Symbol
fakeurl = 'file://' + repositories(fakeurl).server_path
end
- Commit.expects(:fetch_remote_repository).once.with do |gitdir, giturl|
+ CommitsHelper.expects(:fetch_remote_repository).once.with do |gitdir, giturl|
if giturl == url
- Commit.unstub(:fetch_remote_repository)
- Commit.fetch_remote_repository gitdir, fakeurl
+ CommitsHelper.unstub(:fetch_remote_repository)
+ CommitsHelper.fetch_remote_repository gitdir, fakeurl
true
end
end
diff --git a/services/api/test/unit/commit_test.rb b/services/api/test/unit/commit_test.rb
index c5d72c3bf..1c772de04 100644
--- a/services/api/test/unit/commit_test.rb
+++ b/services/api/test/unit/commit_test.rb
@@ -22,7 +22,7 @@ class CommitTest < ActiveSupport::TestCase
test 'find_commit_range does not bypass permissions' do
authorize_with :inactive
assert_raises ArgumentError do
- Commit.find_commit_range 'foo', nil, 'master', []
+ CommitsHelper::find_commit_range 'foo', nil, 'master', []
end
end
@@ -41,9 +41,9 @@ class CommitTest < ActiveSupport::TestCase
].each do |url|
test "find_commit_range uses fetch_remote_repository to get #{url}" do
fake_gitdir = repositories(:foo).server_path
- Commit.expects(:cache_dir_for).once.with(url).returns fake_gitdir
- Commit.expects(:fetch_remote_repository).once.with(fake_gitdir, url).returns true
- c = Commit.find_commit_range url, nil, 'master', []
+ CommitsHelper::expects(:cache_dir_for).once.with(url).returns fake_gitdir
+ CommitsHelper::expects(:fetch_remote_repository).once.with(fake_gitdir, url).returns true
+ c = CommitsHelper::find_commit_range url, nil, 'master', []
refute_empty c
end
end
@@ -57,9 +57,9 @@ class CommitTest < ActiveSupport::TestCase
'github.com/curoverse/arvados.git',
].each do |url|
test "find_commit_range skips fetch_remote_repository for #{url}" do
- Commit.expects(:fetch_remote_repository).never
+ CommitsHelper::expects(:fetch_remote_repository).never
assert_raises ArgumentError do
- Commit.find_commit_range url, nil, 'master', []
+ CommitsHelper::find_commit_range url, nil, 'master', []
end
end
end
@@ -67,12 +67,12 @@ class CommitTest < ActiveSupport::TestCase
test 'fetch_remote_repository does not leak commits across repositories' do
url = "http://localhost:1/fake/fake.git"
fetch_remote_from_local_repo url, :foo
- c = Commit.find_commit_range url, nil, 'master', []
+ c = CommitsHelper::find_commit_range url, nil, 'master', []
assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57'], c
url = "http://localhost:2/fake/fake.git"
fetch_remote_from_local_repo url, 'file://' + File.expand_path('../../.git', Rails.root)
- c = Commit.find_commit_range url, nil, '077ba2ad3ea24a929091a9e6ce545c93199b8e57', []
+ c = CommitsHelper::find_commit_range url, nil, '077ba2ad3ea24a929091a9e6ce545c93199b8e57', []
assert_equal [], c
end
@@ -82,7 +82,7 @@ class CommitTest < ActiveSupport::TestCase
IO.read("|#{gitint} tag -d testtag 2>/dev/null") # "no such tag", fine
assert_match(/^fatal: /, IO.read("|#{gitint} show testtag 2>&1"))
refute $?.success?
- Commit.tag_in_internal_repository 'active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', 'testtag'
+ CommitsHelper::tag_in_internal_repository 'active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', 'testtag'
assert_match(/^commit 31ce37f/, IO.read("|#{gitint} show testtag"))
assert $?.success?
end
@@ -106,7 +106,7 @@ class CommitTest < ActiveSupport::TestCase
must_pipe("git rm bar")
must_pipe("git -c user.email=x at x -c user.name=X commit -m -")
end
- Commit.tag_in_internal_repository 'active/foo', sha1, tag
+ CommitsHelper::tag_in_internal_repository 'active/foo', sha1, tag
gitint = "git --git-dir #{Rails.configuration.Containers.JobsAPI.GitInternalDir.shellescape}"
assert_match(/^commit /, IO.read("|#{gitint} show #{tag.shellescape}"))
assert $?.success?
@@ -122,7 +122,7 @@ class CommitTest < ActiveSupport::TestCase
sha1 = must_pipe("git log -n1 --format=%H").strip
must_pipe("git reset --hard HEAD^")
end
- Commit.tag_in_internal_repository 'active/foo', sha1, tag
+ CommitsHelper::tag_in_internal_repository 'active/foo', sha1, tag
gitint = "git --git-dir #{Rails.configuration.Containers.JobsAPI.GitInternalDir.shellescape}"
assert_match(/^commit /, IO.read("|#{gitint} show #{tag.shellescape}"))
assert $?.success?
@@ -141,50 +141,50 @@ class CommitTest < ActiveSupport::TestCase
test "find_commit_range min_version prefers commits over branch names" do
assert_equal([COMMIT_BRANCH_NAME],
- Commit.find_commit_range("active/shabranchnames",
+ CommitsHelper::find_commit_range("active/shabranchnames",
COMMIT_BRANCH_NAME, nil, nil))
end
test "find_commit_range max_version prefers commits over branch names" do
assert_equal([COMMIT_BRANCH_NAME],
- Commit.find_commit_range("active/shabranchnames",
+ CommitsHelper::find_commit_range("active/shabranchnames",
nil, COMMIT_BRANCH_NAME, nil))
end
test "find_commit_range min_version with short branch name" do
assert_equal([SHORT_BRANCH_COMMIT_2],
- Commit.find_commit_range("active/shabranchnames",
+ CommitsHelper::find_commit_range("active/shabranchnames",
SHORT_COMMIT_BRANCH_NAME, nil, nil))
end
test "find_commit_range max_version with short branch name" do
assert_equal([SHORT_BRANCH_COMMIT_2],
- Commit.find_commit_range("active/shabranchnames",
+ CommitsHelper::find_commit_range("active/shabranchnames",
nil, SHORT_COMMIT_BRANCH_NAME, nil))
end
test "find_commit_range min_version with disambiguated branch name" do
assert_equal([COMMIT_BRANCH_COMMIT_2],
- Commit.find_commit_range("active/shabranchnames",
+ CommitsHelper::find_commit_range("active/shabranchnames",
"heads/#{COMMIT_BRANCH_NAME}",
nil, nil))
end
test "find_commit_range max_version with disambiguated branch name" do
assert_equal([COMMIT_BRANCH_COMMIT_2],
- Commit.find_commit_range("active/shabranchnames", nil,
+ CommitsHelper::find_commit_range("active/shabranchnames", nil,
"heads/#{COMMIT_BRANCH_NAME}", nil))
end
test "find_commit_range min_version with unambiguous short name" do
assert_equal([COMMIT_BRANCH_NAME],
- Commit.find_commit_range("active/shabranchnames",
+ CommitsHelper::find_commit_range("active/shabranchnames",
COMMIT_BRANCH_NAME[0..-2], nil, nil))
end
test "find_commit_range max_version with unambiguous short name" do
assert_equal([COMMIT_BRANCH_NAME],
- Commit.find_commit_range("active/shabranchnames", nil,
+ CommitsHelper::find_commit_range("active/shabranchnames", nil,
COMMIT_BRANCH_NAME[0..-2], nil))
end
@@ -192,77 +192,77 @@ class CommitTest < ActiveSupport::TestCase
authorize_with :active
# single
- a = Commit.find_commit_range('active/foo', nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ a = CommitsHelper::find_commit_range('active/foo', nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
assert_equal ['31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
#test "test_branch1" do
- a = Commit.find_commit_range('active/foo', nil, 'master', nil)
+ a = CommitsHelper::find_commit_range('active/foo', nil, 'master', nil)
assert_includes(a, '077ba2ad3ea24a929091a9e6ce545c93199b8e57')
#test "test_branch2" do
- a = Commit.find_commit_range('active/foo', nil, 'b1', nil)
+ a = CommitsHelper::find_commit_range('active/foo', nil, 'b1', nil)
assert_equal ['1de84a854e2b440dc53bf42f8548afa4c17da332'], a
#test "test_branch3" do
- a = Commit.find_commit_range('active/foo', nil, 'HEAD', nil)
+ a = CommitsHelper::find_commit_range('active/foo', nil, 'HEAD', nil)
assert_equal ['1de84a854e2b440dc53bf42f8548afa4c17da332'], a
#test "test_single_revision_repo" do
- a = Commit.find_commit_range('active/foo', nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ a = CommitsHelper::find_commit_range('active/foo', nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
assert_equal ['31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
- a = Commit.find_commit_range('arvados', nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ a = CommitsHelper::find_commit_range('arvados', nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
assert_equal [], a
#test "test_multi_revision" do
# complains "fatal: bad object 077ba2ad3ea24a929091a9e6ce545c93199b8e57"
- a = Commit.find_commit_range('active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', nil)
+ a = CommitsHelper::find_commit_range('active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', nil)
assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '4fe459abe02d9b365932b8f5dc419439ab4e2577', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
#test "test_tag" do
# complains "fatal: ambiguous argument 'tag1': unknown revision or path
# not in the working tree."
- a = Commit.find_commit_range('active/foo', 'tag1', 'master', nil)
+ a = CommitsHelper::find_commit_range('active/foo', 'tag1', 'master', nil)
assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '4fe459abe02d9b365932b8f5dc419439ab4e2577'], a
#test "test_multi_revision_exclude" do
- a = Commit.find_commit_range('active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['4fe459abe02d9b365932b8f5dc419439ab4e2577'])
+ a = CommitsHelper::find_commit_range('active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['4fe459abe02d9b365932b8f5dc419439ab4e2577'])
assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
#test "test_multi_revision_tagged_exclude" do
# complains "fatal: bad object 077ba2ad3ea24a929091a9e6ce545c93199b8e57"
- a = Commit.find_commit_range('active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['tag1'])
+ a = CommitsHelper::find_commit_range('active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['tag1'])
assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
Dir.mktmpdir do |touchdir|
# invalid input to maximum
- a = Commit.find_commit_range('active/foo', nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", nil)
+ a = CommitsHelper::find_commit_range('active/foo', nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", nil)
assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable"
assert_equal [], a
# invalid input to maximum
- a = Commit.find_commit_range('active/foo', nil, "$(uname>#{touchdir}/uh_oh)", nil)
+ a = CommitsHelper::find_commit_range('active/foo', nil, "$(uname>#{touchdir}/uh_oh)", nil)
assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable"
assert_equal [], a
# invalid input to minimum
- a = Commit.find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
+ a = CommitsHelper::find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable"
assert_equal [], a
# invalid input to minimum
- a = Commit.find_commit_range('active/foo', "$(uname>#{touchdir}/uh_oh)", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
+ a = CommitsHelper::find_commit_range('active/foo', "$(uname>#{touchdir}/uh_oh)", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable"
assert_equal [], a
# invalid input to 'excludes'
# complains "fatal: bad object 077ba2ad3ea24a929091a9e6ce545c93199b8e57"
- a = Commit.find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["4fe459abe02d9b365932b8f5dc419439ab4e2577 ; touch #{touchdir}/uh_oh"])
+ a = CommitsHelper::find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["4fe459abe02d9b365932b8f5dc419439ab4e2577 ; touch #{touchdir}/uh_oh"])
assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable"
assert_equal [], a
# invalid input to 'excludes'
# complains "fatal: bad object 077ba2ad3ea24a929091a9e6ce545c93199b8e57"
- a = Commit.find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["$(uname>#{touchdir}/uh_oh)"])
+ a = CommitsHelper::find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["$(uname>#{touchdir}/uh_oh)"])
assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable"
assert_equal [], a
end
diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb
index 5ece9ab49..b20120635 100644
--- a/services/api/test/unit/job_test.rb
+++ b/services/api/test/unit/job_test.rb
@@ -589,53 +589,6 @@ class JobTest < ActiveSupport::TestCase
assert_equal foobar.uuid, j.uuid
end
- [
- true,
- false,
- ].each do |cascade|
- test "cancel job with cascade #{cascade}" do
- job = Job.find_by_uuid jobs(:running_job_with_components_at_level_1).uuid
- job.cancel cascade: cascade
- assert_equal Job::Cancelled, job.state
-
- descendents = ['zzzzz-8i9sb-jobcomponentsl2',
- 'zzzzz-d1hrv-picomponentsl02',
- 'zzzzz-8i9sb-job1atlevel3noc',
- 'zzzzz-8i9sb-job2atlevel3noc']
-
- jobs = Job.where(uuid: descendents)
- jobs.each do |j|
- assert_equal ('Cancelled' == j.state), cascade
- end
-
- pipelines = PipelineInstance.where(uuid: descendents)
- pipelines.each do |pi|
- assert_equal ('Paused' == pi.state), cascade
- end
- end
- end
-
- test 'cancelling a completed job raises error' do
- job = Job.find_by_uuid jobs(:job_with_latest_version).uuid
- assert job
- assert_equal 'Complete', job.state
-
- assert_raises(ArvadosModel::InvalidStateTransitionError) do
- job.cancel
- end
- end
-
- test 'cancelling a job with circular relationship with another does not result in an infinite loop' do
- job = Job.find_by_uuid jobs(:running_job_2_with_circular_component_relationship).uuid
-
- job.cancel cascade: true
-
- assert_equal Job::Cancelled, job.state
-
- child = Job.find_by_uuid job.components.collect{|_, uuid| uuid}[0]
- assert_equal Job::Cancelled, child.state
- end
-
test 'enable legacy api configuration option = true' do
Rails.configuration.Containers.JobsAPI.Enable = "true"
check_enable_legacy_jobs_api
commit e10aa8e0c8b9c45d69832e71480cfb3d6929834e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 9 09:17:19 2019 -0400
15133: Delete crunchrunner
Not to be confused with crunch-run, this is only used to run stuff in
crunch v1.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/go/crunchrunner/crunchrunner.go b/sdk/go/crunchrunner/crunchrunner.go
deleted file mode 100644
index ca16fc656..000000000
--- a/sdk/go/crunchrunner/crunchrunner.go
+++ /dev/null
@@ -1,439 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package main
-
-import (
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "os"
- "os/exec"
- "os/signal"
- "strings"
- "syscall"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
-)
-
-type TaskDef struct {
- Command []string `json:"command"`
- Env map[string]string `json:"task.env"`
- Stdin string `json:"task.stdin"`
- Stdout string `json:"task.stdout"`
- Stderr string `json:"task.stderr"`
- Vwd map[string]string `json:"task.vwd"`
- SuccessCodes []int `json:"task.successCodes"`
- PermanentFailCodes []int `json:"task.permanentFailCodes"`
- TemporaryFailCodes []int `json:"task.temporaryFailCodes"`
- KeepTmpOutput bool `json:"task.keepTmpOutput"`
-}
-
-type Tasks struct {
- Tasks []TaskDef `json:"tasks"`
-}
-
-type Job struct {
- ScriptParameters Tasks `json:"script_parameters"`
-}
-
-type Task struct {
- JobUUID string `json:"job_uuid"`
- CreatedByJobTaskUUID string `json:"created_by_job_task_uuid"`
- Parameters TaskDef `json:"parameters"`
- Sequence int `json:"sequence"`
- Output string `json:"output"`
- Success bool `json:"success"`
- Progress float32 `json:"sequence"`
-}
-
-type IArvadosClient interface {
- Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error
- Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error)
-}
-
-func setupDirectories(crunchtmpdir, taskUUID string, keepTmp bool) (tmpdir, outdir string, err error) {
- tmpdir = crunchtmpdir + "/tmpdir"
- err = os.Mkdir(tmpdir, 0700)
- if err != nil {
- return "", "", err
- }
-
- if keepTmp {
- outdir = os.Getenv("TASK_KEEPMOUNT_TMP")
- } else {
- outdir = crunchtmpdir + "/outdir"
- err = os.Mkdir(outdir, 0700)
- if err != nil {
- return "", "", err
- }
- }
-
- return tmpdir, outdir, nil
-}
-
-func checkOutputFilename(outdir, fn string) error {
- if strings.HasPrefix(fn, "/") || strings.HasSuffix(fn, "/") {
- return fmt.Errorf("Path must not start or end with '/'")
- }
- if strings.Index("../", fn) != -1 {
- return fmt.Errorf("Path must not contain '../'")
- }
-
- sl := strings.LastIndex(fn, "/")
- if sl != -1 {
- os.MkdirAll(outdir+"/"+fn[0:sl], 0777)
- }
- return nil
-}
-
-func copyFile(dst, src string) error {
- in, err := os.Open(src)
- if err != nil {
- return err
- }
- defer in.Close()
-
- out, err := os.Create(dst)
- if err != nil {
- return err
- }
- defer out.Close()
-
- _, err = io.Copy(out, in)
- return err
-}
-
-func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[string]string) (stdin, stdout, stderr string, err error) {
- if taskp.Vwd != nil {
- for k, v := range taskp.Vwd {
- v = substitute(v, replacements)
- err = checkOutputFilename(outdir, k)
- if err != nil {
- return "", "", "", err
- }
- if taskp.KeepTmpOutput {
- err = copyFile(v, outdir+"/"+k)
- } else {
- err = os.Symlink(v, outdir+"/"+k)
- }
- if err != nil {
- return "", "", "", err
- }
- }
- }
-
- if taskp.Stdin != "" {
- // Set up stdin redirection
- stdin = substitute(taskp.Stdin, replacements)
- cmd.Stdin, err = os.Open(stdin)
- if err != nil {
- return "", "", "", err
- }
- }
-
- if taskp.Stdout != "" {
- err = checkOutputFilename(outdir, taskp.Stdout)
- if err != nil {
- return "", "", "", err
- }
- // Set up stdout redirection
- stdout = outdir + "/" + taskp.Stdout
- cmd.Stdout, err = os.Create(stdout)
- if err != nil {
- return "", "", "", err
- }
- } else {
- cmd.Stdout = os.Stdout
- }
-
- if taskp.Stderr != "" {
- err = checkOutputFilename(outdir, taskp.Stderr)
- if err != nil {
- return "", "", "", err
- }
- // Set up stderr redirection
- stderr = outdir + "/" + taskp.Stderr
- cmd.Stderr, err = os.Create(stderr)
- if err != nil {
- return "", "", "", err
- }
- } else {
- cmd.Stderr = os.Stderr
- }
-
- if taskp.Env != nil {
- // Set up subprocess environment
- cmd.Env = os.Environ()
- for k, v := range taskp.Env {
- v = substitute(v, replacements)
- cmd.Env = append(cmd.Env, k+"="+v)
- }
- }
- return stdin, stdout, stderr, nil
-}
-
-// Set up signal handlers. Go sends signal notifications to a "signal
-// channel".
-func setupSignals(cmd *exec.Cmd) chan os.Signal {
- sigChan := make(chan os.Signal, 1)
- signal.Notify(sigChan, syscall.SIGTERM)
- signal.Notify(sigChan, syscall.SIGINT)
- signal.Notify(sigChan, syscall.SIGQUIT)
- return sigChan
-}
-
-func inCodes(code int, codes []int) bool {
- if codes != nil {
- for _, c := range codes {
- if code == c {
- return true
- }
- }
- }
- return false
-}
-
-const TASK_TEMPFAIL = 111
-
-type TempFail struct{ error }
-type PermFail struct{}
-
-func (s PermFail) Error() string {
- return "PermFail"
-}
-
-func substitute(inp string, subst map[string]string) string {
- for k, v := range subst {
- inp = strings.Replace(inp, k, v, -1)
- }
- return inp
-}
-
-func getKeepTmp(outdir string) (manifest string, err error) {
- fn, err := os.Open(outdir + "/" + ".arvados#collection")
- if err != nil {
- return "", err
- }
- defer fn.Close()
-
- buf, err := ioutil.ReadAll(fn)
- if err != nil {
- return "", err
- }
- collection := arvados.Collection{}
- err = json.Unmarshal(buf, &collection)
- return collection.ManifestText, err
-}
-
-func runner(api IArvadosClient,
- kc IKeepClient,
- jobUUID, taskUUID, crunchtmpdir, keepmount string,
- jobStruct Job, taskStruct Task) error {
-
- var err error
- taskp := taskStruct.Parameters
-
- // If this is task 0 and there are multiple tasks, dispatch subtasks
- // and exit.
- if taskStruct.Sequence == 0 {
- if len(jobStruct.ScriptParameters.Tasks) == 1 {
- taskp = jobStruct.ScriptParameters.Tasks[0]
- } else {
- for _, task := range jobStruct.ScriptParameters.Tasks {
- err := api.Create("job_tasks",
- map[string]interface{}{
- "job_task": Task{
- JobUUID: jobUUID,
- CreatedByJobTaskUUID: taskUUID,
- Sequence: 1,
- Parameters: task}},
- nil)
- if err != nil {
- return TempFail{err}
- }
- }
- err = api.Update("job_tasks", taskUUID,
- map[string]interface{}{
- "job_task": map[string]interface{}{
- "output": "",
- "success": true,
- "progress": 1.0}},
- nil)
- return nil
- }
- }
-
- var tmpdir, outdir string
- tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUUID, taskp.KeepTmpOutput)
- if err != nil {
- return TempFail{err}
- }
-
- replacements := map[string]string{
- "$(task.tmpdir)": tmpdir,
- "$(task.outdir)": outdir,
- "$(task.keep)": keepmount}
-
- log.Printf("crunchrunner: $(task.tmpdir)=%v", tmpdir)
- log.Printf("crunchrunner: $(task.outdir)=%v", outdir)
- log.Printf("crunchrunner: $(task.keep)=%v", keepmount)
-
- // Set up subprocess
- for k, v := range taskp.Command {
- taskp.Command[k] = substitute(v, replacements)
- }
-
- cmd := exec.Command(taskp.Command[0], taskp.Command[1:]...)
-
- cmd.Dir = outdir
-
- var stdin, stdout, stderr string
- stdin, stdout, stderr, err = setupCommand(cmd, taskp, outdir, replacements)
- if err != nil {
- return err
- }
-
- // Run subprocess and wait for it to complete
- if stdin != "" {
- stdin = " < " + stdin
- }
- if stdout != "" {
- stdout = " > " + stdout
- }
- if stderr != "" {
- stderr = " 2> " + stderr
- }
- log.Printf("Running %v%v%v%v", cmd.Args, stdin, stdout, stderr)
-
- var caughtSignal os.Signal
- sigChan := setupSignals(cmd)
-
- err = cmd.Start()
- if err != nil {
- signal.Stop(sigChan)
- return TempFail{err}
- }
-
- finishedSignalNotify := make(chan struct{})
- go func(sig <-chan os.Signal) {
- for sig := range sig {
- caughtSignal = sig
- cmd.Process.Signal(caughtSignal)
- }
- close(finishedSignalNotify)
- }(sigChan)
-
- err = cmd.Wait()
- signal.Stop(sigChan)
-
- close(sigChan)
- <-finishedSignalNotify
-
- if caughtSignal != nil {
- log.Printf("Caught signal %v", caughtSignal)
- return PermFail{}
- }
-
- if err != nil {
- // Run() returns ExitError on non-zero exit code, but we handle
- // that down below. So only return if it's not ExitError.
- if _, ok := err.(*exec.ExitError); !ok {
- return TempFail{err}
- }
- }
-
- var success bool
-
- exitCode := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
-
- log.Printf("Completed with exit code %v", exitCode)
-
- if inCodes(exitCode, taskp.PermanentFailCodes) {
- success = false
- } else if inCodes(exitCode, taskp.TemporaryFailCodes) {
- return TempFail{fmt.Errorf("Process tempfail with exit code %v", exitCode)}
- } else if inCodes(exitCode, taskp.SuccessCodes) || cmd.ProcessState.Success() {
- success = true
- } else {
- success = false
- }
-
- // Upload output directory
- var manifest string
- if taskp.KeepTmpOutput {
- manifest, err = getKeepTmp(outdir)
- } else {
- manifest, err = WriteTree(kc, outdir)
- }
- if err != nil {
- return TempFail{err}
- }
-
- // Set status
- err = api.Update("job_tasks", taskUUID,
- map[string]interface{}{
- "job_task": Task{
- Output: manifest,
- Success: success,
- Progress: 1}},
- nil)
- if err != nil {
- return TempFail{err}
- }
-
- if success {
- return nil
- } else {
- return PermFail{}
- }
-}
-
-func main() {
- api, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Fatal(err)
- }
-
- jobUUID := os.Getenv("JOB_UUID")
- taskUUID := os.Getenv("TASK_UUID")
- tmpdir := os.Getenv("TASK_WORK")
- keepmount := os.Getenv("TASK_KEEPMOUNT")
-
- var jobStruct Job
- var taskStruct Task
-
- err = api.Get("jobs", jobUUID, nil, &jobStruct)
- if err != nil {
- log.Fatal(err)
- }
- err = api.Get("job_tasks", taskUUID, nil, &taskStruct)
- if err != nil {
- log.Fatal(err)
- }
-
- var kc IKeepClient
- kc, err = keepclient.MakeKeepClient(api)
- if err != nil {
- log.Fatal(err)
- }
-
- syscall.Umask(0022)
- err = runner(api, kc, jobUUID, taskUUID, tmpdir, keepmount, jobStruct, taskStruct)
-
- if err == nil {
- os.Exit(0)
- } else if _, ok := err.(TempFail); ok {
- log.Print(err)
- os.Exit(TASK_TEMPFAIL)
- } else if _, ok := err.(PermFail); ok {
- os.Exit(1)
- } else {
- log.Fatal(err)
- }
-}
diff --git a/sdk/go/crunchrunner/crunchrunner_test.go b/sdk/go/crunchrunner/crunchrunner_test.go
deleted file mode 100644
index f2827c6a3..000000000
--- a/sdk/go/crunchrunner/crunchrunner_test.go
+++ /dev/null
@@ -1,478 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package main
-
-import (
- "io"
- "io/ioutil"
- "log"
- "os"
- "syscall"
- "testing"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- . "gopkg.in/check.v1"
-)
-
-// Gocheck boilerplate
-func Test(t *testing.T) {
- TestingT(t)
-}
-
-type TestSuite struct{}
-
-// Gocheck boilerplate
-var _ = Suite(&TestSuite{})
-
-type ArvTestClient struct {
- c *C
- manifest string
- success bool
-}
-
-func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
- return nil
-}
-
-func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
- t.c.Check(resourceType, Equals, "job_tasks")
- t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
- Output: t.manifest,
- Success: t.success,
- Progress: 1}})
- return nil
-}
-
-func (s *TestSuite) TestSimpleRun(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, "", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"echo", "foo"}}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
-}
-
-func checkOutput(c *C, tmpdir string) {
- file, err := os.Open(tmpdir + "/outdir/output.txt")
- c.Assert(err, IsNil)
-
- data := make([]byte, 100)
- var count int
- err = nil
- offset := 0
- for err == nil {
- count, err = file.Read(data[offset:])
- offset += count
- }
- c.Assert(err, Equals, io.EOF)
- c.Check(string(data[0:offset]), Equals, "foo\n")
-}
-
-func (s *TestSuite) TestSimpleRunSubtask(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c,
- ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{
- {Command: []string{"echo", "bar"}},
- {Command: []string{"echo", "foo"}}}}},
- Task{Parameters: TaskDef{
- Command: []string{"echo", "foo"},
- Stdout: "output.txt"},
- Sequence: 1})
- c.Check(err, IsNil)
-
- checkOutput(c, tmpdir)
-}
-
-func (s *TestSuite) TestRedirect(c *C) {
- tmpfile, _ := ioutil.TempFile("", "")
- tmpfile.Write([]byte("foo\n"))
- tmpfile.Close()
- defer os.Remove(tmpfile.Name())
-
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c,
- ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"cat"},
- Stdout: "output.txt",
- Stdin: tmpfile.Name()}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
-
- checkOutput(c, tmpdir)
-}
-
-func (s *TestSuite) TestEnv(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"/bin/sh", "-c", "echo $BAR"},
- Stdout: "output.txt",
- Env: map[string]string{"BAR": "foo"}}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
- checkOutput(c, tmpdir)
-}
-
-func (s *TestSuite) TestEnvSubstitute(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "foo\n",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"/bin/sh", "-c", "echo $BAR"},
- Stdout: "output.txt",
- Env: map[string]string{"BAR": "$(task.keep)"}}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
- checkOutput(c, tmpdir)
-}
-
-func (s *TestSuite) TestEnvReplace(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"/bin/sh", "-c", "echo $PATH"},
- Stdout: "output.txt",
- Env: map[string]string{"PATH": "foo"}}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
- checkOutput(c, tmpdir)
-}
-
-type SubtaskTestClient struct {
- c *C
- parms []Task
- i int
-}
-
-func (t *SubtaskTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
- t.c.Check(resourceType, Equals, "job_tasks")
- t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": t.parms[t.i]})
- t.i += 1
- return nil
-}
-
-func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
- return nil
-}
-
-func (s *TestSuite) TestScheduleSubtask(c *C) {
-
- api := SubtaskTestClient{c, []Task{
- {JobUUID: "zzzz-8i9sb-111111111111111",
- CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
- Sequence: 1,
- Parameters: TaskDef{
- Command: []string{"echo", "bar"}}},
- {JobUUID: "zzzz-8i9sb-111111111111111",
- CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
- Sequence: 1,
- Parameters: TaskDef{
- Command: []string{"echo", "foo"}}}},
- 0}
-
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(&api, KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{
- {Command: []string{"echo", "bar"}},
- {Command: []string{"echo", "foo"}}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
-
-}
-
-func (s *TestSuite) TestRunFail(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
- Task{Sequence: 0})
- c.Check(err, FitsTypeOf, PermFail{})
-}
-
-func (s *TestSuite) TestRunSuccessCode(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, "", true}, KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"/bin/sh", "-c", "exit 1"},
- SuccessCodes: []int{0, 1}}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
-}
-
-func (s *TestSuite) TestRunFailCode(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"/bin/sh", "-c", "exit 0"},
- PermanentFailCodes: []int{0, 1}}}}},
- Task{Sequence: 0})
- c.Check(err, FitsTypeOf, PermFail{})
-}
-
-func (s *TestSuite) TestRunTempFailCode(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"/bin/sh", "-c", "exit 1"},
- TemporaryFailCodes: []int{1}}}}},
- Task{Sequence: 0})
- c.Check(err, FitsTypeOf, TempFail{})
-}
-
-func (s *TestSuite) TestVwd(c *C) {
- tmpfile, _ := ioutil.TempFile("", "")
- tmpfile.Write([]byte("foo\n"))
- tmpfile.Close()
- defer os.Remove(tmpfile.Name())
-
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"ls", "output.txt"},
- Vwd: map[string]string{
- "output.txt": tmpfile.Name()}}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
- checkOutput(c, tmpdir)
-}
-
-func (s *TestSuite) TestSubstitutionStdin(c *C) {
- keepmount, _ := ioutil.TempDir("", "")
- ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
- defer func() {
- os.RemoveAll(keepmount)
- }()
-
- log.Print("Keepmount is ", keepmount)
-
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- log.Print("tmpdir is ", tmpdir)
-
- err := runner(ArvTestClient{c,
- ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- keepmount,
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"cat"},
- Stdout: "output.txt",
- Stdin: "$(task.keep)/file1.txt"}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
- checkOutput(c, tmpdir)
-}
-
-func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
- keepmount, _ := ioutil.TempDir("", "")
- ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
- defer func() {
- os.RemoveAll(keepmount)
- }()
-
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c,
- ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- keepmount,
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"cat", "$(task.keep)/file1.txt"},
- Stdout: "output.txt"}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
-
- checkOutput(c, tmpdir)
-}
-
-func (s *TestSuite) TestSignal(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- go func() {
- time.Sleep(1 * time.Second)
- self, _ := os.FindProcess(os.Getpid())
- self.Signal(syscall.SIGINT)
- }()
-
- err := runner(ArvTestClient{c,
- "", false},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"sleep", "4"}}}}},
- Task{Sequence: 0})
- c.Check(err, FitsTypeOf, PermFail{})
-
-}
-
-func (s *TestSuite) TestQuoting(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- err := runner(ArvTestClient{c,
- "./s\\040ub:dir d3b07384d113edec49eaa6238ad5ff00+4 0:4::e\\040vil\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"echo", "foo"},
- Stdout: "s ub:dir/:e vi\nl"}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
-}
-
-func (s *TestSuite) TestKeepTmp(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- os.Setenv("TASK_KEEPMOUNT_TMP", tmpdir)
- defer os.Setenv("TASK_KEEPMOUNT_TMP", "")
-
- fn, err := os.Create(tmpdir + "/.arvados#collection")
- fn.Write([]byte("{\"manifest_text\":\". unparsed 0:3:foo\\n\",\"uuid\":null}"))
- defer fn.Close()
-
- err = runner(ArvTestClient{c,
- ". unparsed 0:3:foo\n", true},
- KeepTestClient{},
- "zzzz-8i9sb-111111111111111",
- "zzzz-ot0gb-111111111111111",
- tmpdir,
- "",
- Job{ScriptParameters: Tasks{[]TaskDef{{
- Command: []string{"echo", "foo"},
- KeepTmpOutput: true}}}},
- Task{Sequence: 0})
- c.Check(err, IsNil)
-
-}
diff --git a/sdk/go/crunchrunner/upload.go b/sdk/go/crunchrunner/upload.go
deleted file mode 100644
index 2848d1087..000000000
--- a/sdk/go/crunchrunner/upload.go
+++ /dev/null
@@ -1,241 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package main
-
-import (
- "bytes"
- "crypto/md5"
- "errors"
- "fmt"
- "io"
- "log"
- "os"
- "path/filepath"
- "sort"
- "strings"
-
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
- "git.curoverse.com/arvados.git/sdk/go/manifest"
-)
-
-type Block struct {
- data []byte
- offset int64
-}
-
-type ManifestStreamWriter struct {
- *ManifestWriter
- *manifest.ManifestStream
- offset int64
- *Block
- uploader chan *Block
- finish chan []error
-}
-
-type IKeepClient interface {
- PutHB(hash string, buf []byte) (string, int, error)
-}
-
-func (m *ManifestStreamWriter) Write(p []byte) (int, error) {
- n, err := m.ReadFrom(bytes.NewReader(p))
- return int(n), err
-}
-
-func (m *ManifestStreamWriter) ReadFrom(r io.Reader) (n int64, err error) {
- var total int64
- var count int
-
- for err == nil {
- if m.Block == nil {
- m.Block = &Block{make([]byte, keepclient.BLOCKSIZE), 0}
- }
- count, err = r.Read(m.Block.data[m.Block.offset:])
- total += int64(count)
- m.Block.offset += int64(count)
- if m.Block.offset == keepclient.BLOCKSIZE {
- m.uploader <- m.Block
- m.Block = nil
- }
- }
-
- if err == io.EOF {
- return total, nil
- } else {
- return total, err
- }
-
-}
-
-func (m *ManifestStreamWriter) goUpload() {
- var errors []error
- uploader := m.uploader
- finish := m.finish
- for block := range uploader {
- hash := fmt.Sprintf("%x", md5.Sum(block.data[0:block.offset]))
- signedHash, _, err := m.ManifestWriter.IKeepClient.PutHB(hash, block.data[0:block.offset])
- if err != nil {
- errors = append(errors, err)
- } else {
- m.ManifestStream.Blocks = append(m.ManifestStream.Blocks, signedHash)
- }
- }
- finish <- errors
-}
-
-type ManifestWriter struct {
- IKeepClient
- stripPrefix string
- Streams map[string]*ManifestStreamWriter
-}
-
-func (m *ManifestWriter) WalkFunc(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
-
- targetPath, targetInfo := path, info
- if info.Mode()&os.ModeSymlink != 0 {
- // Update targetpath/info to reflect the symlink
- // target, not the symlink itself
- targetPath, err = filepath.EvalSymlinks(path)
- if err != nil {
- return err
- }
- targetInfo, err = os.Stat(targetPath)
- if err != nil {
- return fmt.Errorf("stat symlink %q target %q: %s", path, targetPath, err)
- }
- }
-
- if targetInfo.Mode()&os.ModeType != 0 {
- // Skip directories, pipes, other non-regular files
- return nil
- }
-
- var dir string
- if len(path) > (len(m.stripPrefix) + len(info.Name()) + 1) {
- dir = path[len(m.stripPrefix)+1 : (len(path) - len(info.Name()) - 1)]
- }
- if dir == "" {
- dir = "."
- }
-
- fn := path[(len(path) - len(info.Name())):]
-
- if m.Streams[dir] == nil {
- m.Streams[dir] = &ManifestStreamWriter{
- m,
- &manifest.ManifestStream{StreamName: dir},
- 0,
- nil,
- make(chan *Block),
- make(chan []error)}
- go m.Streams[dir].goUpload()
- }
-
- stream := m.Streams[dir]
-
- fileStart := stream.offset
-
- file, err := os.Open(path)
- if err != nil {
- return err
- }
-
- log.Printf("Uploading %v/%v (%v bytes)", dir, fn, info.Size())
-
- var count int64
- count, err = io.Copy(stream, file)
- if err != nil {
- return err
- }
-
- stream.offset += count
-
- stream.ManifestStream.FileStreamSegments = append(stream.ManifestStream.FileStreamSegments,
- manifest.FileStreamSegment{uint64(fileStart), uint64(count), fn})
-
- return nil
-}
-
-func (m *ManifestWriter) Finish() error {
- var errstring string
- for _, stream := range m.Streams {
- if stream.uploader == nil {
- continue
- }
- if stream.Block != nil {
- stream.uploader <- stream.Block
- }
- close(stream.uploader)
- stream.uploader = nil
-
- errors := <-stream.finish
- close(stream.finish)
- stream.finish = nil
-
- for _, r := range errors {
- errstring = fmt.Sprintf("%v%v\n", errstring, r.Error())
- }
- }
- if errstring != "" {
- return errors.New(errstring)
- } else {
- return nil
- }
-}
-
-func (m *ManifestWriter) ManifestText() string {
- m.Finish()
- var buf bytes.Buffer
-
- dirs := make([]string, len(m.Streams))
- i := 0
- for k := range m.Streams {
- dirs[i] = k
- i++
- }
- sort.Strings(dirs)
-
- for _, k := range dirs {
- v := m.Streams[k]
-
- if k == "." {
- buf.WriteString(".")
- } else {
- k = strings.Replace(k, " ", "\\040", -1)
- k = strings.Replace(k, "\n", "", -1)
- buf.WriteString("./" + k)
- }
- for _, b := range v.Blocks {
- buf.WriteString(" ")
- buf.WriteString(b)
- }
- for _, f := range v.FileStreamSegments {
- buf.WriteString(" ")
- name := strings.Replace(f.Name, " ", "\\040", -1)
- name = strings.Replace(name, "\n", "", -1)
- buf.WriteString(fmt.Sprintf("%d:%d:%s", f.SegPos, f.SegLen, name))
- }
- buf.WriteString("\n")
- }
- return buf.String()
-}
-
-func WriteTree(kc IKeepClient, root string) (manifest string, err error) {
- mw := ManifestWriter{kc, root, map[string]*ManifestStreamWriter{}}
- err = filepath.Walk(root, mw.WalkFunc)
-
- if err != nil {
- return "", err
- }
-
- err = mw.Finish()
- if err != nil {
- return "", err
- }
-
- return mw.ManifestText(), nil
-}
diff --git a/sdk/go/crunchrunner/upload_test.go b/sdk/go/crunchrunner/upload_test.go
deleted file mode 100644
index 5bc749258..000000000
--- a/sdk/go/crunchrunner/upload_test.go
+++ /dev/null
@@ -1,152 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package main
-
-import (
- "crypto/md5"
- "errors"
- "fmt"
- "io/ioutil"
- "os"
- "syscall"
-
- . "gopkg.in/check.v1"
-)
-
-type UploadTestSuite struct{}
-
-// Gocheck boilerplate
-var _ = Suite(&UploadTestSuite{})
-
-type KeepTestClient struct {
-}
-
-func (k KeepTestClient) PutHB(hash string, buf []byte) (string, int, error) {
- return fmt.Sprintf("%x+%v", md5.Sum(buf), len(buf)), len(buf), nil
-}
-
-func (s *TestSuite) TestSimpleUpload(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
-
- str, err := WriteTree(KeepTestClient{}, tmpdir)
- c.Check(err, IsNil)
- c.Check(str, Equals, ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt\n")
-}
-
-func (s *TestSuite) TestSimpleUploadThreeFiles(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- for _, err := range []error{
- ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600),
- ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600),
- os.Symlink("./file2.txt", tmpdir+"/file3.txt"),
- syscall.Mkfifo(tmpdir+"/ignore.fifo", 0600),
- } {
- c.Assert(err, IsNil)
- }
-
- str, err := WriteTree(KeepTestClient{}, tmpdir)
- c.Check(err, IsNil)
- c.Check(str, Equals, ". aa65a413921163458c52fea478d5d3ee+9 0:3:file1.txt 3:3:file2.txt 6:3:file3.txt\n")
-}
-
-func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- os.Mkdir(tmpdir+"/subdir", 0700)
-
- ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
- ioutil.WriteFile(tmpdir+"/subdir/file2.txt", []byte("bar"), 0600)
-
- str, err := WriteTree(KeepTestClient{}, tmpdir)
- c.Check(err, IsNil)
- c.Check(str, Equals, `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
-./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
-`)
-}
-
-func (s *TestSuite) TestSimpleUploadLarge(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- file, _ := os.Create(tmpdir + "/" + "file1.txt")
- data := make([]byte, 1024*1024-1)
- for i := range data {
- data[i] = byte(i % 10)
- }
- for i := 0; i < 65; i++ {
- file.Write(data)
- }
- file.Close()
-
- ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600)
-
- str, err := WriteTree(KeepTestClient{}, tmpdir)
- c.Check(err, IsNil)
- c.Check(str, Equals, ". 00ecf01e0d93385115c9f8bed757425d+67108864 485cd630387b6b1846fe429f261ea05f+1048514 0:68157375:file1.txt 68157375:3:file2.txt\n")
-}
-
-func (s *TestSuite) TestUploadEmptySubdir(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- os.Mkdir(tmpdir+"/subdir", 0700)
-
- ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
-
- str, err := WriteTree(KeepTestClient{}, tmpdir)
- c.Check(err, IsNil)
- c.Check(str, Equals, `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
-`)
-}
-
-func (s *TestSuite) TestUploadEmptyFile(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte(""), 0600)
-
- str, err := WriteTree(KeepTestClient{}, tmpdir)
- c.Check(err, IsNil)
- c.Check(str, Equals, `. d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1.txt
-`)
-}
-
-type KeepErrorTestClient struct {
-}
-
-func (k KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
- return "", 0, errors.New("Failed!")
-}
-
-func (s *TestSuite) TestUploadError(c *C) {
- tmpdir, _ := ioutil.TempDir("", "")
- defer func() {
- os.RemoveAll(tmpdir)
- }()
-
- ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
-
- str, err := WriteTree(KeepErrorTestClient{}, tmpdir)
- c.Check(err, NotNil)
- c.Check(str, Equals, "")
-}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list