[ARVADOS] updated: 1.3.0-709-gb5bfc7d1b
Git user
git at public.curoverse.com
Thu Apr 11 19:16:39 UTC 2019
Summary of changes:
apps/workbench/.gitignore | 2 +
doc/api/methods/collections.html.textile.liquid | 2 +
lib/controller/rpc/conn_test.go | 10 +-
sdk/cwl/setup.py | 1 +
sdk/cwl/tests/federation/framework/check-exist.cwl | 5 +-
services/api/.gitignore | 3 +
services/api/Gemfile | 32 +-
services/api/Gemfile.lock | 264 +++++-----
services/api/Rakefile | 1 +
.../api/app/controllers/application_controller.rb | 54 +-
.../v1/api_client_authorizations_controller.rb | 8 +-
.../arvados/v1/api_clients_controller.rb | 2 +-
.../arvados/v1/containers_controller.rb | 4 +-
.../controllers/arvados/v1/groups_controller.rb | 4 +-
.../arvados/v1/healthcheck_controller.rb | 20 +-
.../app/controllers/arvados/v1/jobs_controller.rb | 4 +-
.../arvados/v1/keep_disks_controller.rb | 4 +-
.../arvados/v1/keep_services_controller.rb | 6 +-
.../app/controllers/arvados/v1/nodes_controller.rb | 6 +-
.../arvados/v1/repositories_controller.rb | 6 +-
.../controllers/arvados/v1/schema_controller.rb | 26 +-
.../arvados/v1/user_agreements_controller.rb | 6 +-
.../app/controllers/arvados/v1/users_controller.rb | 8 +-
.../arvados/v1/virtual_machines_controller.rb | 6 +-
.../api/app/controllers/database_controller.rb | 6 +-
services/api/app/controllers/static_controller.rb | 8 +-
.../app/controllers/user_sessions_controller.rb | 10 +-
.../api/app/models/application_record.rb | 5 +-
services/api/app/models/arvados_model.rb | 32 +-
services/api/app/models/collection.rb | 35 +-
services/api/app/models/container.rb | 8 +-
services/api/app/models/container_request.rb | 8 +-
services/api/app/models/group.rb | 5 +-
services/api/app/models/job.rb | 5 +-
services/api/app/models/jsonb_type.rb | 45 ++
services/api/app/models/link.rb | 7 +-
services/api/app/models/node.rb | 8 +-
services/api/app/models/pipeline_instance.rb | 6 +-
services/api/app/models/user.rb | 22 +-
services/api/{config/boot.rb => bin/bundle} | 8 +-
services/api/bin/rails | 9 +
.../api/bin/rake | 8 +-
services/api/bin/setup | 39 ++
services/api/bin/update | 34 ++
services/api/config/application.default.yml | 9 +-
services/api/config/application.rb | 16 +-
services/api/config/boot.rb | 6 +-
services/api/config/cable.yml | 13 +
services/api/config/environment.rb | 4 +-
.../api/config/environments/production.rb.example | 2 +-
services/api/config/environments/test.rb.example | 4 +-
...types.rb => application_controller_renderer.rb} | 9 +-
services/api/config/initializers/assets.rb | 15 +
.../api/config/initializers/cookies_serializer.rb | 9 +
services/api/config/initializers/custom_types.rb | 8 +
.../{mime_types.rb => filter_parameter_logging.rb} | 5 +-
.../config/initializers/new_framework_defaults.rb | 26 +
services/api/config/initializers/session_store.rb | 2 +-
services/api/config/puma.rb | 51 ++
services/api/config/secrets.yml | 26 +
services/api/config/spring.rb | 10 +
.../migrate/20121016005009_create_collections.rb | 2 +-
.../db/migrate/20130105203021_create_metadata.rb | 2 +-
.../20130105224358_rename_metadata_class.rb | 2 +-
...05224618_rename_collection_created_by_client.rb | 2 +-
.../20130107181109_add_uuid_to_collections.rb | 2 +-
.../api/db/migrate/20130107212832_create_nodes.rb | 2 +-
.../db/migrate/20130109175700_create_pipelines.rb | 2 +-
.../20130109220548_create_pipeline_invocations.rb | 2 +-
...214204_add_index_to_collections_and_metadata.rb | 2 +-
.../db/migrate/20130116024233_create_specimens.rb | 2 +-
.../db/migrate/20130116215213_create_projects.rb | 2 +-
.../20130118002239_rename_metadata_attributes.rb | 2 +-
.../api/db/migrate/20130122020042_create_users.rb | 2 +-
.../api/db/migrate/20130122201442_create_logs.rb | 2 +-
.../20130122221616_add_modified_at_to_logs.rb | 2 +-
.../20130123174514_add_uuid_index_to_users.rb | 2 +-
.../migrate/20130123180224_create_api_clients.rb | 2 +-
...30123180228_create_api_client_authorizations.rb | 2 +-
.../20130125220425_rename_created_by_to_owner.rb | 2 +-
.../20130128202518_rename_metadata_to_links.rb | 2 +-
.../20130128231343_add_properties_to_specimen.rb | 2 +-
...130130205749_add_manifest_text_to_collection.rb | 2 +-
.../api/db/migrate/20130203104818_create_jobs.rb | 2 +-
.../db/migrate/20130203104824_create_job_steps.rb | 2 +-
.../migrate/20130203115329_add_priority_to_jobs.rb | 2 +-
.../20130207195855_add_index_on_timestamps.rb | 2 +-
...81504_add_properties_to_pipeline_invocations.rb | 2 +-
...130226170000_remove_native_target_from_links.rb | 2 +-
.../20130313175417_rename_projects_to_groups.rb | 2 +-
.../20130315155820_add_is_locked_by_to_jobs.rb | 2 +-
.../db/migrate/20130315183626_add_log_to_jobs.rb | 2 +-
.../20130315213205_add_tasks_summary_to_jobs.rb | 2 +-
.../20130318002138_add_resource_limits_to_jobs.rb | 2 +-
.../20130319165853_rename_job_command_to_script.rb | 2 +-
...ame_pipeline_invocation_to_pipeline_instance.rb | 2 +-
...94637_rename_pipelines_to_pipeline_templates.rb | 2 +-
...20130319201431_rename_job_steps_to_job_tasks.rb | 2 +-
.../20130319235957_add_default_owner_to_users.rb | 2 +-
...d_default_owner_to_api_client_authorizations.rb | 2 +-
.../db/migrate/20130326173804_create_commits.rb | 2 +-
.../20130326182917_create_commit_ancestors.rb | 2 +-
.../20130415020241_rename_orvos_to_arvados.rb | 2 +-
.../db/migrate/20130425024459_create_keep_disks.rb | 2 +-
...vice_port_and_service_ssl_flag_to_keep_disks.rb | 2 +-
...3060112_add_created_by_job_task_to_job_tasks.rb | 2 +-
.../20130523060213_add_qsequence_to_job_tasks.rb | 2 +-
.../20130524042319_fix_job_task_qsequence_type.rb | 2 +-
.../migrate/20130528134100_update_nodes_index.rb | 2 +-
.../20130606183519_create_authorized_keys.rb | 2 +-
.../20130608053730_create_virtual_machines.rb | 2 +-
.../migrate/20130610202538_create_repositories.rb | 2 +-
..._key_authorized_user_to_authorized_user_uuid.rb | 2 +-
...042554_add_name_unique_index_to_repositories.rb | 2 +-
...20130617150007_add_is_trusted_to_api_clients.rb | 2 +-
.../20130626002829_add_is_active_to_users.rb | 2 +-
.../migrate/20130626022810_activate_all_admins.rb | 2 +-
.../api/db/migrate/20130627154537_create_traits.rb | 2 +-
.../api/db/migrate/20130627184333_create_humans.rb | 2 +-
...0130708163414_rename_foreign_uuid_attributes.rb | 2 +-
...708182912_rename_job_foreign_uuid_attributes.rb | 2 +-
.../20130708185153_rename_user_default_owner.rb | 2 +-
...3034_add_scopes_to_api_client_authorizations.rb | 2 +-
...ename_resource_limits_to_runtime_constraints.rb | 2 +-
.../20140117231056_normalize_collection_uuid.rb | 2 +-
.../20140124222114_fix_link_kind_underscores.rb | 2 +-
...malize_collection_uuids_in_script_parameters.rb | 2 +-
...317135600_add_nondeterministic_column_to_job.rb | 2 +-
...0547_separate_repository_from_script_version.rb | 2 +-
.../20140321191343_add_repository_column_to_job.rb | 2 +-
...140324024606_add_output_is_persistent_to_job.rb | 2 +-
.../migrate/20140325175653_remove_kind_columns.rb | 2 +-
.../db/migrate/20140402001908_add_system_group.rb | 2 +-
...20140407184311_rename_log_info_to_properties.rb | 2 +-
.../20140421140924_add_group_class_to_groups.rb | 2 +-
.../20140421151939_rename_auth_keys_user_index.rb | 2 +-
.../migrate/20140421151940_timestamps_not_null.rb | 2 +-
.../20140422011506_pipeline_instance_state.rb | 2 +-
.../20140423132913_add_object_owner_to_logs.rb | 2 +-
.../db/migrate/20140423133559_new_scope_format.rb | 2 +-
...0140501165548_add_unique_name_index_to_links.rb | 2 +-
.../migrate/20140519205916_create_keep_services.rb | 2 +-
...152921_add_description_to_pipeline_templates.rb | 2 +-
.../20140530200539_add_supplied_script_version.rb | 2 +-
.../20140601022548_remove_name_from_collections.rb | 2 +-
...e_active_and_success_from_pipeline_instances.rb | 2 +-
.../20140607150616_rename_folder_to_project.rb | 2 +-
.../20140611173003_add_docker_locator_to_jobs.rb | 2 +-
.../db/migrate/20140627210837_anonymous_group.rb | 2 +-
.../20140709172343_job_task_serial_qsequence.rb | 2 +-
.../db/migrate/20140714184006_empty_collection.rb | 2 +-
.../20140811184643_collection_use_regular_uuids.rb | 2 +-
.../20140817035914_add_unique_name_constraints.rb | 2 +-
...125735_add_not_null_constraint_to_group_name.rb | 2 +-
...826180337_remove_output_is_persistent_column.rb | 2 +-
.../migrate/20140828141043_job_priority_fixup.rb | 2 +-
...add_start_finish_time_to_tasks_and_pipelines.rb | 2 +-
...d_description_to_pipeline_instances_and_jobs.rb | 2 +-
...140918141529_change_user_owner_uuid_not_null.rb | 2 +-
.../20140918153541_add_properties_to_node.rb | 2 +-
.../db/migrate/20140918153705_add_state_to_job.rb | 2 +-
.../20140924091559_add_job_uuid_to_nodes.rb | 2 +-
...141111133038_add_arvados_sdk_version_to_jobs.rb | 2 +-
.../db/migrate/20141208164553_owner_uuid_index.rb | 2 +-
.../20141208174553_descriptions_are_strings.rb | 2 +-
.../20141208174653_collection_file_names.rb | 2 +-
.../api/db/migrate/20141208185217_search_index.rb | 2 +-
...0150122175935_no_description_in_search_index.rb | 2 +-
.../db/migrate/20150123142953_full_text_search.rb | 2 +-
...203180223_set_group_class_on_anonymous_group.rb | 2 +-
...206210804_all_users_can_read_anonymous_group.rb | 2 +-
...20150206230342_rename_replication_attributes.rb | 2 +-
...ollection_name_owner_unique_only_non_expired.rb | 2 +-
...tion_portable_data_hash_with_hinted_manifest.rb | 2 +-
...136_change_collection_expires_at_to_datetime.rb | 2 +-
.../20150317132720_add_username_to_users.rb | 2 +-
...backward_compatibility_for_user_repositories.rb | 2 +-
...5759_no_filenames_in_collection_search_index.rb | 2 +-
.../20150512193020_read_only_on_keep_services.rb | 2 +-
...50526180251_leading_space_on_full_text_index.rb | 2 +-
...0151202151426_create_containers_and_requests.rb | 2 +-
.../migrate/20151215134304_fix_containers_index.rb | 2 +-
.../20151229214707_add_exit_code_to_containers.rb | 2 +-
...8210629_add_uuid_to_api_client_authorization.rb | 2 +-
...209155729_add_uuid_to_api_token_search_index.rb | 2 +-
.../20160324144017_add_components_to_job.rb | 2 +-
.../20160506175108_add_auths_to_container.rb | 2 +-
...9143250_add_auth_and_lock_to_container_index.rb | 2 +-
.../db/migrate/20160808151559_create_workflows.rb | 2 +-
...9195557_add_script_parameters_digest_to_jobs.rb | 2 +-
...0819195725_populate_script_parameters_digest.rb | 2 +-
...160901210110_repair_script_parameters_digest.rb | 2 +-
...20160909181442_rename_workflow_to_definition.rb | 6 +-
.../migrate/20160926194129_add_container_count.rb | 2 +-
...71346_add_use_existing_to_container_requests.rb | 2 +-
...43147_add_scheduling_parameters_to_container.rb | 2 +-
...add_output_and_log_uuid_to_container_request.rb | 2 +-
..._log_uuids_to_container_request_search_index.rb | 2 +-
.../20161213172944_full_text_search_indexes.rb | 2 +-
...61222153434_split_expiry_to_trash_and_delete.rb | 2 +-
...090712_add_output_name_to_container_requests.rb | 2 +-
...utput_name_to_container_request_search_index.rb | 2 +-
...170105160301_add_output_name_to_cr_fts_index.rb | 2 +-
...t_finished_at_on_finished_pipeline_instances.rb | 2 +-
...s_and_workflow_def_in_full_text_search_index.rb | 2 +-
.../20170301225558_no_downgrade_after_json.rb | 2 +-
...0170319063406_serialized_columns_accept_null.rb | 2 +-
..._add_portable_data_hash_index_to_collections.rb | 2 +-
...0012505_add_output_ttl_to_container_requests.rb | 2 +-
...1_add_created_by_job_task_index_to_job_tasks.rb | 2 +-
...0170419173712_add_object_owner_index_to_logs.rb | 2 +-
...esting_container_index_to_container_requests.rb | 2 +-
.../db/migrate/20170628185847_jobs_yaml_to_json.rb | 2 +-
.../api/db/migrate/20170704160233_yaml_to_json.rb | 2 +-
.../20170706141334_json_collection_properties.rb | 2 +-
.../db/migrate/20170824202826_trashable_groups.rb | 2 +-
.../20170906224040_materialized_permission_view.rb | 2 +-
.../20171027183824_add_index_to_containers.rb | 2 +-
.../20171208203841_fix_trash_flag_follow.rb | 2 +-
...53352_add_gin_index_to_collection_properties.rb | 2 +-
...216203422_add_storage_classes_to_collections.rb | 2 +-
...180228220311_add_secret_mounts_to_containers.rb | 2 +-
...80313180114_change_container_priority_bigint.rb | 2 +-
...501182859_add_redirect_to_user_uuid_to_users.rb | 2 +-
...20180514135529_add_container_auth_uuid_index.rb | 2 +-
.../migrate/20180607175050_properties_to_jsonb.rb | 2 +-
.../20180608123145_add_properties_to_groups.rb | 2 +-
.../migrate/20180806133039_index_all_filenames.rb | 2 +-
...30357_add_pdh_and_trash_index_to_collections.rb | 2 +-
.../20180820132617_add_lock_index_to_containers.rb | 2 +-
...180820135808_drop_pdh_index_from_collections.rb | 2 +-
.../20180824152014_add_md5_index_to_containers.rb | 2 +-
...20180824155207_add_queue_index_to_containers.rb | 2 +-
...80904110712_add_runtime_status_to_containers.rb | 2 +-
...180913175443_add_version_info_to_collections.rb | 2 +-
...5335_set_current_version_uuid_on_collections.rb | 2 +-
.../20180917200000_replace_full_text_indexes.rb | 2 +-
.../20180917205609_recompute_file_names_index.rb | 2 +-
...001158_recreate_collection_unique_name_index.rb | 2 +-
...01175023_add_preserve_version_to_collections.rb | 2 +-
...rent_version_uuid_to_collection_search_index.rb | 2 +-
.../20181005192222_add_container_runtime_token.rb | 6 +-
...0181011184200_add_runtime_token_to_container.rb | 6 +-
...20181213183234_add_expression_index_to_links.rb | 6 +-
.../20190214214814_add_container_lock_count.rb | 6 +-
.../20190322174136_add_file_info_to_collection.rb | 63 +++
services/api/db/structure.sql | 577 ++++++++-------------
services/api/lib/can_be_an_owner.rb | 1 +
services/api/lib/enable_jobs_api.rb | 4 +-
services/api/lib/group_pdhs.rb | 39 ++
services/api/lib/has_uuid.rb | 4 +-
services/api/lib/load_param.rb | 4 +-
services/api/script/fail-jobs.rb | 4 +-
services/api/script/get_anonymous_user_token.rb | 4 +-
services/api/script/salvage_collection.rb | 4 +-
services/api/script/setup-new-user.rb | 6 +-
services/api/test/fixtures/collections.yml | 16 +
services/api/test/fixtures/nodes.yml | 5 +
.../test/functional/application_controller_test.rb | 12 +-
.../api_client_authorizations_controller_test.rb | 26 +-
.../arvados/v1/collections_controller_test.rb | 237 ++++++---
.../v1/container_requests_controller_test.rb | 10 +-
.../arvados/v1/containers_controller_test.rb | 23 +-
.../api/test/functional/arvados/v1/filters_test.rb | 37 +-
.../arvados/v1/groups_controller_test.rb | 153 ++----
.../arvados/v1/job_reuse_controller_test.rb | 262 +++++-----
.../functional/arvados/v1/jobs_controller_test.rb | 182 +++----
.../arvados/v1/keep_disks_controller_test.rb | 12 +-
.../arvados/v1/keep_services_controller_test.rb | 4 +-
.../functional/arvados/v1/links_controller_test.rb | 74 +--
.../functional/arvados/v1/logs_controller_test.rb | 8 +-
.../functional/arvados/v1/nodes_controller_test.rb | 38 +-
.../v1/pipeline_instances_controller_test.rb | 6 +-
.../api/test/functional/arvados/v1/query_test.rb | 16 +-
.../arvados/v1/repositories_controller_test.rb | 4 +-
.../functional/arvados/v1/users_controller_test.rb | 102 ++--
.../arvados/v1/virtual_machines_controller_test.rb | 4 +-
.../functional/user_sessions_controller_test.rb | 6 +-
.../api_client_authorizations_api_test.rb | 52 +-
.../api_client_authorizations_scopes_test.rb | 22 +-
.../api/test/integration/collections_api_test.rb | 226 ++++----
.../integration/collections_performance_test.rb | 20 +-
.../api/test/integration/container_auth_test.rb | 46 +-
services/api/test/integration/cross_origin_test.rb | 12 +-
.../api/test/integration/crunch_dispatch_test.rb | 23 +-
.../api/test/integration/database_reset_test.rb | 29 +-
services/api/test/integration/errors_test.rb | 2 +-
services/api/test/integration/groups_test.rb | 149 ++++--
services/api/test/integration/jobs_api_test.rb | 55 +-
services/api/test/integration/keep_proxy_test.rb | 8 +-
.../api/test/integration/login_workflow_test.rb | 15 +-
.../api/test/integration/noop_deep_munge_test.rb | 35 +-
services/api/test/integration/permissions_test.rb | 430 ++++++++-------
services/api/test/integration/pipeline_test.rb | 10 +-
.../api/test/integration/reader_tokens_test.rb | 10 +-
services/api/test/integration/remote_user_test.rb | 116 +++--
services/api/test/integration/select_test.rb | 34 +-
.../test/integration/serialized_encoding_test.rb | 3 +-
.../api/test/integration/user_sessions_test.rb | 4 +-
services/api/test/integration/users_test.rb | 158 +++---
services/api/test/integration/valid_links_test.rb | 48 +-
services/api/test/performance/links_index_test.rb | 24 +-
services/api/test/performance/permission_test.rb | 5 +-
services/api/test/test_helper.rb | 14 +
services/api/test/unit/arvados_model_test.rb | 6 +-
services/api/test/unit/collection_test.rb | 50 ++
services/api/test/unit/container_test.rb | 2 +
services/api/test/unit/crunch_dispatch_test.rb | 11 -
services/api/test/unit/group_pdhs_test.rb | 27 +
services/api/test/unit/job_test.rb | 5 +-
310 files changed, 2916 insertions(+), 2077 deletions(-)
copy apps/workbench/app/controllers/keep_services_controller.rb => services/api/app/models/application_record.rb (55%)
create mode 100644 services/api/app/models/jsonb_type.rb
copy services/api/{config/boot.rb => bin/bundle} (58%)
mode change 100644 => 100755
create mode 100755 services/api/bin/rails
copy apps/workbench/config/initializers/validate_wb2_url_config.rb => services/api/bin/rake (52%)
mode change 100644 => 100755
create mode 100755 services/api/bin/setup
create mode 100755 services/api/bin/update
create mode 100644 services/api/config/cable.yml
copy services/api/config/initializers/{mime_types.rb => application_controller_renderer.rb} (50%)
create mode 100644 services/api/config/initializers/assets.rb
create mode 100644 services/api/config/initializers/cookies_serializer.rb
create mode 100644 services/api/config/initializers/custom_types.rb
copy services/api/config/initializers/{mime_types.rb => filter_parameter_logging.rb} (52%)
create mode 100644 services/api/config/initializers/new_framework_defaults.rb
create mode 100644 services/api/config/puma.rb
create mode 100644 services/api/config/secrets.yml
create mode 100644 services/api/config/spring.rb
create mode 100755 services/api/db/migrate/20190322174136_add_file_info_to_collection.rb
create mode 100644 services/api/lib/group_pdhs.rb
create mode 100644 services/api/test/unit/group_pdhs_test.rb
discards 633543684bec7bcd34f9236759a983018770b8f3 (commit)
via b5bfc7d1b3832e3e7ad5f89b34e5a666b0066b77 (commit)
via b2addc8887d200219b44121c87a7a44bf4566e42 (commit)
via 7e1bf9eba617e109d245d10ab350097bb357d904 (commit)
via 5f0826eb93fdb82cc367f173987c7c6913dae7a5 (commit)
via 61b19018e3bff1557d0e640ab4383d55aab9b59d (commit)
via 0a2adc4256358d620fb063489aafbaba8de62b84 (commit)
via 4a27ec5683273596d797e84924c4c583cfc53560 (commit)
via 248c7167e95d970b770c43102ee68cf1319973f7 (commit)
via a7d2e8bc017ffaebd6fa7df0187ef6421f6fa9df (commit)
via 1963cf522fe547dd88474f29d2263a7d98e99fea (commit)
via 4ec3bf28abdaccca697563dac1bd126ef6df9975 (commit)
via 49754950aa80cd1163a053de0f37d975c277e012 (commit)
via 972945983769bd25f6ee1e7254287f48fe6d8c96 (commit)
via a048d6957ff6284c4234198ed9b868bff07d057f (commit)
via 7dcebbc2afce3dae3f0ce0302759ba4ccc2afe99 (commit)
via 66cab5a1f2b24c903a1a1c8b2316cc8b3f35ce1b (commit)
via 08d0b1ab43499b7f13462d5e3555d239b4634d22 (commit)
via a5cd06261d3ef5005c3bd921c610abfa21dc672f (commit)
via 425836b285a32c31ef643f8c5d4b48b8b42b7ac4 (commit)
via a689825a37a32ded05866937b60742da415ce1f3 (commit)
via 265b94326ca8bcb69aea6915a549a4edfcaf0e28 (commit)
via 97d2da2083d046057cb589115867eec430706d68 (commit)
via 33021029867be4a2240f0d3673045dfac7598350 (commit)
via 4eaad199ac21e552eee2a049d33a3c076d8bed60 (commit)
via eefc23215940fc40ec7eff4c6e7c52d6b263efee (commit)
via 39607cc8eaa09bc1247fccbcd7ec13635db1a1ef (commit)
via 79316b0ebbf5bfe934267579478b89f770c0e5ba (commit)
via dd2e6f664a3e59e02349901a04e182bda6286f6f (commit)
via fc636d5e169d944981ce2951e05d59fad04563a3 (commit)
via e26648fc591101349db5644c9927651f84972c3d (commit)
via 59a1fc872723c0bafa9764b95756723f54419631 (commit)
via ce0caee0ecb9c8f6c6cbefc1a12a37560d0f7554 (commit)
via 536953b16482799359990462db6f9f018bf94f1e (commit)
via df5f366f4bd87ee90cf39248542435b1b924b58a (commit)
via cb4efac6793d18892dde09c631895cb98c3df470 (commit)
via bd2ac2038b13b6ebe92b44cf722c8cf0fa15255b (commit)
via c5c82ef67b9dc3cb3619e2bef3a86b9b0f0912e8 (commit)
via 6709876170511ade8e24fe60bf77da24bc4a03d4 (commit)
via fe08de0ddf1d3e536cb3518dcb9c82ca62197273 (commit)
via d6fb38cac6a7f9f98f534b4638ce5918ba94c135 (commit)
via 728522fbc034711e87954bcbdcfa4a3bf7c470e3 (commit)
via 392d1f416f8d6e0ca74984376b57c809f7d1e0a4 (commit)
via 041378345708e781b5b3a4a618e9c4848e465218 (commit)
via a737669021ac34683deecda8130e21b243e14174 (commit)
via 696ee0e5e854347aeb37bdabe3ae3d7712403d06 (commit)
via 8ba20b331fc99252acc29161902017ab98c6a979 (commit)
via 995b43b51c6c651efe266446f6120e67d31f06ae (commit)
via 2a17d214467d5302e97008618ef5f560ff1fd45b (commit)
via 18296ccf884022679b9e1f7fa85e7f6d1dbfaad0 (commit)
via 661fbce61e9a150a0bfc5ab9dd5bff04afdd4286 (commit)
via 82d7893e9b0816896885b3486b5e388002ec8bcb (commit)
via 4447a6160034e0b1f53a54ace8da7e0956c4c452 (commit)
via 4e03c7e92230d5ceb5adf09844f514eacbfc3a41 (commit)
via 7253776cc43c48cbb383f90aa582be2aa73cf09b (commit)
via 261543dfc3ef9c8c6d425e591007bf065ecb254d (commit)
via 8c82f404b48a159797bd0e96e3d0098f0cf3ba16 (commit)
via b0dbec2e1e496e29551dcb01a85328f8982f026f (commit)
via bc408e4ada3c5ee173961b9c61600e3ddf76b88a (commit)
via beb3f5f868010c009b0f2473a18bd98dbe6bc7fb (commit)
via 55bf4eda20444c8cd875c0e5f4e464e77b393946 (commit)
via 9c728077f0d2f8a166d31704918067b2cf526f8b (commit)
via 19ea2726815a0d74f718339b1e42f76cc4bb463c (commit)
via cd4d5b513a7d24bfbf819a7c1c91e6793e71773f (commit)
via f71634c065eff38970b178958349ca9381aee996 (commit)
via cecb9918b556f52218f9ff31e73cb78b0f48eaa4 (commit)
via 3cc37a0ba2c572731ef998a7f5724294c9415fdb (commit)
via aeb966ee09166bf074ea19a4a2dd5045ca795a98 (commit)
via 6b79272e3b6193ccbb2bab29090a6c93efe6e2f2 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (633543684bec7bcd34f9236759a983018770b8f3)
\
N -- N -- N (b5bfc7d1b3832e3e7ad5f89b34e5a666b0066b77)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b5bfc7d1b3832e3e7ad5f89b34e5a666b0066b77
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 11 15:16:34 2019 -0400
14287: Refactor controller to use strong types in API handlers.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index a37a0f731..981b69528 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -77,6 +77,10 @@ doc
lib/cli
lib/cmd
lib/controller
+lib/controller/federation
+lib/controller/railsproxy
+lib/controller/router
+lib/controller/rpc
lib/crunchstat
lib/cloud
lib/cloud/azure
@@ -394,7 +398,7 @@ start_services() {
return 0
fi
. "$VENVDIR/bin/activate"
- echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
+ echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
mkdir -p "$WORKSPACE/services/api/log"
fi
@@ -821,6 +825,7 @@ do_install_once() {
title "install $1"
timer_reset
+ result=
if which deactivate >/dev/null; then deactivate; fi
if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then
result=1
@@ -974,50 +979,7 @@ pythonstuff=(
)
declare -a gostuff
-gostuff=(
- cmd/arvados-client
- cmd/arvados-server
- lib/cli
- lib/cmd
- lib/controller
- lib/crunchstat
- lib/cloud
- lib/cloud/azure
- lib/cloud/ec2
- lib/dispatchcloud
- lib/dispatchcloud/container
- lib/dispatchcloud/scheduler
- lib/dispatchcloud/ssh_executor
- lib/dispatchcloud/worker
- lib/service
- sdk/go/arvados
- sdk/go/arvadosclient
- sdk/go/auth
- sdk/go/blockdigest
- sdk/go/dispatch
- sdk/go/health
- sdk/go/httpserver
- sdk/go/manifest
- sdk/go/asyncbuf
- sdk/go/crunchrunner
- sdk/go/stats
- services/arv-git-httpd
- services/crunchstat
- services/health
- services/keep-web
- services/keepstore
- sdk/go/keepclient
- services/keep-balance
- services/keepproxy
- services/crunch-dispatch-local
- services/crunch-dispatch-slurm
- services/crunch-run
- services/ws
- tools/keep-block-check
- tools/keep-exercise
- tools/keep-rsync
- tools/sync-groups
-)
+gostuff=($(git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
install_apps/workbench() {
cd "$WORKSPACE/apps/workbench" \
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
new file mode 100644
index 000000000..a08ec48f4
--- /dev/null
+++ b/lib/controller/federation/conn.go
@@ -0,0 +1,309 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package federation
+
+import (
+ "context"
+ "crypto/md5"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+
+ "git.curoverse.com/arvados.git/lib/controller/railsproxy"
+ "git.curoverse.com/arvados.git/lib/controller/rpc"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+)
+
+type Interface interface {
+ CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error)
+ CollectionUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Collection, error)
+ CollectionGet(ctx context.Context, options arvados.GetOptions) (arvados.Collection, error)
+ CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error)
+ CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error)
+ ContainerCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Container, error)
+ ContainerUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error)
+ ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error)
+ ContainerList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerList, error)
+ ContainerDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Container, error)
+ ContainerLock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error)
+ ContainerUnlock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error)
+ SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error)
+ SpecimenUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Specimen, error)
+ SpecimenGet(ctx context.Context, options arvados.GetOptions) (arvados.Specimen, error)
+ SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error)
+ SpecimenDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Specimen, error)
+ APIClientAuthorizationCurrent(ctx context.Context, options arvados.GetOptions) (arvados.APIClientAuthorization, error)
+}
+
+type Conn struct {
+ cluster *arvados.Cluster
+ local backend
+ remotes map[string]backend
+}
+
+func New(cluster *arvados.Cluster, np *arvados.NodeProfile) Interface {
+ local := railsproxy.NewConn(cluster, np)
+ remotes := map[string]backend{}
+ for id, remote := range cluster.RemoteClusters {
+ if !remote.Proxy {
+ continue
+ }
+ remotes[id] = rpc.NewConn(id, &url.URL{Scheme: remote.Scheme, Host: remote.Host}, remote.Insecure, saltedTokenProvider(local, id))
+ }
+
+ return &Conn{
+ cluster: cluster,
+ local: local,
+ remotes: remotes,
+ }
+}
+
+// Return a new rpc.TokenProvider that takes the client-provided
+// tokens from an incoming request context, determines whether they
+// should (and can) be salted for the given remoteID, and returns the
+// resulting tokens.
+func saltedTokenProvider(local backend, remoteID string) rpc.TokenProvider {
+ return func(ctx context.Context) ([]string, error) {
+ var tokens []string
+ incoming, ok := ctx.Value(auth.ContextKeyCredentials).(*auth.Credentials)
+ if !ok {
+ return nil, errors.New("no token provided")
+ }
+ for _, token := range incoming.Tokens {
+ salted, err := auth.SaltToken(token, remoteID)
+ switch err {
+ case nil:
+ tokens = append(tokens, salted)
+ case auth.ErrSalted:
+ tokens = append(tokens, token)
+ case auth.ErrObsoleteToken:
+ ctx := context.WithValue(ctx, auth.ContextKeyCredentials, &auth.Credentials{Tokens: []string{token}})
+ aca, err := local.APIClientAuthorizationCurrent(ctx, arvados.GetOptions{})
+ if errStatus(err) == http.StatusUnauthorized {
+ // pass through unmodified
+ tokens = append(tokens, token)
+ continue
+ } else if err != nil {
+ return nil, err
+ }
+ salted, err := auth.SaltToken(aca.TokenV2(), remoteID)
+ if err != nil {
+ return nil, err
+ }
+ tokens = append(tokens, salted)
+ default:
+ return nil, err
+ }
+ }
+ return tokens, nil
+ }
+}
+
+// Return suitable backend for a query about the given cluster ID
+// ("aaaaa") or object UUID ("aaaaa-dz642-abcdefghijklmno").
+func (conn *Conn) chooseBackend(id string) backend {
+ if len(id) > 5 {
+ id = id[:5]
+ }
+ if id == conn.cluster.ClusterID {
+ return conn.local
+ } else if be, ok := conn.remotes[id]; ok {
+ return be
+ } else {
+ // TODO: return an "always error" backend?
+ return conn.local
+ }
+}
+
+// Call fn with the local backend; then, if fn returned 404, call fn
+// on the available remote backends (possibly concurrently) until one
+// succeeds.
+//
+// The second argument to fn is the cluster ID of the remote backend,
+// or "" for the local backend.
+//
+// A non-nil error means all backends failed.
+func (conn *Conn) tryLocalThenRemotes(ctx context.Context, fn func(context.Context, string, backend) error) error {
+ if err := fn(ctx, "", conn.local); err == nil || errStatus(err) != http.StatusNotFound {
+ return err
+ }
+
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ errchan := make(chan error, len(conn.remotes))
+ for remoteID, be := range conn.remotes {
+ remoteID, be := remoteID, be
+ go func() {
+ errchan <- fn(ctx, remoteID, be)
+ }()
+ }
+ all404 := true
+ var errs []error
+ for i := 0; i < cap(errchan); i++ {
+ err := <-errchan
+ if err == nil {
+ return nil
+ }
+ all404 = all404 && errStatus(err) == http.StatusNotFound
+ errs = append(errs, err)
+ }
+ if all404 {
+ return notFoundError{}
+ }
+ // FIXME: choose appropriate HTTP status
+ return fmt.Errorf("errors: %v", errs)
+}
+
+func (conn *Conn) CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error) {
+ return conn.chooseBackend(options.ClusterID).CollectionCreate(ctx, options)
+}
+
+func (conn *Conn) CollectionUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Collection, error) {
+ return conn.chooseBackend(options.UUID).CollectionUpdate(ctx, options)
+}
+
+func rewriteManifest(mt, remoteID string) string {
+ return regexp.MustCompile(` [0-9a-f]{32}\+[^ ]*`).ReplaceAllStringFunc(mt, func(tok string) string {
+ return strings.Replace(tok, "+A", "+R"+remoteID+"-", -1)
+ })
+}
+
+// this could be in sdk/go/arvados
+func portableDataHash(mt string) string {
+ h := md5.New()
+ blkRe := regexp.MustCompile(`^ [0-9a-f]{32}\+\d+`)
+ size := 0
+ _ = regexp.MustCompile(` ?[^ ]*`).ReplaceAllFunc([]byte(mt), func(tok []byte) []byte {
+ if m := blkRe.Find(tok); m != nil {
+ // write hash+size, ignore remaining block hints
+ tok = m
+ }
+ n, err := h.Write(tok)
+ if err != nil {
+ panic(err)
+ }
+ size += n
+ return nil
+ })
+ return fmt.Sprintf("%x+%d", h.Sum(nil), size)
+}
+
+func (conn *Conn) CollectionGet(ctx context.Context, options arvados.GetOptions) (arvados.Collection, error) {
+ if len(options.UUID) == 27 {
+ // UUID is really a UUID
+ c, err := conn.chooseBackend(options.UUID).CollectionGet(ctx, options)
+ if err == nil && options.UUID[:5] != conn.cluster.ClusterID {
+ c.ManifestText = rewriteManifest(c.ManifestText, options.UUID[:5])
+ }
+ return c, err
+ } else {
+ // UUID is a PDH
+ first := make(chan arvados.Collection, 1)
+ err := conn.tryLocalThenRemotes(ctx, func(ctx context.Context, remoteID string, be backend) error {
+ c, err := be.CollectionGet(ctx, options)
+ if err != nil {
+ return err
+ }
+ if pdh := portableDataHash(c.ManifestText); pdh != options.UUID {
+ ctxlog.FromContext(ctx).Warnf("bad portable data hash %q received from remote %q (expected %q)", pdh, remoteID, options.UUID)
+ return notFoundError{}
+ }
+ if remoteID != "" {
+ c.ManifestText = rewriteManifest(c.ManifestText, remoteID)
+ }
+ select {
+ case first <- c:
+ return nil
+ default:
+ // lost race, return value doesn't matter
+ return nil
+ }
+ })
+ if err != nil {
+ return arvados.Collection{}, err
+ }
+ return <-first, nil
+ }
+}
+
+func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) {
+ return conn.local.CollectionList(ctx, options)
+}
+
+func (conn *Conn) CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) {
+ return conn.chooseBackend(options.UUID).CollectionDelete(ctx, options)
+}
+
+func (conn *Conn) ContainerCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Container, error) {
+ return conn.chooseBackend(options.ClusterID).ContainerCreate(ctx, options)
+}
+
+func (conn *Conn) ContainerUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error) {
+ return conn.chooseBackend(options.UUID).ContainerUpdate(ctx, options)
+}
+
+func (conn *Conn) ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
+ return conn.chooseBackend(options.UUID).ContainerGet(ctx, options)
+}
+
+func (conn *Conn) ContainerList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerList, error) {
+ return conn.local.ContainerList(ctx, options)
+}
+
+func (conn *Conn) ContainerDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Container, error) {
+ return conn.chooseBackend(options.UUID).ContainerDelete(ctx, options)
+}
+
+func (conn *Conn) ContainerLock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
+ return conn.chooseBackend(options.UUID).ContainerLock(ctx, options)
+}
+
+func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
+ return conn.chooseBackend(options.UUID).ContainerUnlock(ctx, options)
+}
+
+func (conn *Conn) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
+ return conn.chooseBackend(options.ClusterID).SpecimenCreate(ctx, options)
+}
+
+func (conn *Conn) SpecimenUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Specimen, error) {
+ return conn.chooseBackend(options.UUID).SpecimenUpdate(ctx, options)
+}
+
+func (conn *Conn) SpecimenGet(ctx context.Context, options arvados.GetOptions) (arvados.Specimen, error) {
+ return conn.chooseBackend(options.UUID).SpecimenGet(ctx, options)
+}
+
+func (conn *Conn) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
+ return conn.local.SpecimenList(ctx, options)
+}
+
+func (conn *Conn) SpecimenDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Specimen, error) {
+ return conn.chooseBackend(options.UUID).SpecimenDelete(ctx, options)
+}
+
+func (conn *Conn) APIClientAuthorizationCurrent(ctx context.Context, options arvados.GetOptions) (arvados.APIClientAuthorization, error) {
+ return conn.chooseBackend(options.UUID).APIClientAuthorizationCurrent(ctx, options)
+}
+
+type backend interface{ Interface }
+
+type notFoundError struct{}
+
+func (notFoundError) HTTPStatus() int { return http.StatusNotFound }
+func (notFoundError) Error() string { return "not found" }
+
+func errStatus(err error) int {
+ if httpErr, ok := err.(interface{ HTTPStatus() int }); ok {
+ return httpErr.HTTPStatus()
+ } else {
+ return http.StatusInternalServerError
+ }
+}
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 62916acd2..06c8f0086 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -39,7 +39,8 @@ type FederationSuite struct {
// provided by the integration test environment.
remoteServer *httpserver.Server
// remoteMock ("zmock") appends each incoming request to
- // remoteMockRequests, and returns an empty 200 response.
+ // remoteMockRequests, and returns 200 with an empty JSON
+ // object.
remoteMock *httpserver.Server
remoteMockRequests []http.Request
}
@@ -68,6 +69,7 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
MaxItemsPerResponse: 1000,
MultiClusterRequestConcurrency: 4,
},
+ EnableBetaController14287: enableBetaController14287,
}, NodeProfile: &nodeProfile}
s.testServer = newServerFromIntegrationTestEnv(c)
s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
@@ -96,6 +98,8 @@ func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Req
req.Body.Close()
req.Body = ioutil.NopCloser(b)
s.remoteMockRequests = append(s.remoteMockRequests, *req)
+ // Repond 200 with a valid JSON object
+ fmt.Fprint(w, "{}")
}
func (s *FederationSuite) TearDownTest(c *check.C) {
@@ -107,15 +111,15 @@ func (s *FederationSuite) TearDownTest(c *check.C) {
}
}
-func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
+func (s *FederationSuite) testRequest(req *http.Request) *httptest.ResponseRecorder {
resp := httptest.NewRecorder()
s.testServer.Server.Handler.ServeHTTP(resp, req)
- return resp.Result()
+ return resp
}
func (s *FederationSuite) TestLocalRequest(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
s.checkHandledLocally(c, resp)
}
@@ -130,7 +134,7 @@ func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
func (s *FederationSuite) TestNoAuth(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
s.checkJSONErrorMatches(c, resp, `Not logged in`)
}
@@ -138,7 +142,7 @@ func (s *FederationSuite) TestNoAuth(c *check.C) {
func (s *FederationSuite) TestBadAuth(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
s.checkJSONErrorMatches(c, resp, `Not logged in`)
}
@@ -146,7 +150,7 @@ func (s *FederationSuite) TestBadAuth(c *check.C) {
func (s *FederationSuite) TestNoAccess(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
s.checkJSONErrorMatches(c, resp, `.*not found`)
}
@@ -154,7 +158,7 @@ func (s *FederationSuite) TestNoAccess(c *check.C) {
func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
}
@@ -166,7 +170,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
}
@@ -174,7 +178,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var wf arvados.Workflow
c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
@@ -185,7 +189,7 @@ func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
func (s *FederationSuite) TestOptionsMethod(c *check.C) {
req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
req.Header.Set("Origin", "https://example.com")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
body, err := ioutil.ReadAll(resp.Body)
c.Check(err, check.IsNil)
@@ -201,7 +205,7 @@ func (s *FederationSuite) TestOptionsMethod(c *check.C) {
func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
- s.testRequest(req)
+ s.testRequest(req).Result()
c.Assert(s.remoteMockRequests, check.HasLen, 1)
pr := s.remoteMockRequests[0]
// Token is salted and moved from query to Authorization header.
@@ -210,28 +214,51 @@ func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
}
func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
- req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
- req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- s.testRequest(req)
- c.Assert(s.remoteMockRequests, check.HasLen, 1)
- pr := s.remoteMockRequests[0]
- // The salted token here has a "zzzzz-" UUID instead of a
- // "ztest-" UUID because ztest's local database has the
- // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
- // arvadostest.ActiveToken, "zmock") = "7fd3...".
- c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+ defer s.localServiceReturns404(c).Close()
+ for _, path := range []string{
+ // During the transition to the strongly typed
+ // controller implementation (#14287), workflows and
+ // collections test different code paths.
+ "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+ "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+ } {
+ c.Log("testing path ", path)
+ s.remoteMockRequests = nil
+ req := httptest.NewRequest("GET", path, nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ s.testRequest(req).Result()
+ c.Assert(s.remoteMockRequests, check.HasLen, 1)
+ pr := s.remoteMockRequests[0]
+ // The salted token here has a "zzzzz-" UUID instead of a
+ // "ztest-" UUID because ztest's local database has the
+ // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
+ // arvadostest.ActiveToken, "zmock") = "7fd3...".
+ c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+ }
}
func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
+ defer s.localServiceReturns404(c).Close()
// remoteToken can be any v1 token that doesn't appear in
// ztest's local db.
remoteToken := "abcdef00000000000000000000000000000000000000000000"
- req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
- req.Header.Set("Authorization", "Bearer "+remoteToken)
- s.testRequest(req)
- c.Assert(s.remoteMockRequests, check.HasLen, 1)
- pr := s.remoteMockRequests[0]
- c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+
+ for _, path := range []string{
+ // During the transition to the strongly typed
+ // controller implementation (#14287), workflows and
+ // collections test different code paths.
+ "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+ "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+ } {
+ c.Log("testing path ", path)
+ s.remoteMockRequests = nil
+ req := httptest.NewRequest("GET", path, nil)
+ req.Header.Set("Authorization", "Bearer "+remoteToken)
+ s.testRequest(req).Result()
+ c.Assert(s.remoteMockRequests, check.HasLen, 1)
+ pr := s.remoteMockRequests[0]
+ c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+ }
}
func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
@@ -273,7 +300,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
req.Header.Set("Content-type", "application/x-www-form-urlencoded")
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
s.checkResponseOK(c, resp)
err := json.NewDecoder(resp.Body).Decode(&wf)
c.Check(err, check.IsNil)
@@ -283,7 +310,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
{
req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
s.checkResponseOK(c, resp)
err := json.NewDecoder(resp.Body).Decode(&wf)
c.Check(err, check.IsNil)
@@ -291,7 +318,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
{
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
}
}
@@ -333,7 +360,15 @@ func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *https
func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- w.WriteHeader(404)
+ if req.URL.Path == "/arvados/v1/api_client_authorizations/current" {
+ if req.Header.Get("Authorization") == "Bearer "+arvadostest.ActiveToken {
+ json.NewEncoder(w).Encode(arvados.APIClientAuthorization{UUID: arvadostest.ActiveTokenUUID, APIToken: arvadostest.ActiveToken})
+ } else {
+ w.WriteHeader(http.StatusUnauthorized)
+ }
+ } else {
+ w.WriteHeader(404)
+ }
}))
}
@@ -350,7 +385,7 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var col arvados.Collection
@@ -367,7 +402,7 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
}).Encode()))
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
- resp = s.testRequest(req)
+ resp = s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
col = arvados.Collection{}
@@ -383,7 +418,7 @@ func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var col arvados.Collection
c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
@@ -398,7 +433,7 @@ func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
}
@@ -425,7 +460,7 @@ func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var col arvados.Collection
@@ -441,7 +476,7 @@ func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
@@ -459,7 +494,7 @@ func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
defer resp.Body.Close()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
@@ -498,7 +533,7 @@ func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
defer resp.Body.Close()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
@@ -514,7 +549,7 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var col arvados.Collection
@@ -535,7 +570,7 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
}
@@ -544,7 +579,7 @@ func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
defer s.localServiceReturns404(c).Close()
req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cr arvados.ContainerRequest
c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -559,7 +594,7 @@ func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
req.Header.Set("Content-type", "application/json")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cr arvados.ContainerRequest
c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -587,7 +622,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
`))
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
req.Header.Set("Content-type", "application/json")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cr arvados.ContainerRequest
c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -624,7 +659,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
s.testHandler.Cluster.NodeProfiles["*"] = np
s.testHandler.NodeProfile = &np
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cr struct {
arvados.ContainerRequest `json:"container_request"`
@@ -655,7 +690,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c
`))
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
req.Header.Set("Content-type", "application/json")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cr struct {
arvados.ContainerRequest `json:"container_request"`
@@ -684,7 +719,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c
`))
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
req.Header.Set("Content-type", "application/json")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cr struct {
arvados.ContainerRequest `json:"container_request"`
@@ -710,7 +745,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
`))
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
req.Header.Set("Content-type", "application/json")
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
}
@@ -719,7 +754,7 @@ func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
resp := s.testRequest(req)
- c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
var cn arvados.Container
c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
@@ -730,10 +765,11 @@ func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cn arvados.ContainerList
c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+ c.Assert(cn.Items, check.HasLen, 1)
c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
}
@@ -750,7 +786,7 @@ func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
url.QueryEscape(`["uuid", "command"]`)),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
var cn arvados.ContainerList
c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
@@ -773,7 +809,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
url.QueryEscape(`["uuid", "command"]`)),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
}
@@ -799,7 +835,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
arvadostest.QueuedContainerUUID))),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
c.Check(callCount, check.Equals, 2)
var cn arvados.ContainerList
@@ -835,7 +871,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
arvadostest.QueuedContainerUUID))),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
c.Check(callCount, check.Equals, 2)
var cn arvados.ContainerList
@@ -856,7 +892,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C)
arvadostest.QueuedContainerUUID))),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
}
@@ -867,7 +903,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
arvadostest.QueuedContainerUUID))),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
}
@@ -878,7 +914,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
arvadostest.QueuedContainerUUID))),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
}
@@ -889,7 +925,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
arvadostest.QueuedContainerUUID))),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
}
@@ -901,7 +937,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
url.QueryEscape(`["command"]`)),
nil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp := s.testRequest(req)
+ resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 53125ae55..c799b617f 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -8,13 +8,14 @@ import (
"context"
"database/sql"
"errors"
- "net"
"net/http"
"net/url"
"strings"
"sync"
"time"
+ "git.curoverse.com/arvados.git/lib/controller/railsproxy"
+ "git.curoverse.com/arvados.git/lib/controller/router"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -61,7 +62,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
func (h *Handler) CheckHealth() error {
h.setupOnce.Do(h.setup)
- _, _, err := findRailsAPI(h.Cluster, h.NodeProfile)
+ _, _, err := railsproxy.FindRailsAPI(h.Cluster, h.NodeProfile)
return err
}
@@ -73,6 +74,13 @@ func (h *Handler) setup() {
Token: h.Cluster.ManagementToken,
Prefix: "/_health/",
})
+
+ if h.Cluster.EnableBetaController14287 {
+ rtr := router.New(h.Cluster, h.NodeProfile)
+ mux.Handle("/arvados/v1/collections", rtr)
+ mux.Handle("/arvados/v1/collections/", rtr)
+ }
+
hs := http.NotFoundHandler()
hs = prepend(hs, h.proxyRailsAPI)
hs = h.setupProxyRemoteCluster(hs)
@@ -126,7 +134,7 @@ func prepend(next http.Handler, middleware middlewareFunc) http.Handler {
}
func (h *Handler) localClusterRequest(req *http.Request) (*http.Response, error) {
- urlOut, insecure, err := findRailsAPI(h.Cluster, h.NodeProfile)
+ urlOut, insecure, err := railsproxy.FindRailsAPI(h.Cluster, h.NodeProfile)
if err != nil {
return nil, err
}
@@ -151,23 +159,3 @@ func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next h
httpserver.Logger(req).WithError(err).WithField("bytesCopied", n).Error("error copying response body")
}
}
-
-// For now, findRailsAPI always uses the rails API running on this
-// node.
-func findRailsAPI(cluster *arvados.Cluster, np *arvados.NodeProfile) (*url.URL, bool, error) {
- hostport := np.RailsAPI.Listen
- if len(hostport) > 1 && hostport[0] == ':' && strings.TrimRight(hostport[1:], "0123456789") == "" {
- // ":12345" => connect to indicated port on localhost
- hostport = "localhost" + hostport
- } else if _, _, err := net.SplitHostPort(hostport); err == nil {
- // "[::1]:12345" => connect to indicated address & port
- } else {
- return nil, false, err
- }
- proto := "http"
- if np.RailsAPI.TLS {
- proto = "https"
- }
- url, err := url.Parse(proto + "://" + hostport)
- return url, np.RailsAPI.Insecure, err
-}
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 96110ea85..7041d3504 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -22,9 +22,13 @@ import (
check "gopkg.in/check.v1"
)
+var enableBetaController14287 bool
+
// Gocheck boilerplate
func Test(t *testing.T) {
- check.TestingT(t)
+ for _, enableBetaController14287 = range []bool{false, true} {
+ check.TestingT(t)
+ }
}
var _ = check.Suite(&HandlerSuite{})
@@ -48,6 +52,7 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true},
},
},
+ EnableBetaController14287: enableBetaController14287,
}
node := s.cluster.NodeProfiles["*"]
s.handler = newHandler(s.ctx, s.cluster, &node, "")
diff --git a/lib/controller/proxy.go b/lib/controller/proxy.go
index c0b94c2b5..9eac9362c 100644
--- a/lib/controller/proxy.go
+++ b/lib/controller/proxy.go
@@ -25,20 +25,23 @@ func (h HTTPError) Error() string {
return h.Message
}
-// headers that shouldn't be forwarded when proxying. See
-// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
var dropHeaders = map[string]bool{
+ // Headers that shouldn't be forwarded when proxying. See
+ // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
"Connection": true,
"Keep-Alive": true,
"Proxy-Authenticate": true,
"Proxy-Authorization": true,
- // this line makes gofmt 1.10 and 1.11 agree
- "TE": true,
- "Trailer": true,
- "Transfer-Encoding": true, // *-Encoding headers interfer with Go's automatic compression/decompression
- "Content-Encoding": true,
+ // (comment/space here makes gofmt1.10 agree with gofmt1.11)
+ "TE": true,
+ "Trailer": true,
+ "Upgrade": true,
+
+ // Headers that would interfere with Go's automatic
+ // compression/decompression if we forwarded them.
"Accept-Encoding": true,
- "Upgrade": true,
+ "Content-Encoding": true,
+ "Transfer-Encoding": true,
}
type ResponseFilter func(*http.Response, error) (*http.Response, error)
diff --git a/lib/controller/railsproxy/railsproxy.go b/lib/controller/railsproxy/railsproxy.go
new file mode 100644
index 000000000..db1a3f5e6
--- /dev/null
+++ b/lib/controller/railsproxy/railsproxy.go
@@ -0,0 +1,56 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+// Package railsproxy implements Arvados APIs by proxying to the
+// RailsAPI server on the local machine.
+package railsproxy
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net"
+ "net/url"
+ "strings"
+
+ "git.curoverse.com/arvados.git/lib/controller/rpc"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+)
+
+// For now, FindRailsAPI always uses the rails API running on this
+// node.
+func FindRailsAPI(cluster *arvados.Cluster, np *arvados.NodeProfile) (*url.URL, bool, error) {
+ hostport := np.RailsAPI.Listen
+ if len(hostport) > 1 && hostport[0] == ':' && strings.TrimRight(hostport[1:], "0123456789") == "" {
+ // ":12345" => connect to indicated port on localhost
+ hostport = "localhost" + hostport
+ } else if _, _, err := net.SplitHostPort(hostport); err == nil {
+ // "[::1]:12345" => connect to indicated address & port
+ } else {
+ return nil, false, err
+ }
+ proto := "http"
+ if np.RailsAPI.TLS {
+ proto = "https"
+ }
+ url, err := url.Parse(proto + "://" + hostport)
+ return url, np.RailsAPI.Insecure, err
+}
+
+func NewConn(cluster *arvados.Cluster, np *arvados.NodeProfile) *rpc.Conn {
+ url, insecure, err := FindRailsAPI(cluster, np)
+ if err != nil {
+ panic(fmt.Sprintf("NodeProfile RailsAPI %#v: %s", np.RailsAPI, err))
+ }
+ return rpc.NewConn(cluster.ClusterID, url, insecure, provideIncomingToken)
+}
+
+func provideIncomingToken(ctx context.Context) ([]string, error) {
+ incoming, ok := ctx.Value(auth.ContextKeyCredentials).(*auth.Credentials)
+ if !ok {
+ return nil, errors.New("no token provided")
+ }
+ return incoming.Tokens, nil
+}
diff --git a/lib/controller/router/error.go b/lib/controller/router/error.go
new file mode 100644
index 000000000..6db5f3155
--- /dev/null
+++ b/lib/controller/router/error.go
@@ -0,0 +1,18 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package router
+
+type errorWithStatus struct {
+ code int
+ error
+}
+
+func (err errorWithStatus) HTTPStatus() int {
+ return err.code
+}
+
+func httpError(code int, err error) error {
+ return errorWithStatus{code: code, error: err}
+}
diff --git a/lib/controller/router/request.go b/lib/controller/router/request.go
new file mode 100644
index 000000000..67d4e0ffb
--- /dev/null
+++ b/lib/controller/router/request.go
@@ -0,0 +1,112 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package router
+
+import (
+ "encoding/json"
+ "io"
+ "mime"
+ "net/http"
+ "strconv"
+ "strings"
+
+ "github.com/julienschmidt/httprouter"
+)
+
+// Parse req as an Arvados V1 API request and return the request
+// parameters.
+//
+// If the request has a parameter whose name is attrsKey (e.g.,
+// "collection"), it is renamed to "attrs".
+func (rtr *router) loadRequestParams(req *http.Request, attrsKey string) (map[string]interface{}, error) {
+ err := req.ParseForm()
+ if err != nil {
+ return nil, httpError(http.StatusBadRequest, err)
+ }
+ params := map[string]interface{}{}
+ for k, values := range req.Form {
+ for _, v := range values {
+ switch {
+ case v == "null" || v == "":
+ params[k] = nil
+ case strings.HasPrefix(v, "["):
+ var j []interface{}
+ err := json.Unmarshal([]byte(v), &j)
+ if err != nil {
+ return nil, err
+ }
+ params[k] = j
+ case strings.HasPrefix(v, "{"):
+ var j map[string]interface{}
+ err := json.Unmarshal([]byte(v), &j)
+ if err != nil {
+ return nil, err
+ }
+ params[k] = j
+ case strings.HasPrefix(v, "\""):
+ var j string
+ err := json.Unmarshal([]byte(v), &j)
+ if err != nil {
+ return nil, err
+ }
+ params[k] = j
+ case k == "limit" || k == "offset":
+ params[k], err = strconv.ParseInt(v, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ default:
+ params[k] = v
+ }
+ // TODO: Need to accept "?foo[]=bar&foo[]=baz"
+ // as foo=["bar","baz"]?
+ }
+ }
+ if ct, _, err := mime.ParseMediaType(req.Header.Get("Content-Type")); err != nil && ct == "application/json" {
+ jsonParams := map[string]interface{}{}
+ err := json.NewDecoder(req.Body).Decode(jsonParams)
+ if err != nil {
+ return nil, httpError(http.StatusBadRequest, err)
+ }
+ for k, v := range jsonParams {
+ params[k] = v
+ }
+ if attrsKey != "" && params[attrsKey] == nil {
+ // Copy top-level parameters from JSON request
+ // body into params[attrsKey]. Some SDKs rely
+ // on this Rails API feature; see
+ // https://api.rubyonrails.org/v5.2.1/classes/ActionController/ParamsWrapper.html
+ params[attrsKey] = jsonParams
+ }
+ }
+
+ routeParams, _ := req.Context().Value(httprouter.ParamsKey).(httprouter.Params)
+ for _, p := range routeParams {
+ params[p.Key] = p.Value
+ }
+
+ if v, ok := params[attrsKey]; ok && attrsKey != "" {
+ params["attrs"] = v
+ delete(params, attrsKey)
+ }
+ return params, nil
+}
+
+// Copy src to dst, using json as an intermediate format in order to
+// invoke src's json-marshaling and dst's json-unmarshaling behaviors.
+func (rtr *router) transcode(src interface{}, dst interface{}) error {
+ var errw error
+ pr, pw := io.Pipe()
+ go func() {
+ defer pw.Close()
+ errw = json.NewEncoder(pw).Encode(src)
+ }()
+ defer pr.Close()
+ err := json.NewDecoder(pr).Decode(dst)
+ if errw != nil {
+ return errw
+ }
+ return err
+}
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
new file mode 100644
index 000000000..65e0159fa
--- /dev/null
+++ b/lib/controller/router/response.go
@@ -0,0 +1,53 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package router
+
+import (
+ "encoding/json"
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+type responseOptions struct {
+ Select []string
+}
+
+func (rtr *router) responseOptions(opts interface{}) (responseOptions, error) {
+ var rOpts responseOptions
+ switch opts := opts.(type) {
+ case *arvados.GetOptions:
+ rOpts.Select = opts.Select
+ }
+ return rOpts, nil
+}
+
+func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts responseOptions) {
+ var tmp map[string]interface{}
+ err := rtr.transcode(resp, &tmp)
+ if err != nil {
+ rtr.sendError(w, err)
+ return
+ }
+ if len(opts.Select) > 0 {
+ selected := map[string]interface{}{}
+ for _, attr := range opts.Select {
+ if v, ok := tmp[attr]; ok {
+ selected[attr] = v
+ }
+ }
+ tmp = selected
+ }
+ json.NewEncoder(w).Encode(tmp)
+}
+
+func (rtr *router) sendError(w http.ResponseWriter, err error) {
+ code := http.StatusInternalServerError
+ if err, ok := err.(interface{ HTTPStatus() int }); ok {
+ code = err.HTTPStatus()
+ }
+ httpserver.Error(w, err.Error(), code)
+}
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
new file mode 100644
index 000000000..4a6f9b5af
--- /dev/null
+++ b/lib/controller/router/router.go
@@ -0,0 +1,210 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package router
+
+import (
+ "context"
+ "net/http"
+
+ "git.curoverse.com/arvados.git/lib/controller/federation"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/julienschmidt/httprouter"
+)
+
+type router struct {
+ mux *httprouter.Router
+ fed federation.Interface
+}
+
+func New(cluster *arvados.Cluster, np *arvados.NodeProfile) *router {
+ rtr := &router{
+ mux: httprouter.New(),
+ fed: federation.New(cluster, np),
+ }
+ rtr.addRoutes(cluster)
+ return rtr
+}
+
+func (rtr *router) addRoutes(cluster *arvados.Cluster) {
+ for _, route := range []struct {
+ endpoint arvados.APIEndpoint
+ defaultOpts func() interface{}
+ exec func(ctx context.Context, opts interface{}) (interface{}, error)
+ }{
+ {
+ arvados.EndpointCollectionCreate,
+ func() interface{} { return &arvados.CreateOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.CollectionCreate(ctx, *opts.(*arvados.CreateOptions))
+ },
+ },
+ {
+ arvados.EndpointCollectionUpdate,
+ func() interface{} { return &arvados.UpdateOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.CollectionUpdate(ctx, *opts.(*arvados.UpdateOptions))
+ },
+ },
+ {
+ arvados.EndpointCollectionGet,
+ func() interface{} { return &arvados.GetOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.CollectionGet(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+ {
+ arvados.EndpointCollectionList,
+ func() interface{} { return &arvados.ListOptions{Limit: -1} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.CollectionList(ctx, *opts.(*arvados.ListOptions))
+ },
+ },
+ {
+ arvados.EndpointCollectionDelete,
+ func() interface{} { return &arvados.DeleteOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.CollectionDelete(ctx, *opts.(*arvados.DeleteOptions))
+ },
+ },
+ {
+ arvados.EndpointContainerCreate,
+ func() interface{} { return &arvados.CreateOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.ContainerCreate(ctx, *opts.(*arvados.CreateOptions))
+ },
+ },
+ {
+ arvados.EndpointContainerUpdate,
+ func() interface{} { return &arvados.UpdateOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.ContainerUpdate(ctx, *opts.(*arvados.UpdateOptions))
+ },
+ },
+ {
+ arvados.EndpointContainerGet,
+ func() interface{} { return &arvados.GetOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.ContainerGet(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+ {
+ arvados.EndpointContainerList,
+ func() interface{} { return &arvados.ListOptions{Limit: -1} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.ContainerList(ctx, *opts.(*arvados.ListOptions))
+ },
+ },
+ {
+ arvados.EndpointContainerDelete,
+ func() interface{} { return &arvados.DeleteOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.ContainerDelete(ctx, *opts.(*arvados.DeleteOptions))
+ },
+ },
+ {
+ arvados.EndpointContainerLock,
+ func() interface{} {
+ return &arvados.GetOptions{Select: []string{"uuid", "state", "priority", "auth_uuid", "locked_by_uuid"}}
+ },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.ContainerLock(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+ {
+ arvados.EndpointContainerUnlock,
+ func() interface{} {
+ return &arvados.GetOptions{Select: []string{"uuid", "state", "priority", "auth_uuid", "locked_by_uuid"}}
+ },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+ {
+ arvados.EndpointSpecimenCreate,
+ func() interface{} { return &arvados.CreateOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.SpecimenCreate(ctx, *opts.(*arvados.CreateOptions))
+ },
+ },
+ {
+ arvados.EndpointSpecimenUpdate,
+ func() interface{} { return &arvados.UpdateOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.SpecimenUpdate(ctx, *opts.(*arvados.UpdateOptions))
+ },
+ },
+ {
+ arvados.EndpointSpecimenGet,
+ func() interface{} { return &arvados.GetOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.SpecimenGet(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+ {
+ arvados.EndpointSpecimenList,
+ func() interface{} { return &arvados.ListOptions{Limit: -1} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.SpecimenList(ctx, *opts.(*arvados.ListOptions))
+ },
+ },
+ {
+ arvados.EndpointSpecimenDelete,
+ func() interface{} { return &arvados.DeleteOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.SpecimenDelete(ctx, *opts.(*arvados.DeleteOptions))
+ },
+ },
+ } {
+ route := route
+ methods := []string{route.endpoint.Method}
+ if route.endpoint.Method == "PATCH" {
+ methods = append(methods, "PUT")
+ }
+ for _, method := range methods {
+ rtr.mux.HandlerFunc(method, "/"+route.endpoint.Path, func(w http.ResponseWriter, req *http.Request) {
+ params, err := rtr.loadRequestParams(req, route.endpoint.AttrsKey)
+ if err != nil {
+ rtr.sendError(w, err)
+ return
+ }
+ opts := route.defaultOpts()
+ err = rtr.transcode(params, opts)
+ if err != nil {
+ rtr.sendError(w, err)
+ return
+ }
+ respOpts, err := rtr.responseOptions(opts)
+ if err != nil {
+ rtr.sendError(w, err)
+ return
+ }
+
+ creds := auth.CredentialsFromRequest(req)
+ ctx := req.Context()
+ ctx = context.WithValue(ctx, auth.ContextKeyCredentials, creds)
+ ctx = arvados.ContextWithRequestID(ctx, req.Header.Get("X-Request-Id"))
+ resp, err := route.exec(ctx, opts)
+ if err != nil {
+ ctxlog.FromContext(ctx).WithError(err).Infof("returning error response for %#v", err)
+ rtr.sendError(w, err)
+ return
+ }
+ rtr.sendResponse(w, resp, respOpts)
+ })
+ }
+ }
+}
+
+func (rtr *router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ r.ParseForm()
+ if m := r.FormValue("_method"); m != "" {
+ r2 := *r
+ r = &r2
+ r.Method = m
+ }
+ rtr.mux.ServeHTTP(w, r)
+}
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
new file mode 100644
index 000000000..97710d265
--- /dev/null
+++ b/lib/controller/router/router_test.go
@@ -0,0 +1,127 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package router
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&RouterSuite{})
+
+type RouterSuite struct {
+ rtr *router
+}
+
+func (s *RouterSuite) SetUpTest(c *check.C) {
+ s.rtr = New(&arvados.Cluster{}, &arvados.NodeProfile{RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true}})
+}
+
+func (s *RouterSuite) TearDownTest(c *check.C) {
+ err := arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil)
+ c.Check(err, check.IsNil)
+}
+
+func (s *RouterSuite) doRequest(c *check.C, token, method, path string, hdrs http.Header, body io.Reader) (*http.Request, *httptest.ResponseRecorder, map[string]interface{}) {
+ req := httptest.NewRequest(method, path, body)
+ for k, v := range hdrs {
+ req.Header[k] = v
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+ rw := httptest.NewRecorder()
+ s.rtr.ServeHTTP(rw, req)
+ c.Logf("response body: %s", rw.Body.String())
+ var jresp map[string]interface{}
+ err := json.Unmarshal(rw.Body.Bytes(), &jresp)
+ c.Check(err, check.IsNil)
+ return req, rw, jresp
+}
+
+func (s *RouterSuite) TestContainerList(c *check.C) {
+ token := arvadostest.ActiveTokenV2
+
+ _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/containers?limit=0`, nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
+ c.Check(jresp["items_available"].(float64) > 2, check.Equals, true)
+ c.Check(jresp["items"], check.HasLen, 0)
+
+ _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/containers?limit=2&select=["uuid","command"]`, nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
+ c.Check(jresp["items_available"].(float64) > 2, check.Equals, true)
+ c.Check(jresp["items"], check.HasLen, 2)
+ item0 := jresp["items"].([]interface{})[0].(map[string]interface{})
+ c.Check(item0["uuid"], check.HasLen, 27)
+ c.Check(item0["command"], check.FitsTypeOf, []interface{}{})
+ c.Check(item0["command"].([]interface{})[0], check.FitsTypeOf, "")
+ c.Check(item0["mounts"], check.IsNil)
+
+ _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/containers`, nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
+ c.Check(jresp["items_available"].(float64) > 2, check.Equals, true)
+ avail := int(jresp["items_available"].(float64))
+ c.Check(jresp["items"], check.HasLen, avail)
+ item0 = jresp["items"].([]interface{})[0].(map[string]interface{})
+ c.Check(item0["uuid"], check.HasLen, 27)
+ c.Check(item0["command"], check.FitsTypeOf, []interface{}{})
+ c.Check(item0["command"].([]interface{})[0], check.FitsTypeOf, "")
+ c.Check(item0["mounts"], check.NotNil)
+}
+
+func (s *RouterSuite) TestContainerLock(c *check.C) {
+ uuid := arvadostest.QueuedContainerUUID
+ token := arvadostest.ActiveTokenV2
+ _, rw, jresp := s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["uuid"], check.HasLen, 27)
+ c.Check(jresp["state"], check.Equals, "Locked")
+ _, rw, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusUnprocessableEntity)
+ c.Check(rw.Body.String(), check.Not(check.Matches), `.*"uuid":.*`)
+ _, rw, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["uuid"], check.HasLen, 27)
+ c.Check(jresp["state"], check.Equals, "Queued")
+ c.Check(jresp["environment"], check.IsNil)
+ _, rw, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusUnprocessableEntity)
+ c.Check(jresp["uuid"], check.IsNil)
+}
+
+func (s *RouterSuite) TestSelectParam(c *check.C) {
+ uuid := arvadostest.QueuedContainerUUID
+ token := arvadostest.ActiveTokenV2
+ for _, sel := range [][]string{
+ {"uuid", "command"},
+ {"uuid", "command", "uuid"},
+ {"", "command", "uuid"},
+ } {
+ j, err := json.Marshal(sel)
+ c.Assert(err, check.IsNil)
+ _, rw, resp := s.doRequest(c, token, "GET", "/arvados/v1/containers/"+uuid+"?select="+string(j), nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+
+ c.Check(resp["uuid"], check.HasLen, 27)
+ c.Check(resp["command"], check.HasLen, 2)
+ c.Check(resp["mounts"], check.IsNil)
+ _, hasMounts := resp["mounts"]
+ c.Check(hasMounts, check.Equals, false)
+ }
+}
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
new file mode 100644
index 000000000..7c23ed170
--- /dev/null
+++ b/lib/controller/rpc/conn.go
@@ -0,0 +1,234 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package rpc
+
+import (
+ "context"
+ "crypto/tls"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+type contextKey string
+
+const ContextKeyCredentials contextKey = "credentials"
+
+type TokenProvider func(context.Context) ([]string, error)
+
+type Conn struct {
+ clusterID string
+ httpClient http.Client
+ baseURL url.URL
+ tokenProvider TokenProvider
+}
+
+func NewConn(clusterID string, url *url.URL, insecure bool, tp TokenProvider) *Conn {
+ transport := http.DefaultTransport
+ if insecure {
+ // It's not safe to copy *http.DefaultTransport
+ // because it has a mutex (which might be locked)
+ // protecting a private map (which might not be nil).
+ // So we build our own, using the Go 1.12 default
+ // values, ignoring any changes the application has
+ // made to http.DefaultTransport.
+ transport = &http.Transport{
+ DialContext: (&net.Dialer{
+ Timeout: 30 * time.Second,
+ KeepAlive: 30 * time.Second,
+ DualStack: true,
+ }).DialContext,
+ MaxIdleConns: 100,
+ IdleConnTimeout: 90 * time.Second,
+ TLSHandshakeTimeout: 10 * time.Second,
+ ExpectContinueTimeout: 1 * time.Second,
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }
+ }
+ return &Conn{
+ clusterID: clusterID,
+ httpClient: http.Client{Transport: transport},
+ baseURL: *url,
+ tokenProvider: tp,
+ }
+}
+
+func (conn *Conn) requestAndDecode(ctx context.Context, dst interface{}, ep arvados.APIEndpoint, body io.Reader, opts interface{}) error {
+ aClient := arvados.Client{
+ Client: &conn.httpClient,
+ Scheme: conn.baseURL.Scheme,
+ APIHost: conn.baseURL.Host,
+ }
+ tokens, err := conn.tokenProvider(ctx)
+ if err != nil {
+ return err
+ } else if len(tokens) == 0 {
+ return fmt.Errorf("bug: token provider returned no tokens and no error")
+ }
+ ctx = context.WithValue(ctx, "Authorization", "Bearer "+tokens[0])
+
+ // Encode opts to JSON and decode from there to a
+ // map[string]interface{}, so we can munge the query params
+ // using the JSON key names specified by opts' struct tags.
+ j, err := json.Marshal(opts)
+ if err != nil {
+ return fmt.Errorf("%T: requestAndDecode: Marshal opts: %s", conn, err)
+ }
+ var params map[string]interface{}
+ err = json.Unmarshal(j, ¶ms)
+ if err != nil {
+ return fmt.Errorf("%T: requestAndDecode: Unmarshal opts: %s", conn, err)
+ }
+ if attrs, ok := params["attrs"]; ok && ep.AttrsKey != "" {
+ params[ep.AttrsKey] = attrs
+ delete(params, "attrs")
+ }
+ if limit, ok := params["limit"].(float64); ok && limit < 0 {
+ // Negative limit means "not specified" here, but some
+ // servers/versions do not accept that, so we need to
+ // remove it entirely.
+ delete(params, "limit")
+ }
+ path := ep.Path
+ if strings.Contains(ep.Path, "/:uuid") {
+ uuid, _ := params["uuid"].(string)
+ path = strings.Replace(path, "/:uuid", "/"+uuid, 1)
+ delete(params, "uuid")
+ }
+ return aClient.RequestAndDecodeContext(ctx, dst, ep.Method, path, body, params)
+}
+
+func (conn *Conn) CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error) {
+ ep := arvados.EndpointCollectionCreate
+ var resp arvados.Collection
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) CollectionUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Collection, error) {
+ ep := arvados.EndpointCollectionUpdate
+ var resp arvados.Collection
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) CollectionGet(ctx context.Context, options arvados.GetOptions) (arvados.Collection, error) {
+ ep := arvados.EndpointCollectionGet
+ var resp arvados.Collection
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) {
+ ep := arvados.EndpointCollectionList
+ var resp arvados.CollectionList
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) {
+ ep := arvados.EndpointCollectionDelete
+ var resp arvados.Collection
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) ContainerCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Container, error) {
+ ep := arvados.EndpointContainerCreate
+ var resp arvados.Container
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) ContainerUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error) {
+ ep := arvados.EndpointContainerUpdate
+ var resp arvados.Container
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
+ ep := arvados.EndpointContainerGet
+ var resp arvados.Container
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) ContainerList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerList, error) {
+ ep := arvados.EndpointContainerList
+ var resp arvados.ContainerList
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) ContainerDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Container, error) {
+ ep := arvados.EndpointContainerDelete
+ var resp arvados.Container
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) ContainerLock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
+ ep := arvados.EndpointContainerLock
+ var resp arvados.Container
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
+ ep := arvados.EndpointContainerUnlock
+ var resp arvados.Container
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
+ ep := arvados.EndpointSpecimenCreate
+ var resp arvados.Specimen
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) SpecimenUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Specimen, error) {
+ ep := arvados.EndpointSpecimenUpdate
+ var resp arvados.Specimen
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) SpecimenGet(ctx context.Context, options arvados.GetOptions) (arvados.Specimen, error) {
+ ep := arvados.EndpointSpecimenGet
+ var resp arvados.Specimen
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
+ ep := arvados.EndpointSpecimenList
+ var resp arvados.SpecimenList
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) SpecimenDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Specimen, error) {
+ ep := arvados.EndpointSpecimenDelete
+ var resp arvados.Specimen
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) APIClientAuthorizationCurrent(ctx context.Context, options arvados.GetOptions) (arvados.APIClientAuthorization, error) {
+ ep := arvados.EndpointAPIClientAuthorizationCurrent
+ var resp arvados.APIClientAuthorization
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
diff --git a/lib/controller/rpc/conn_test.go b/lib/controller/rpc/conn_test.go
new file mode 100644
index 000000000..80e90a043
--- /dev/null
+++ b/lib/controller/rpc/conn_test.go
@@ -0,0 +1,79 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package rpc
+
+import (
+ "context"
+ "net/url"
+ "os"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&RPCSuite{})
+
+const contextKeyTestTokens = "testTokens"
+
+type RPCSuite struct {
+ log logrus.FieldLogger
+ ctx context.Context
+ conn *Conn
+}
+
+func (s *RPCSuite) SetUpTest(c *check.C) {
+ ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c))
+ s.ctx = context.WithValue(ctx, contextKeyTestTokens, []string{arvadostest.ActiveToken})
+ s.conn = NewConn("zzzzz", &url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_TEST_API_HOST")}, true, func(ctx context.Context) ([]string, error) {
+ return ctx.Value(contextKeyTestTokens).([]string), nil
+ })
+}
+
+func (s *RPCSuite) TestCollectionCreate(c *check.C) {
+ coll, err := s.conn.CollectionCreate(s.ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
+ }})
+ c.Check(err, check.IsNil)
+ c.Check(coll.UUID, check.HasLen, 27)
+}
+
+func (s *RPCSuite) TestSpecimenCRUD(c *check.C) {
+ sp, err := s.conn.SpecimenCreate(s.ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "properties": map[string]string{"foo": "bar"},
+ }})
+ c.Check(err, check.IsNil)
+ c.Check(sp.UUID, check.HasLen, 27)
+ c.Check(sp.Properties, check.HasLen, 1)
+ c.Check(sp.Properties["foo"], check.Equals, "bar")
+
+ spGet, err := s.conn.SpecimenGet(s.ctx, arvados.GetOptions{UUID: sp.UUID})
+ c.Check(spGet.UUID, check.Equals, sp.UUID)
+ c.Check(spGet.Properties["foo"], check.Equals, "bar")
+
+ spList, err := s.conn.SpecimenList(s.ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(spList.ItemsAvailable, check.Equals, 1)
+ c.Assert(spList.Items, check.HasLen, 1)
+ c.Check(spList.Items[0].UUID, check.Equals, sp.UUID)
+ c.Check(spList.Items[0].Properties["foo"], check.Equals, "bar")
+
+ anonCtx := context.WithValue(context.Background(), contextKeyTestTokens, []string{arvadostest.AnonymousToken})
+ spList, err = s.conn.SpecimenList(anonCtx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(spList.ItemsAvailable, check.Equals, 0)
+ c.Check(spList.Items, check.HasLen, 0)
+
+ spDel, err := s.conn.SpecimenDelete(s.ctx, arvados.DeleteOptions{UUID: sp.UUID})
+ c.Check(spDel.UUID, check.Equals, sp.UUID)
+}
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index ae89c3d7e..e5fd41712 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -42,6 +42,7 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
NodeProfiles: map[string]arvados.NodeProfile{
"*": nodeProfile,
},
+ EnableBetaController14287: enableBetaController14287,
}, NodeProfile: &nodeProfile}
srv := &httpserver.Server{
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
new file mode 100644
index 000000000..4cdf7c0e1
--- /dev/null
+++ b/sdk/go/arvados/api.go
@@ -0,0 +1,61 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+type APIEndpoint struct {
+ Method string
+ Path string
+ // "new attributes" key for create/update requests
+ AttrsKey string
+}
+
+var (
+ EndpointCollectionCreate = APIEndpoint{"POST", "arvados/v1/collections", "collection"}
+ EndpointCollectionUpdate = APIEndpoint{"PATCH", "arvados/v1/collections/:uuid", "collection"}
+ EndpointCollectionGet = APIEndpoint{"GET", "arvados/v1/collections/:uuid", ""}
+ EndpointCollectionList = APIEndpoint{"GET", "arvados/v1/collections", ""}
+ EndpointCollectionDelete = APIEndpoint{"DELETE", "arvados/v1/collections/:uuid", ""}
+ EndpointSpecimenCreate = APIEndpoint{"POST", "arvados/v1/specimens", "specimen"}
+ EndpointSpecimenUpdate = APIEndpoint{"PATCH", "arvados/v1/specimens/:uuid", "specimen"}
+ EndpointSpecimenGet = APIEndpoint{"GET", "arvados/v1/specimens/:uuid", ""}
+ EndpointSpecimenList = APIEndpoint{"GET", "arvados/v1/specimens", ""}
+ EndpointSpecimenDelete = APIEndpoint{"DELETE", "arvados/v1/specimens/:uuid", ""}
+ EndpointContainerCreate = APIEndpoint{"POST", "arvados/v1/containers", "container"}
+ EndpointContainerUpdate = APIEndpoint{"PATCH", "arvados/v1/containers/:uuid", "container"}
+ EndpointContainerGet = APIEndpoint{"GET", "arvados/v1/containers/:uuid", ""}
+ EndpointContainerList = APIEndpoint{"GET", "arvados/v1/containers", ""}
+ EndpointContainerDelete = APIEndpoint{"DELETE", "arvados/v1/containers/:uuid", ""}
+ EndpointContainerLock = APIEndpoint{"POST", "arvados/v1/containers/:uuid/lock", ""}
+ EndpointContainerUnlock = APIEndpoint{"POST", "arvados/v1/containers/:uuid/unlock", ""}
+ EndpointAPIClientAuthorizationCurrent = APIEndpoint{"GET", "arvados/v1/api_client_authorizations/current", ""}
+)
+
+type GetOptions struct {
+ UUID string `json:"uuid"`
+ Select []string `json:"select"`
+}
+
+type ListOptions struct {
+ Select []string `json:"select"`
+ Filters []Filter `json:"filters"`
+ Limit int `json:"limit"`
+ Offset int `json:"offset"`
+}
+
+type CreateOptions struct {
+ ClusterID string `json:"cluster_id"`
+ EnsureUniqueName bool `json:"ensure_unique_name"`
+ Select []string `json:"select"`
+ Attrs map[string]interface{} `json:"attrs"`
+}
+
+type UpdateOptions struct {
+ UUID string `json:"uuid"`
+ Attrs map[string]interface{} `json:"attrs"`
+}
+
+type DeleteOptions struct {
+ UUID string `json:"uuid"`
+}
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index cbc2ca72f..2ea6baf88 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -35,6 +35,9 @@ type Client struct {
// DefaultSecureClient or InsecureHTTPClient will be used.
Client *http.Client `json:"-"`
+ // Protocol scheme: "http", "https", or "" (https)
+ Scheme string
+
// Hostname (or host:port) of Arvados API server.
APIHost string
@@ -79,6 +82,7 @@ func NewClientFromConfig(cluster *Cluster) (*Client, error) {
return nil, fmt.Errorf("no host in config Services.Controller.ExternalURL: %v", ctrlURL)
}
return &Client{
+ Scheme: ctrlURL.Scheme,
APIHost: ctrlURL.Host,
Insecure: cluster.TLS.Insecure,
}, nil
@@ -105,6 +109,7 @@ func NewClientFromEnv() *Client {
insecure = true
}
return &Client{
+ Scheme: "https",
APIHost: os.Getenv("ARVADOS_API_HOST"),
AuthToken: os.Getenv("ARVADOS_API_TOKEN"),
Insecure: insecure,
@@ -117,7 +122,9 @@ var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
// Do adds Authorization and X-Request-Id headers and then calls
// (*http.Client)Do().
func (c *Client) Do(req *http.Request) (*http.Response, error) {
- if c.AuthToken != "" {
+ if auth, _ := req.Context().Value("Authorization").(string); auth != "" {
+ req.Header.Add("Authorization", auth)
+ } else if c.AuthToken != "" {
req.Header.Add("Authorization", "OAuth2 "+c.AuthToken)
}
@@ -203,6 +210,9 @@ func anythingToValues(params interface{}) (url.Values, error) {
if err != nil {
return nil, err
}
+ if string(j) == "null" {
+ continue
+ }
urlValues.Set(k, string(j))
}
return urlValues, nil
@@ -216,6 +226,10 @@ func anythingToValues(params interface{}) (url.Values, error) {
//
// path must not contain a query string.
func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
+ return c.RequestAndDecodeContext(context.Background(), dst, method, path, body, params)
+}
+
+func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, method, path string, body io.Reader, params interface{}) error {
if body, ok := body.(io.Closer); ok {
// Ensure body is closed even if we error out early
defer body.Close()
@@ -243,6 +257,7 @@ func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.
if err != nil {
return err
}
+ req = req.WithContext(ctx)
req.Header.Set("Content-type", "application/x-www-form-urlencoded")
return c.DoAndDecode(dst, req)
}
@@ -265,13 +280,9 @@ func (c *Client) UpdateBody(rsc resource) io.Reader {
return bytes.NewBufferString(v.Encode())
}
-type contextKey string
-
-var contextKeyRequestID contextKey = "X-Request-Id"
-
func (c *Client) WithRequestID(reqid string) *Client {
cc := *c
- cc.ctx = context.WithValue(cc.context(), contextKeyRequestID, reqid)
+ cc.ctx = ContextWithRequestID(cc.context(), reqid)
return &cc
}
@@ -294,7 +305,11 @@ func (c *Client) httpClient() *http.Client {
}
func (c *Client) apiURL(path string) string {
- return "https://" + c.APIHost + "/" + path
+ scheme := c.Scheme
+ if scheme == "" {
+ scheme = "https"
+ }
+ return scheme + "://" + c.APIHost + "/" + path
}
// DiscoveryDocument is the Arvados server's description of itself.
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 5b6130060..f374eea07 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -73,7 +73,6 @@ func (c *Collection) SizedDigests() ([]SizedDigest, error) {
return sds, scanner.Err()
}
-// CollectionList is an arvados#collectionList resource.
type CollectionList struct {
Items []Collection `json:"items"`
ItemsAvailable int `json:"items_available"`
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 2965d5ecb..d309748f4 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -71,6 +71,8 @@ type Cluster struct {
RequestLimits RequestLimits
Logging Logging
TLS TLS
+
+ EnableBetaController14287 bool
}
type Services struct {
diff --git a/sdk/go/arvados/context.go b/sdk/go/arvados/context.go
new file mode 100644
index 000000000..555cfc8e9
--- /dev/null
+++ b/sdk/go/arvados/context.go
@@ -0,0 +1,17 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "context"
+)
+
+type contextKey string
+
+var contextKeyRequestID contextKey = "X-Request-Id"
+
+func ContextWithRequestID(ctx context.Context, reqid string) context.Context {
+ return context.WithValue(ctx, contextKeyRequestID, reqid)
+}
diff --git a/sdk/go/arvados/error.go b/sdk/go/arvados/error.go
index 9a0485578..5329a5146 100644
--- a/sdk/go/arvados/error.go
+++ b/sdk/go/arvados/error.go
@@ -31,6 +31,10 @@ func (e TransactionError) Error() (s string) {
return
}
+func (e TransactionError) HTTPStatus() int {
+ return e.StatusCode
+}
+
func newTransactionError(req *http.Request, resp *http.Response, buf []byte) *TransactionError {
var e TransactionError
if json.Unmarshal(buf, &e) != nil {
diff --git a/sdk/go/arvados/resource_list.go b/sdk/go/arvados/resource_list.go
index 14ce098cf..505ba51ec 100644
--- a/sdk/go/arvados/resource_list.go
+++ b/sdk/go/arvados/resource_list.go
@@ -4,7 +4,10 @@
package arvados
-import "encoding/json"
+import (
+ "encoding/json"
+ "fmt"
+)
// ResourceListParams expresses which results are requested in a
// list/index API.
@@ -27,7 +30,35 @@ type Filter struct {
Operand interface{}
}
-// MarshalJSON encodes a Filter in the form expected by the API.
+// MarshalJSON encodes a Filter to a JSON array.
func (f *Filter) MarshalJSON() ([]byte, error) {
return json.Marshal([]interface{}{f.Attr, f.Operator, f.Operand})
}
+
+// UnmarshalJSON decodes a JSON array to a Filter.
+func (f *Filter) UnmarshalJSON(data []byte) error {
+ var elements []interface{}
+ err := json.Unmarshal(data, &elements)
+ if err != nil {
+ return err
+ }
+ if len(elements) != 3 {
+ return fmt.Errorf("invalid filter %q: must have 3 elements", data)
+ }
+ attr, ok := elements[0].(string)
+ if !ok {
+ return fmt.Errorf("invalid filter attr %q", elements[0])
+ }
+ op, ok := elements[1].(string)
+ if !ok {
+ return fmt.Errorf("invalid filter operator %q", elements[1])
+ }
+ operand := elements[2]
+ switch operand.(type) {
+ case string, float64, []interface{}:
+ default:
+ return fmt.Errorf("invalid filter operand %q", elements[2])
+ }
+ *f = Filter{attr, op, operand}
+ return nil
+}
diff --git a/sdk/go/arvados/specimen.go b/sdk/go/arvados/specimen.go
new file mode 100644
index 000000000..e320ca2c3
--- /dev/null
+++ b/sdk/go/arvados/specimen.go
@@ -0,0 +1,23 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "time"
+
+type Specimen struct {
+ UUID string `json:"uuid"`
+ OwnerUUID string `json:"owner_uuid"`
+ CreatedAt time.Time `json:"created_at"`
+ ModifiedAt time.Time `json:"modified_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ Properties map[string]interface{} `json:"properties"`
+}
+
+type SpecimenList struct {
+ Items []Specimen `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+}
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
index 3c266e0d3..de3b1e952 100644
--- a/sdk/go/auth/auth.go
+++ b/sdk/go/auth/auth.go
@@ -20,7 +20,7 @@ func NewCredentials() *Credentials {
}
func CredentialsFromRequest(r *http.Request) *Credentials {
- if c, ok := r.Context().Value(contextKeyCredentials).(*Credentials); ok {
+ if c, ok := r.Context().Value(ContextKeyCredentials).(*Credentials); ok {
// preloaded by middleware
return c
}
diff --git a/sdk/go/auth/handlers.go b/sdk/go/auth/handlers.go
index ad1fa5141..9fa501ab7 100644
--- a/sdk/go/auth/handlers.go
+++ b/sdk/go/auth/handlers.go
@@ -11,15 +11,15 @@ import (
type contextKey string
-var contextKeyCredentials contextKey = "credentials"
+var ContextKeyCredentials contextKey = "credentials"
// LoadToken wraps the next handler, adding credentials to the request
// context so subsequent handlers can access them efficiently via
// CredentialsFromRequest.
func LoadToken(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if _, ok := r.Context().Value(contextKeyCredentials).(*Credentials); !ok {
- r = r.WithContext(context.WithValue(r.Context(), contextKeyCredentials, CredentialsFromRequest(r)))
+ if _, ok := r.Context().Value(ContextKeyCredentials).(*Credentials); !ok {
+ r = r.WithContext(context.WithValue(r.Context(), ContextKeyCredentials, CredentialsFromRequest(r)))
}
next.ServeHTTP(w, r)
})
diff --git a/sdk/go/httpserver/error.go b/sdk/go/httpserver/error.go
index 1ccf8c047..b222e18ea 100644
--- a/sdk/go/httpserver/error.go
+++ b/sdk/go/httpserver/error.go
@@ -14,10 +14,7 @@ type ErrorResponse struct {
}
func Error(w http.ResponseWriter, error string, code int) {
- w.Header().Set("Content-Type", "application/json")
- w.Header().Set("X-Content-Type-Options", "nosniff")
- w.WriteHeader(code)
- json.NewEncoder(w).Encode(ErrorResponse{Errors: []string{error}})
+ Errors(w, []string{error}, code)
}
func Errors(w http.ResponseWriter, errors []string, code int) {
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index ab610d65e..c8dd09de8 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -551,7 +551,7 @@ func (kc *KeepClient) httpClient() HTTPClient {
// It's not safe to copy *http.DefaultTransport
// because it has a mutex (which might be locked)
// protecting a private map (which might not be nil).
- // So we build our own, using the Go 1.10 default
+ // So we build our own, using the Go 1.12 default
// values, ignoring any changes the application has
// made to http.DefaultTransport.
Transport: &http.Transport{
@@ -563,7 +563,7 @@ func (kc *KeepClient) httpClient() HTTPClient {
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: tlsTimeout,
- ExpectContinueTimeout: time.Second,
+ ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: arvadosclient.MakeTLSConfig(kc.Arvados.ApiInsecure),
},
}
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 84b578a3e..3261291b5 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -987,7 +987,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
go func() {
_, err := io.Copy(response.Conn, stdinRdr)
if err != nil {
- runner.CrunchLog.Printf("While writing stdin collection to docker container %q", err)
+ runner.CrunchLog.Printf("While writing stdin collection to docker container: %v", err)
runner.stop(nil)
}
stdinRdr.Close()
@@ -997,7 +997,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
go func() {
_, err := io.Copy(response.Conn, bytes.NewReader(stdinJson))
if err != nil {
- runner.CrunchLog.Printf("While writing stdin json to docker container %q", err)
+ runner.CrunchLog.Printf("While writing stdin json to docker container: %v", err)
runner.stop(nil)
}
response.CloseWrite()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list