[ARVADOS] updated: 1.3.0-839-g689901263
Git user
git at public.curoverse.com
Fri Apr 26 17:47:41 UTC 2019
Summary of changes:
.gitignore | 1 +
.licenseignore | 2 +
build/run-build-docker-jobs-image.sh | 2 +-
build/run-tests.sh | 7 +
doc/Rakefile | 25 +-
doc/_config.yml | 6 +-
.../install-keep-balance.html.textile.liquid | 2 +-
doc/sdk/index.html.textile.liquid | 3 +-
doc/sdk/java-v2/example.html.textile.liquid | 49 ++
doc/sdk/java-v2/index.html.textile.liquid | 146 ++++
doc/sdk/java-v2/javadoc.html.textile.liquid | 15 +
doc/sdk/java/index.html.textile.liquid | 4 +-
lib/config/cmd.go | 31 +-
lib/config/cmd_test.go | 28 +-
lib/config/deprecated.go | 2 +-
lib/config/load.go | 22 +-
lib/config/load_test.go | 38 +-
sdk/java-v2/.gitignore | 9 +
sdk/java-v2/.licenseignore | 4 +
sdk/java-v2/COPYING | 15 +
sdk/java-v2/README.md | 115 +++
agpl-3.0.txt => sdk/java-v2/agpl-3.0.txt | 0
apache-2.0.txt => sdk/java-v2/apache-2.0.txt | 0
sdk/java-v2/build.gradle | 113 +++
sdk/java-v2/gradle.properties | 8 +
sdk/java-v2/settings.gradle | 1 +
.../arvados/client/api/client/BaseApiClient.java | 83 ++
.../client/api/client/BaseStandardApiClient.java | 153 ++++
.../client/api/client/CollectionsApiClient.java | 45 ++
.../client/api/client/CountingFileRequestBody.java | 84 ++
.../arvados/client/api/client/GroupsApiClient.java | 61 ++
.../client/api/client/KeepServerApiClient.java | 54 ++
.../client/api/client/KeepServicesApiClient.java | 43 +
.../client/api/client/KeepWebApiClient.java | 37 +
.../client/api/client/ProgressListener.java | 14 +
.../arvados/client/api/client/UsersApiClient.java | 51 ++
.../api/client/factory/OkHttpClientFactory.java | 89 +++
.../org/arvados/client/api/model/ApiError.java | 42 +
.../org/arvados/client/api/model/Collection.java | 137 ++++
.../arvados/client/api/model/CollectionList.java | 32 +
.../java/org/arvados/client/api/model/Group.java | 319 ++++++++
.../org/arvados/client/api/model/GroupList.java | 32 +
.../java/org/arvados/client/api/model/Item.java | 123 +++
.../org/arvados/client/api/model/ItemList.java | 80 ++
.../org/arvados/client/api/model/KeepService.java | 77 ++
.../arvados/client/api/model/KeepServiceList.java | 32 +
.../client/api/model/RuntimeConstraints.java | 60 ++
.../java/org/arvados/client/api/model/User.java | 147 ++++
.../org/arvados/client/api/model/UserList.java | 32 +
.../client/api/model/argument/Argument.java | 24 +
.../client/api/model/argument/ContentsGroup.java | 63 ++
.../arvados/client/api/model/argument/Filter.java | 118 +++
.../client/api/model/argument/ListArgument.java | 123 +++
.../client/api/model/argument/UntrashGroup.java | 28 +
.../java/org/arvados/client/common/Characters.java | 21 +
.../java/org/arvados/client/common/Headers.java | 15 +
.../java/org/arvados/client/common/Patterns.java | 19 +
.../org/arvados/client/config/ConfigProvider.java | 40 +
.../client/config/ExternalConfigProvider.java | 181 +++++
.../arvados/client/config/FileConfigProvider.java | 107 +++
.../client/exception/ArvadosApiException.java | 25 +
.../client/exception/ArvadosClientException.java | 27 +
.../org/arvados/client/facade/ArvadosFacade.java | 307 ++++++++
.../client/logic/collection/CollectionFactory.java | 134 ++++
.../arvados/client/logic/collection/FileToken.java | 60 ++
.../client/logic/collection/ManifestDecoder.java | 74 ++
.../client/logic/collection/ManifestFactory.java | 67 ++
.../client/logic/collection/ManifestStream.java | 45 ++
.../arvados/client/logic/keep/FileDownloader.java | 256 ++++++
.../client/logic/keep/FileTransferHandler.java | 55 ++
.../arvados/client/logic/keep/FileUploader.java | 101 +++
.../org/arvados/client/logic/keep/KeepClient.java | 244 ++++++
.../org/arvados/client/logic/keep/KeepLocator.java | 86 ++
.../DownloadFolderAlreadyExistsException.java | 24 +
.../keep/exception/FileAlreadyExistsException.java | 23 +
.../java/org/arvados/client/utils/FileMerge.java | 26 +
.../java/org/arvados/client/utils/FileSplit.java | 44 ++
sdk/java-v2/src/main/resources/reference.conf | 23 +
.../api/client/BaseStandardApiClientTest.java | 50 ++
.../api/client/CollectionsApiClientTest.java | 112 +++
.../client/api/client/GroupsApiClientTest.java | 95 +++
.../client/api/client/KeepServerApiClientTest.java | 70 ++
.../api/client/KeepServicesApiClientTest.java | 78 ++
.../client/api/client/UsersApiClientTest.java | 126 +++
.../client/factory/OkHttpClientFactoryTest.java | 96 +++
.../facade/ArvadosFacadeIntegrationTest.java | 258 ++++++
.../arvados/client/facade/ArvadosFacadeTest.java | 200 +++++
.../client/junit/categories/IntegrationTests.java | 10 +
.../client/logic/collection/FileTokenTest.java | 42 +
.../logic/collection/ManifestDecoderTest.java | 108 +++
.../logic/collection/ManifestFactoryTest.java | 40 +
.../logic/collection/ManifestStreamTest.java | 27 +
.../client/logic/keep/FileDownloaderTest.java | 145 ++++
.../arvados/client/logic/keep/KeepClientTest.java | 118 +++
.../arvados/client/logic/keep/KeepLocatorTest.java | 57 ++
.../client/test/utils/ApiClientTestUtils.java | 45 ++
.../test/utils/ArvadosClientIntegrationTest.java | 28 +
.../utils/ArvadosClientMockedWebServerTest.java | 27 +
.../client/test/utils/ArvadosClientUnitTest.java | 24 +
.../arvados/client/test/utils/FileTestUtils.java | 50 ++
.../arvados/client/test/utils/RequestMethod.java | 13 +
.../org/arvados/client/utils/FileMergeTest.java | 48 ++
.../org/arvados/client/utils/FileSplitTest.java | 48 ++
sdk/java-v2/src/test/resources/application.conf | 10 +
.../resources/integration-tests-application.conf | 23 +
.../integration-tests-application.conf.example | 16 +
.../org.mockito.plugins.MockMaker | 1 +
.../api/client/collections-create-manifest.json | 22 +
.../api/client/collections-create-simple.json | 22 +
.../api/client/collections-download-file.json | 22 +
.../arvados/client/api/client/collections-get.json | 22 +
.../client/api/client/collections-list.json | 871 +++++++++++++++++++++
.../org/arvados/client/api/client/groups-get.json | 21 +
.../org/arvados/client/api/client/groups-list.json | 430 ++++++++++
.../client/api/client/keep-client-test-file.txt | 1 +
.../client/keep-services-accessible-disk-only.json | 42 +
.../api/client/keep-services-accessible.json | 42 +
.../client/api/client/keep-services-get.json | 16 +
.../client/api/client/keep-services-list.json | 58 ++
.../api/client/keep-services-not-accessible.json | 9 +
.../arvados/client/api/client/users-create.json | 26 +
.../org/arvados/client/api/client/users-get.json | 26 +
.../org/arvados/client/api/client/users-list.json | 115 +++
.../arvados/client/api/client/users-system.json | 24 +
.../src/test/resources/selfsigned.keystore.jks | Bin 0 -> 2247 bytes
sdk/java-v2/test-in-docker.sh | 48 ++
services/keep-balance/balance.go | 43 +
services/keep-balance/balance_run_test.go | 58 ++
services/keep-balance/server.go | 11 +-
services/keepstore/azure_blob_volume.go | 38 +-
services/keepstore/azure_blob_volume_test.go | 24 +-
services/keepstore/handlers.go | 20 +-
.../crunchstat_summary/summarizer.py | 21 +-
133 files changed, 8782 insertions(+), 50 deletions(-)
create mode 100644 doc/sdk/java-v2/example.html.textile.liquid
create mode 100644 doc/sdk/java-v2/index.html.textile.liquid
create mode 100644 doc/sdk/java-v2/javadoc.html.textile.liquid
create mode 100644 sdk/java-v2/.gitignore
create mode 100644 sdk/java-v2/.licenseignore
create mode 100644 sdk/java-v2/COPYING
create mode 100644 sdk/java-v2/README.md
copy agpl-3.0.txt => sdk/java-v2/agpl-3.0.txt (100%)
copy apache-2.0.txt => sdk/java-v2/apache-2.0.txt (100%)
create mode 100644 sdk/java-v2/build.gradle
create mode 100644 sdk/java-v2/gradle.properties
create mode 100644 sdk/java-v2/settings.gradle
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/BaseApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/BaseStandardApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/CollectionsApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/CountingFileRequestBody.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/GroupsApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepServerApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepServicesApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/ProgressListener.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/UsersApiClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/client/factory/OkHttpClientFactory.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/ApiError.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/Collection.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/CollectionList.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/Group.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/GroupList.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/Item.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/ItemList.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/KeepService.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/KeepServiceList.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/RuntimeConstraints.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/User.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/UserList.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/Argument.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/ContentsGroup.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/Filter.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/ListArgument.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/UntrashGroup.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/common/Characters.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/common/Headers.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/common/Patterns.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/config/ConfigProvider.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/config/ExternalConfigProvider.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/config/FileConfigProvider.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/exception/ArvadosApiException.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/exception/ArvadosClientException.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/facade/ArvadosFacade.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/collection/CollectionFactory.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/collection/FileToken.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/collection/ManifestDecoder.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/collection/ManifestFactory.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/collection/ManifestStream.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileTransferHandler.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileUploader.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/keep/KeepClient.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/keep/KeepLocator.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/keep/exception/DownloadFolderAlreadyExistsException.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/logic/keep/exception/FileAlreadyExistsException.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/utils/FileMerge.java
create mode 100644 sdk/java-v2/src/main/java/org/arvados/client/utils/FileSplit.java
create mode 100644 sdk/java-v2/src/main/resources/reference.conf
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/api/client/BaseStandardApiClientTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/api/client/CollectionsApiClientTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/api/client/GroupsApiClientTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepServerApiClientTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepServicesApiClientTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/api/client/UsersApiClientTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/api/client/factory/OkHttpClientFactoryTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/facade/ArvadosFacadeIntegrationTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/facade/ArvadosFacadeTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/junit/categories/IntegrationTests.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/logic/collection/FileTokenTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/logic/collection/ManifestDecoderTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/logic/collection/ManifestFactoryTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/logic/collection/ManifestStreamTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/logic/keep/FileDownloaderTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/logic/keep/KeepClientTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/logic/keep/KeepLocatorTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/test/utils/ApiClientTestUtils.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/test/utils/ArvadosClientIntegrationTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/test/utils/ArvadosClientMockedWebServerTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/test/utils/ArvadosClientUnitTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/test/utils/FileTestUtils.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/test/utils/RequestMethod.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/utils/FileMergeTest.java
create mode 100644 sdk/java-v2/src/test/java/org/arvados/client/utils/FileSplitTest.java
create mode 100644 sdk/java-v2/src/test/resources/application.conf
create mode 100644 sdk/java-v2/src/test/resources/integration-tests-application.conf
create mode 100644 sdk/java-v2/src/test/resources/integration-tests-application.conf.example
create mode 100644 sdk/java-v2/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/collections-create-manifest.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/collections-create-simple.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/collections-download-file.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/collections-get.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/collections-list.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/groups-get.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/groups-list.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/keep-client-test-file.txt
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/keep-services-accessible-disk-only.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/keep-services-accessible.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/keep-services-get.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/keep-services-list.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/keep-services-not-accessible.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/users-create.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/users-get.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/users-list.json
create mode 100644 sdk/java-v2/src/test/resources/org/arvados/client/api/client/users-system.json
create mode 100644 sdk/java-v2/src/test/resources/selfsigned.keystore.jks
create mode 100755 sdk/java-v2/test-in-docker.sh
via 689901263ebfdd996da3711236615038e6245db3 (commit)
via 1f62181aa86195fe76d27d351a9135e44940c8ef (commit)
via 2f3aaf928716c35b1ca07803dce8623b65f2e114 (commit)
via 6f18406060f2ac32d505db14ceab97b08431ce04 (commit)
via 86f72edb9b3bc75af47475bf83da07ce3ce86231 (commit)
via 96be8f8e8ca201f82cad8e41f8ce3ffa1d837844 (commit)
via ff875d9f7ca4b04a40fa3205913db1b68ea00db3 (commit)
via 0e6733b5009d22022c3b68bbd76b75ebdf5a12db (commit)
via b6a55c2430033ee1a17b2266b529b425e641d19f (commit)
via 6bd68f542545ae8694da311c7a6757a8a7a6e7b5 (commit)
via 379f15edca3764e7236cdd89937efd5bade04059 (commit)
via ceabb22f25be2573c1d37dafaf2b34d2d89e1c62 (commit)
via ed35acd2e173b4fae4499d6ec21211cd2076e91c (commit)
via 6c1b0fb33cc6dcd5430233cf4399fe93fd863a9b (commit)
via 2266cd0b5066ef366c5ee76b5818b9aea344f9ae (commit)
via c3ba32d18b341a28d2b8c31930063aba14e75a30 (commit)
via d8a49eed1dfe8dcb8b10b596dbda44929e78e72b (commit)
via cf1dcfb8715822f7ac5fe4fce067197dd84bad54 (commit)
via 8b65fa06a3e82f8ff0830f11142b9046e848d5d0 (commit)
via 30298819bc92b5ec90c8544bee7e102cb64e888d (commit)
via d795b68eea8475b350085a652cfd0f643eac77e5 (commit)
via 849e41d7ee6849e76a4beae075d825509ac51067 (commit)
via 547a2659a7003ffbb6134170388e98e354579ef1 (commit)
via 60321351d092de49566423cd79ffd450ed5845a4 (commit)
via 7bf3e08491c5b8cf9aa6409734a514b94bbfbcd4 (commit)
via 36c3ad6c921ed0eaf1199f80b1cca9c817518d02 (commit)
via 72ff1a0bace622efe975820e41c456d9a7934a75 (commit)
via 0df5f0feeced5bff0adfb806dae2d3811257827f (commit)
via aade7ad1ade9b6a1f2a0f295368f92f130e0ed9a (commit)
via ef9ad9c4771dbcd90d884fdb2c55344434d59bd9 (commit)
via 96fefe6c6fc2e3bd7e4efbb0b6a7dedb9221aff7 (commit)
via d66aa15210d809c64a046b1133865015095ac172 (commit)
via a7e7392d54753f52b44a207fec134bce931a5a06 (commit)
via 6289afb4f83c5412552f1605046c3c8fb75ccd3a (commit)
via 9a7e2a24f5f3d261e554ac3815b7e2a4c2e24503 (commit)
via e030e175bf4093eb7cdecd9c534548ad2b9a2b05 (commit)
via 40adb6fe873edb0b09d056a9f794b818b30dffbb (commit)
via 8cd2bd00d2ef2b816bc4f9af008f97f44e258e17 (commit)
via f017af924e61134b1380fe85a663b6edaa561e91 (commit)
via 16f704326f44fd1e5e5e60b936c9b5895d6a6ff8 (commit)
via 55f103e336ca9fb8bf1720d2ef4ee8dd4e221118 (commit)
via c2f91c556a0eec39634f3e89091b6addc850fe92 (commit)
via 22ae07f51e144baa27ddc56d56a8c6ac6f63ad58 (commit)
from cb13593decb097f501c0a1d64510a653b3233395 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 689901263ebfdd996da3711236615038e6245db3
Merge: 6bd68f542 1f62181aa
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 26 13:47:21 2019 -0400
15003: Merge branch 'master' into 15003-preprocess-config
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit 6bd68f542545ae8694da311c7a6757a8a7a6e7b5
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 24 13:34:02 2019 -0400
15003: Warn about unknown keys and bad value types.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/cmd.go b/lib/config/cmd.go
index e3419cee5..41a1d7d21 100644
--- a/lib/config/cmd.go
+++ b/lib/config/cmd.go
@@ -63,7 +63,7 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
err = fmt.Errorf("usage: %s <config-src.yaml && echo 'no changes needed'", prog)
return 2
}
- log := ctxlog.New(stderr, "text", "info")
+ log := &plainLogger{w: stderr}
buf, err := ioutil.ReadAll(stdin)
if err != nil {
return 1
@@ -72,7 +72,7 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
if err != nil {
return 1
}
- withDepr, err := load(bytes.NewBuffer(buf), log, true)
+ withDepr, err := load(bytes.NewBuffer(buf), nil, true)
if err != nil {
return 1
}
@@ -91,9 +91,28 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
}
diff, err := cmd.CombinedOutput()
- if err == nil {
- return 0
+ if bytes.HasPrefix(diff, []byte("--- ")) {
+ fmt.Fprintln(stdout, "Your configuration is relying on deprecated entries. Suggest making the following changes.")
+ stdout.Write(diff)
+ return 1
+ } else if len(diff) > 0 {
+ fmt.Fprintf(stderr, "Unexpected diff output:\n%s", diff)
+ return 1
+ } else if err != nil {
+ return 1
+ }
+ if log.used {
+ return 1
}
- _, err = stdout.Write(diff)
- return 1
+ return 0
+}
+
+type plainLogger struct {
+ w io.Writer
+ used bool
+}
+
+func (pl *plainLogger) Warnf(format string, args ...interface{}) {
+ pl.used = true
+ fmt.Fprintf(pl.w, format+"\n", args...)
}
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index 925bbe648..bedcc0dd8 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -55,7 +55,31 @@ Clusters:
c.Check(stdout.String(), check.Matches, `(?ms).*API:\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
}
-func (s *CommandSuite) TestUnknownKey(c *check.C) {
+func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ in := `
+Clusters:
+ z1234:
+ Bogus1: foo
+ BogusSection:
+ Bogus2: foo
+ API:
+ Bogus3:
+ Bogus4: true
+ PostgreSQL:
+ ConnectionPool:
+ {Bogus5: true}
+`
+ code := CheckCommand.RunCommand("arvados config-check", nil, bytes.NewBufferString(in), &stdout, &stderr)
+ c.Log(stderr.String())
+ c.Check(code, check.Equals, 1)
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.Bogus1\n.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.BogusSection\n.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.API.Bogus3\n.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*unexpected object in config entry: Clusters.z1234.PostgreSQL.ConnectionPool\n.*`)
+}
+
+func (s *CommandSuite) TestDumpUnknownKey(c *check.C) {
var stdout, stderr bytes.Buffer
in := `
Clusters:
@@ -65,7 +89,7 @@ Clusters:
`
code := DumpCommand.RunCommand("arvados config-dump", nil, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
- c.Check(stderr.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.UnknownKey.*`)
c.Check(stdout.String(), check.Matches, `(?ms)Clusters:\n z1234:\n.*`)
c.Check(stdout.String(), check.Matches, `(?ms).*\n *ManagementToken: secret\n.*`)
c.Check(stdout.String(), check.Not(check.Matches), `(?ms).*UnknownKey.*`)
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 2fc839cf2..c8f943f3c 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -45,7 +45,7 @@ func applyDeprecatedConfig(cfg *arvados.Config, configdata []byte, log logger) e
for name, np := range dcluster.NodeProfiles {
if name == "*" || name == os.Getenv("ARVADOS_NODE_PROFILE") || name == hostname {
name = "localhost"
- } else {
+ } else if log != nil {
log.Warnf("overriding Clusters.%s.Services using Clusters.%s.NodeProfiles.%s (guessing %q is a hostname)", id, id, name, name)
}
applyDeprecatedNodeProfile(name, np.RailsAPI, &cluster.Services.RailsAPI)
diff --git a/lib/config/load.go b/lib/config/load.go
index 644dcfdb8..526a050fb 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -78,6 +78,7 @@ func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error
if err != nil {
return nil, fmt.Errorf("loading config data: %s", err)
}
+ logExtraKeys(log, merged, src, "")
err = mergo.Merge(&merged, src, mergo.WithOverride)
if err != nil {
return nil, fmt.Errorf("merging config data: %s", err)
@@ -121,9 +122,28 @@ func checkKeyConflict(label string, m map[string]string) error {
for k := range m {
k = strings.ToLower(k)
if saw[k] {
- return fmt.Errorf("%s: multiple keys with tolower(key)==%q (use same case as defaults)", label, k)
+ return fmt.Errorf("%s: multiple entries for %q (fix by using same capitalization as default/example file)", label, k)
}
saw[k] = true
}
return nil
}
+
+func logExtraKeys(log logger, expected, supplied map[string]interface{}, prefix string) {
+ if log == nil {
+ return
+ }
+ for k, vsupp := range supplied {
+ if vexp, ok := expected[k]; !ok {
+ log.Warnf("deprecated or unknown config entry: %s%s", prefix, k)
+ } else if vsupp, ok := vsupp.(map[string]interface{}); !ok {
+ // if vsupp is a map but vexp isn't map, this
+ // will be caught elsewhere; see TestBadType.
+ continue
+ } else if vexp, ok := vexp.(map[string]interface{}); !ok {
+ log.Warnf("unexpected object in config entry: %s%s", prefix, k)
+ } else {
+ logExtraKeys(log, vexp, vsupp, prefix+k+".")
+ }
+ }
+}
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index fddf3660d..2bf341fe1 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -62,7 +62,43 @@ Clusters:
dbname: dbname
host: host
`), ctxlog.TestLogger(c))
- c.Assert(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple keys with .*"(dbname|host)".*`)
+ c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
+}
+
+func (s *LoadSuite) TestBadType(c *check.C) {
+ for _, data := range []string{`
+Clusters:
+ zzzzz:
+ PostgreSQL: true
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: true
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: "foo"
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: []
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: [] # {foo: bar} isn't caught here; we rely on config-check
+`,
+ } {
+ c.Log(data)
+ v, err := Load(bytes.NewBufferString(data), ctxlog.TestLogger(c))
+ if v != nil {
+ c.Logf("%#v", v.Clusters["zzzzz"].PostgreSQL.ConnectionPool)
+ }
+ c.Check(err, check.ErrorMatches, `.*cannot unmarshal .*PostgreSQL.*`)
+ }
}
func (s *LoadSuite) TestMovedKeys(c *check.C) {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list