[arvados] created: 2.7.0-5656-gcd98fa4173
git repository hosting
git at public.arvados.org
Thu Feb 8 17:41:12 UTC 2024
at cd98fa41731b9fbba04c282977b3a7a1b52db0e8 (commit)
commit cd98fa41731b9fbba04c282977b3a7a1b52db0e8
Author: szlenkj <jakub.szlenk at contractors.roche.com>
Date: Fri Jan 26 08:30:33 2024 +0100
Download file with resume
Arvados-DCO-1.1-Signed-off-by: Jakub Szlenk jakubszlenk at gmail.com
diff --git a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java
index 2595956074..ad37dad2bb 100644
--- a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java
+++ b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java
@@ -34,24 +34,25 @@ public class KeepWebApiClient extends BaseApiClient {
return newFileCall(request);
}
- public byte[] downloadPartial(String collectionUuid, String filePathName, long offset) throws IOException {
+ public InputStream get(String collectionUuid, String filePathName, long start, Long end) throws IOException {
Request.Builder builder = this.getRequestBuilder();
- if (offset > 0) {
- builder.addHeader("Range", "bytes=" + offset + "-");
+ String rangeValue = "bytes=" + start + "-";
+ if (end != null) {
+ rangeValue += end;
}
+ builder.addHeader("Range", rangeValue);
Request request = builder.url(this.getUrlBuilder(collectionUuid, filePathName).build()).get().build();
- try (Response response = client.newCall(request).execute()) {
- if (!response.isSuccessful()) {
- throw new IOException("Failed to download file: " + response);
- }
- try (ResponseBody body = response.body()) {
- if (body != null) {
- return body.bytes();
- } else {
- throw new IOException("Response body is null for request: " + request);
- }
- }
+ Response response = client.newCall(request).execute();
+ if (!response.isSuccessful()) {
+ response.close();
+ throw new IOException("Failed to download file: " + response);
}
+ ResponseBody body = response.body();
+ if (body == null) {
+ response.close();
+ throw new IOException("Response body is null for request: " + request);
+ }
+ return body.byteStream();
}
public String delete(String collectionUuid, String filePathName) {
diff --git a/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java b/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java
index 3fe4e6433a..5bfcabc109 100644
--- a/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java
+++ b/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java
@@ -20,7 +20,6 @@ import org.arvados.client.logic.keep.exception.DownloadFolderAlreadyExistsExcept
import org.arvados.client.logic.keep.exception.FileAlreadyExistsException;
import org.slf4j.Logger;
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -73,9 +72,9 @@ public class FileDownloader {
return downloadedFile;
}
- public File downloadFileWithResume(String collectionUuid, String fileName, String pathToDownloadFolder, long offset, int bufferSize) throws IOException {
- if (bufferSize <= 0) {
- throw new IllegalArgumentException("Buffer size must be greater than 0");
+ public File downloadFileWithResume(String collectionUuid, String fileName, String pathToDownloadFolder, long start, Long end) throws IOException {
+ if (end != null && end < start) {
+ throw new IllegalArgumentException("End index must be greater than or equal to the start index");
}
File destinationFile = new File(pathToDownloadFolder, fileName);
@@ -87,14 +86,17 @@ public class FileDownloader {
}
}
- try (RandomAccessFile outputFile = new RandomAccessFile(destinationFile, "rw")) {
- outputFile.seek(offset);
+ try (RandomAccessFile outputFile = new RandomAccessFile(destinationFile, "rw");
+ InputStream inputStream = keepWebApiClient.get(collectionUuid, fileName, start, end)) {
+ outputFile.seek(start);
- byte[] buffer = new byte[bufferSize];
+ long remaining = (end == null) ? Long.MAX_VALUE : end - start + 1;
+ byte[] buffer = new byte[4096];
int bytesRead;
- InputStream inputStream = new ByteArrayInputStream(keepWebApiClient.downloadPartial(collectionUuid, fileName, offset));
- while ((bytesRead = inputStream.read(buffer)) != -1) {
- outputFile.write(buffer, 0, bytesRead);
+ while ((bytesRead = inputStream.read(buffer)) != -1 && remaining > 0) {
+ int bytesToWrite = (int) Math.min(bytesRead, remaining);
+ outputFile.write(buffer, 0, bytesToWrite);
+ remaining -= bytesToWrite;
}
}
diff --git a/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java b/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java
index e796c69032..9b6b4fa17f 100644
--- a/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java
+++ b/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java
@@ -10,7 +10,10 @@ package org.arvados.client.api.client;
import org.arvados.client.test.utils.ArvadosClientMockedWebServerTest;
import org.junit.Test;
+import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
import java.nio.file.Files;
import okhttp3.mockwebserver.MockResponse;
@@ -18,7 +21,7 @@ import okio.Buffer;
import static org.arvados.client.test.utils.ApiClientTestUtils.getResponse;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertNotNull;
public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest {
@@ -46,7 +49,8 @@ public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest {
// given
String collectionUuid = "some-collection-uuid";
String filePathName = "sample-file-path";
- long offset = 1024;
+ long start = 1024;
+ Long end = null;
byte[] expectedData = "test data".getBytes();
@@ -54,12 +58,24 @@ public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest {
server.enqueue(new MockResponse().setBody(buffer));
// when
- byte[] actualData = client.downloadPartial(collectionUuid, filePathName, offset);
+ InputStream inputStream = client.get(collectionUuid, filePathName, start, end);
+ byte[] actualData = inputStreamToByteArray(inputStream);
// then
assertNotNull(actualData);
- assertEquals(new String(expectedData), new String(actualData));
+ assertArrayEquals(expectedData, actualData);
}
}
+ private byte[] inputStreamToByteArray(InputStream inputStream) throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ int nRead;
+ byte[] data = new byte[1024];
+ while ((nRead = inputStream.read(data, 0, data.length)) != -1) {
+ buffer.write(data, 0, nRead);
+ }
+ buffer.flush();
+ return buffer.toByteArray();
+ }
+
}
diff --git a/sdk/java-v2/src/test/java/org/arvados/client/logic/keep/FileDownloaderTest.java b/sdk/java-v2/src/test/java/org/arvados/client/logic/keep/FileDownloaderTest.java
index 0fb1f0206c..2f2f81294c 100644
--- a/sdk/java-v2/src/test/java/org/arvados/client/logic/keep/FileDownloaderTest.java
+++ b/sdk/java-v2/src/test/java/org/arvados/client/logic/keep/FileDownloaderTest.java
@@ -27,8 +27,12 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -36,6 +40,7 @@ import java.util.UUID;
import static org.arvados.client.test.utils.FileTestUtils.*;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertArrayEquals;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
@@ -90,7 +95,7 @@ public class FileDownloaderTest {
for(int i = 0; i < downloadedFiles.size(); i ++) {
File downloaded = new File(collectionDir + Characters.SLASH + files.get(i).getName());
- Assert.assertArrayEquals(FileUtils.readFileToByteArray(downloaded), FileUtils.readFileToByteArray(files.get(i)));
+ assertArrayEquals(FileUtils.readFileToByteArray(downloaded), FileUtils.readFileToByteArray(files.get(i)));
}
}
@@ -110,7 +115,36 @@ public class FileDownloaderTest {
//then
Assert.assertTrue(downloadedFile.exists());
Assert.assertEquals(file.getName(), downloadedFile.getName());
- Assert.assertArrayEquals(FileUtils.readFileToByteArray(downloadedFile), FileUtils.readFileToByteArray(file));
+ assertArrayEquals(FileUtils.readFileToByteArray(downloadedFile), FileUtils.readFileToByteArray(file));
+ }
+
+ @Test
+ public void testDownloadFileWithResume() throws Exception {
+ //given
+ String collectionUuid = "some-collection-uuid";
+ String fileName = "sample-file-name";
+ String pathToDownloadFolder = "downloads";
+ long start = 1024;
+ Long end = null;
+
+ byte[] expectedData = "test data".getBytes();
+ InputStream inputStream = new ByteArrayInputStream(expectedData);
+
+ when(keepWebApiClient.get(collectionUuid, fileName, start, end)).thenReturn(inputStream);
+
+ //when
+ File downloadedFile = fileDownloader.downloadFileWithResume(collectionUuid, fileName, pathToDownloadFolder, start, end);
+
+ //then
+ Assert.assertNotNull(downloadedFile);
+ Assert.assertTrue(downloadedFile.exists());
+ Assert.assertEquals(downloadedFile.length(), expectedData.length);
+
+ byte[] actualData = Files.readAllBytes(downloadedFile.toPath());
+ assertArrayEquals(expectedData, actualData);
+
+ Files.delete(downloadedFile.toPath());
+ Files.delete(Paths.get(pathToDownloadFolder));
}
@After
commit b584bdc5d741d713413db74ce4486201ed4f9cc1
Author: szlenkj <jakub.szlenk at contractors.roche.com>
Date: Fri Jan 26 08:30:33 2024 +0100
Download file with resume
Arvados-DCO-1.1-Signed-off-by: Jakub Szlenk jakubszlenk at gmail.com
diff --git a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java
index 2c3168649f..2595956074 100644
--- a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java
+++ b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java
@@ -10,9 +10,13 @@ package org.arvados.client.api.client;
import okhttp3.HttpUrl;
import okhttp3.Request;
import okhttp3.RequestBody;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
+
import org.arvados.client.config.ConfigProvider;
import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
public class KeepWebApiClient extends BaseApiClient {
@@ -30,6 +34,26 @@ public class KeepWebApiClient extends BaseApiClient {
return newFileCall(request);
}
+ public byte[] downloadPartial(String collectionUuid, String filePathName, long offset) throws IOException {
+ Request.Builder builder = this.getRequestBuilder();
+ if (offset > 0) {
+ builder.addHeader("Range", "bytes=" + offset + "-");
+ }
+ Request request = builder.url(this.getUrlBuilder(collectionUuid, filePathName).build()).get().build();
+ try (Response response = client.newCall(request).execute()) {
+ if (!response.isSuccessful()) {
+ throw new IOException("Failed to download file: " + response);
+ }
+ try (ResponseBody body = response.body()) {
+ if (body != null) {
+ return body.bytes();
+ } else {
+ throw new IOException("Response body is null for request: " + request);
+ }
+ }
+ }
+ }
+
public String delete(String collectionUuid, String filePathName) {
Request request = getRequestBuilder()
.url(getUrlBuilder(collectionUuid, filePathName).build())
diff --git a/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java b/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java
index c1e8849e39..3fe4e6433a 100644
--- a/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java
+++ b/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java
@@ -20,9 +20,12 @@ import org.arvados.client.logic.keep.exception.DownloadFolderAlreadyExistsExcept
import org.arvados.client.logic.keep.exception.FileAlreadyExistsException;
import org.slf4j.Logger;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@@ -70,6 +73,34 @@ public class FileDownloader {
return downloadedFile;
}
+ public File downloadFileWithResume(String collectionUuid, String fileName, String pathToDownloadFolder, long offset, int bufferSize) throws IOException {
+ if (bufferSize <= 0) {
+ throw new IllegalArgumentException("Buffer size must be greater than 0");
+ }
+
+ File destinationFile = new File(pathToDownloadFolder, fileName);
+
+ if (!destinationFile.exists()) {
+ boolean isCreated = destinationFile.createNewFile();
+ if (!isCreated) {
+ throw new IOException("Failed to create new file: " + destinationFile.getAbsolutePath());
+ }
+ }
+
+ try (RandomAccessFile outputFile = new RandomAccessFile(destinationFile, "rw")) {
+ outputFile.seek(offset);
+
+ byte[] buffer = new byte[bufferSize];
+ int bytesRead;
+ InputStream inputStream = new ByteArrayInputStream(keepWebApiClient.downloadPartial(collectionUuid, fileName, offset));
+ while ((bytesRead = inputStream.read(buffer)) != -1) {
+ outputFile.write(buffer, 0, bytesRead);
+ }
+ }
+
+ return destinationFile;
+ }
+
public List<File> downloadFilesFromCollectionUsingKeepWeb(String collectionUuid, String pathToDownloadFolder) {
String collectionTargetDir = setTargetDirectory(collectionUuid, pathToDownloadFolder).getAbsolutePath();
List<FileToken> fileTokens = listFileInfoFromCollection(collectionUuid);
diff --git a/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java b/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java
index 07b7b25339..e796c69032 100644
--- a/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java
+++ b/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java
@@ -13,12 +13,17 @@ import org.junit.Test;
import java.io.File;
import java.nio.file.Files;
+import okhttp3.mockwebserver.MockResponse;
+import okio.Buffer;
+
import static org.arvados.client.test.utils.ApiClientTestUtils.getResponse;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest {
- private KeepWebApiClient client = new KeepWebApiClient(CONFIG);
+ private final KeepWebApiClient client = new KeepWebApiClient(CONFIG);
@Test
public void uploadFile() throws Exception {
@@ -36,4 +41,25 @@ public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest {
assertThat(uploadResponse).isEqualTo("Created");
}
+ @Test
+ public void downloadPartialIsPerformedSuccessfully() throws Exception {
+ // given
+ String collectionUuid = "some-collection-uuid";
+ String filePathName = "sample-file-path";
+ long offset = 1024;
+
+ byte[] expectedData = "test data".getBytes();
+
+ try (Buffer buffer = new Buffer().write(expectedData)) {
+ server.enqueue(new MockResponse().setBody(buffer));
+
+ // when
+ byte[] actualData = client.downloadPartial(collectionUuid, filePathName, offset);
+
+ // then
+ assertNotNull(actualData);
+ assertEquals(new String(expectedData), new String(actualData));
+ }
+ }
+
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list