[ARVADOS-WORKBENCH2] updated: 1.1.4-536-gf1bbac6
Git user
git at public.curoverse.com
Mon Aug 6 18:31:56 EDT 2018
Summary of changes:
src/components/file-upload/file-upload.tsx | 9 ++-
.../collection-files-service.ts | 1 -
.../collection-manifest-parser.ts | 4 +-
.../collection-service/collection-service.ts | 86 +++++++++++++++++-----
.../creator/collection-creator-action.ts | 5 +-
.../create-collection-dialog.tsx | 5 +-
6 files changed, 82 insertions(+), 28 deletions(-)
via f1bbac648067f651d408d3cad39fd31d9a36354d (commit)
from 6e830f14ac2732c6d8661275fff691dd8a6179fa (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f1bbac648067f651d408d3cad39fd31d9a36354d
Author: Daniel Kos <daniel.kos at contractors.roche.com>
Date: Tue Aug 7 00:31:54 2018 +0200
Add multiple files uploading
Feature #13856
Arvados-DCO-1.1-Signed-off-by: Daniel Kos <daniel.kos at contractors.roche.com>
diff --git a/src/components/file-upload/file-upload.tsx b/src/components/file-upload/file-upload.tsx
index 5fb2842..8c6e04a 100644
--- a/src/components/file-upload/file-upload.tsx
+++ b/src/components/file-upload/file-upload.tsx
@@ -41,13 +41,14 @@ export const FileUpload = withStyles(styles)(
Upload data
</Typography>
<Dropzone className={classes.dropzone} onDrop={files => onDrop(files)}>
- <Grid container justify="center" alignItems="center" className={classes.container} direction={"column"}>
- <Grid item component={"span"}>
+ <Grid container justify="center" alignItems="center" className={classes.container} direction={"row"}>
+ <Grid item component={"span"} style={{width: "100%", textAlign: "center"}}>
<Typography variant={"subheading"}>
- <CloudUploadIcon className={classes.uploadIcon}/> Drag and drop data or <a>browse</a>
+ <CloudUploadIcon className={classes.uploadIcon}/> Drag and drop data or click to browse
</Typography>
</Grid>
- <Grid item>
+
+ <Grid item style={{width: "100%"}}>
<List>
{files.map((f, idx) =>
<ListItem button key={idx}>
diff --git a/src/services/collection-files-service/collection-files-service.ts b/src/services/collection-files-service/collection-files-service.ts
index dfeed0b..5e6891c 100644
--- a/src/services/collection-files-service/collection-files-service.ts
+++ b/src/services/collection-files-service/collection-files-service.ts
@@ -5,7 +5,6 @@
import { CollectionService } from "../collection-service/collection-service";
import { parseKeepManifestText, stringifyKeepManifest } from "./collection-manifest-parser";
import { mapManifestToCollectionFilesTree } from "./collection-manifest-mapper";
-import { CollectionFile } from "../../models/collection-file";
import { CommonResourceService } from "../../common/api/common-resource-service";
import * as _ from "lodash";
diff --git a/src/services/collection-files-service/collection-manifest-parser.ts b/src/services/collection-files-service/collection-manifest-parser.ts
index bed2678..b0fc55a 100644
--- a/src/services/collection-files-service/collection-manifest-parser.ts
+++ b/src/services/collection-files-service/collection-manifest-parser.ts
@@ -7,7 +7,7 @@ import { KeepManifestStream, KeepManifestStreamFile, KeepManifest } from "../../
/**
* Documentation [http://doc.arvados.org/api/storage.html](http://doc.arvados.org/api/storage.html)
*/
-export const parseKeepManifestText = (text: string) =>
+export const parseKeepManifestText: (text: string) => KeepManifestStream[] = (text: string) =>
text
.split(/\n/)
.filter(streamText => streamText.length > 0)
@@ -52,4 +52,4 @@ const parseFile = (token: string): KeepManifestStreamFile => {
};
const stringifyFile = (file: KeepManifestStreamFile) =>
- `${file.position}:${file.size}:${file.name}`;
\ No newline at end of file
+ `${file.position}:${file.size}:${file.name}`;
diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts
index 88f3dae..cf2d53e 100644
--- a/src/services/collection-service/collection-service.ts
+++ b/src/services/collection-service/collection-service.ts
@@ -7,36 +7,86 @@ import { CollectionResource } from "../../models/collection";
import axios, { AxiosInstance } from "axios";
import { KeepService } from "../keep-service/keep-service";
import { FilterBuilder } from "../../common/api/filter-builder";
+import { CollectionFile, CollectionFileType, createCollectionFile } from "../../models/collection-file";
+import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser";
+import * as _ from "lodash";
+import { KeepManifestStream } from "../../models/keep-manifest";
export class CollectionService extends CommonResourceService<CollectionResource> {
constructor(serverApi: AxiosInstance, private keepService: KeepService) {
super(serverApi, "collections");
}
- uploadFiles(files: File[]) {
- console.log("Uploading files", files);
-
+ uploadFile(keepServiceHost: string, file: File, fileIdx = 0): Promise<CollectionFile> {
const fd = new FormData();
- fd.append("filters", `[["service_type","=","proxy"]]`);
- fd.append("_method", "GET");
+ fd.append(`file_${fileIdx}`, file);
- const filters = new FilterBuilder();
- filters.addEqual("service_type", "proxy");
+ return axios.post<string>(keepServiceHost, fd, {
+ onUploadProgress: (e: ProgressEvent) => {
+ console.log(`${e.loaded} / ${e.total}`);
+ }
+ }).then(data => createCollectionFile({
+ id: data.data,
+ name: file.name,
+ size: file.size
+ }));
+ }
- return this.keepService.list({ filters }).then(data => {
- console.log(data);
+ private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise<CollectionResource> {
+ const collection = await this.get(collectionUuid);
+ const manifest: KeepManifestStream[] = parseKeepManifestText(collection.manifestText);
- const serviceHost = (data.items[0].serviceSslFlag ? "https://" : "http://") + data.items[0].serviceHost + ":" + data.items[0].servicePort;
- console.log("Servicehost", serviceHost);
+ files.forEach(f => {
+ let kms = manifest.find(stream => stream.name === f.path);
+ if (!kms) {
+ kms = {
+ files: [],
+ locators: [],
+ name: f.path
+ };
+ manifest.push(kms);
+ }
+ kms.locators.push(f.id);
+ const len = kms.files.length;
+ const nextPos = len > 0
+ ? parseInt(kms.files[len - 1].position, 10) + kms.files[len - 1].size
+ : 0;
+ kms.files.push({
+ name: f.name,
+ position: nextPos.toString(),
+ size: f.size
+ });
+ });
- const fd = new FormData();
- files.forEach((f, idx) => fd.append(`file_${idx}`, f));
+ console.log(manifest);
- axios.post(serviceHost, fd, {
- onUploadProgress: (e: ProgressEvent) => {
- console.log(`${e.loaded} / ${e.total}`);
- }
- });
+ const manifestText = stringifyKeepManifest(manifest);
+ const data = { ...collection, manifestText };
+ return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data));
+ }
+
+ uploadFiles(collectionUuid: string, files: File[]) {
+ console.log("Uploading files", files);
+
+ const filters = FilterBuilder.create()
+ .addEqual("service_type", "proxy");
+
+ return this.keepService.list({ filters }).then(data => {
+ if (data.items && data.items.length > 0) {
+ const serviceHost =
+ (data.items[0].serviceSslFlag ? "https://" : "http://") +
+ data.items[0].serviceHost +
+ ":" + data.items[0].servicePort;
+
+ console.log("Servicehost", serviceHost);
+
+ const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx));
+ Promise.all(files$).then(values => {
+ this.updateManifest(collectionUuid, values).then(() => {
+ console.log("Upload done!");
+ });
+ });
+ }
});
}
}
diff --git a/src/store/collections/creator/collection-creator-action.ts b/src/store/collections/creator/collection-creator-action.ts
index b06cf0f..3afe0e9 100644
--- a/src/store/collections/creator/collection-creator-action.ts
+++ b/src/store/collections/creator/collection-creator-action.ts
@@ -26,7 +26,10 @@ export const createCollection = (collection: Partial<CollectionResource>) =>
dispatch(collectionCreateActions.CREATE_COLLECTION(collectiontData));
return services.collectionService
.create(collectiontData)
- .then(collection => dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection)));
+ .then(collection => {
+ dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
+ return collection;
+ });
};
export type CollectionCreateAction = UnionOf<typeof collectionCreateActions>;
diff --git a/src/views-components/create-collection-dialog/create-collection-dialog.tsx b/src/views-components/create-collection-dialog/create-collection-dialog.tsx
index 281bd54..8711c5f 100644
--- a/src/views-components/create-collection-dialog/create-collection-dialog.tsx
+++ b/src/views-components/create-collection-dialog/create-collection-dialog.tsx
@@ -13,6 +13,7 @@ import { dataExplorerActions } from "../../store/data-explorer/data-explorer-act
import { PROJECT_PANEL_ID } from "../../views/project-panel/project-panel";
import { snackbarActions } from "../../store/snackbar/snackbar-actions";
import { ServiceRepository } from "../../services/services";
+import { CollectionResource } from "../../models/collection";
const mapStateToProps = (state: RootState) => ({
open: state.collections.creator.opened
@@ -32,12 +33,12 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({
const addCollection = (data: { name: string, description: string, files: File[] }) =>
(dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
- return dispatch<any>(createCollection(data)).then(() => {
+ return dispatch<any>(createCollection(data)).then((collection: CollectionResource) => {
dispatch(snackbarActions.OPEN_SNACKBAR({
message: "Collection has been successfully created.",
hideDuration: 2000
}));
- services.collectionService.uploadFiles(data.files).then(() => {
+ services.collectionService.uploadFiles(collection.uuid, data.files).then(() => {
dispatch(dataExplorerActions.REQUEST_ITEMS({ id: PROJECT_PANEL_ID }));
});
});
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list