[ARVADOS-WORKBENCH2] updated: 1.1.4-537-g961394c
Git user
git at public.curoverse.com
Tue Aug 7 05:33:36 EDT 2018
Summary of changes:
src/common/formatters.ts | 14 ++++-
src/components/file-upload/file-upload.tsx | 53 +++++++++++++------
.../collection-service/collection-service.ts | 60 ++++++++++++++--------
.../creator/collection-creator-action.ts | 12 ++++-
.../uploader/collection-uploader-actions.ts | 15 +++++-
.../uploader/collection-uploader-reducer.ts | 42 ++++++++++-----
.../create-collection-dialog.tsx | 18 +++----
.../dialog-create/dialog-collection-create.tsx | 14 ++---
8 files changed, 153 insertions(+), 75 deletions(-)
via 961394c0876cdc07f2195d3bc1843a5c9a6fa950 (commit)
from f1bbac648067f651d408d3cad39fd31d9a36354d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 961394c0876cdc07f2195d3bc1843a5c9a6fa950
Author: Daniel Kos <daniel.kos at contractors.roche.com>
Date: Tue Aug 7 11:33:12 2018 +0200
Fix correct bytes not being sent, fix showing upload progress and speed
Feature #13856
Arvados-DCO-1.1-Signed-off-by: Daniel Kos <daniel.kos at contractors.roche.com>
diff --git a/src/common/formatters.ts b/src/common/formatters.ts
index 38ef022..49e0690 100644
--- a/src/common/formatters.ts
+++ b/src/common/formatters.ts
@@ -19,6 +19,18 @@ export const formatFileSize = (size?: number) => {
return "";
};
+export const formatProgress = (loaded: number, total: number) => {
+ const progress = loaded >= 0 && total > 0 ? loaded * 100 / total : 0;
+ return `${progress.toFixed(2)}%`;
+};
+
+export function formatUploadSpeed(prevLoaded: number, loaded: number, prevTime: number, currentTime: number) {
+ const speed = loaded > prevLoaded && currentTime > prevTime
+ ? (loaded - prevLoaded) / (currentTime - prevTime)
+ : 0;
+ return `${(speed / 1000).toFixed(2)} KB/s`;
+}
+
const FILE_SIZES = [
{
base: 1000000000000,
@@ -40,4 +52,4 @@ const FILE_SIZES = [
base: 1,
unit: "B"
}
-];
\ No newline at end of file
+];
diff --git a/src/components/file-upload/file-upload.tsx b/src/components/file-upload/file-upload.tsx
index 8c6e04a..aa3c0e9 100644
--- a/src/components/file-upload/file-upload.tsx
+++ b/src/components/file-upload/file-upload.tsx
@@ -3,11 +3,18 @@
// SPDX-License-Identifier: AGPL-3.0
import * as React from 'react';
-import { Grid, List, ListItem, ListItemText, StyleRulesCallback, Typography, WithStyles } from '@material-ui/core';
+import {
+ Grid,
+ StyleRulesCallback,
+ Table, TableBody, TableCell, TableHead, TableRow,
+ Typography,
+ WithStyles
+} from '@material-ui/core';
import { withStyles } from '@material-ui/core';
import Dropzone from 'react-dropzone';
import { CloudUploadIcon } from "../icon/icon";
-import { formatFileSize } from "../../common/formatters";
+import { formatFileSize, formatProgress, formatUploadSpeed } from "../../common/formatters";
+import { UploadFile } from "../../store/collections/uploader/collection-uploader-actions";
type CssRules = "root" | "dropzone" | "container" | "uploadIcon";
@@ -30,7 +37,7 @@ const styles: StyleRulesCallback<CssRules> = theme => ({
});
interface FileUploadProps {
- files: File[];
+ files: UploadFile[];
onDrop: (files: File[]) => void;
}
@@ -41,24 +48,36 @@ export const FileUpload = withStyles(styles)(
Upload data
</Typography>
<Dropzone className={classes.dropzone} onDrop={files => onDrop(files)}>
- <Grid container justify="center" alignItems="center" className={classes.container} direction={"row"}>
- <Grid item component={"span"} style={{width: "100%", textAlign: "center"}}>
+ {files.length === 0 &&
+ <Grid container justify="center" alignItems="center" className={classes.container}>
+ <Grid item component={"span"}>
<Typography variant={"subheading"}>
<CloudUploadIcon className={classes.uploadIcon}/> Drag and drop data or click to browse
</Typography>
</Grid>
-
- <Grid item style={{width: "100%"}}>
- <List>
- {files.map((f, idx) =>
- <ListItem button key={idx}>
- <ListItemText
- primary={f.name} primaryTypographyProps={{variant: "body2"}}
- secondary={formatFileSize(f.size)}/>
- </ListItem>)}
- </List>
- </Grid>
- </Grid>
+ </Grid>}
+ {files.length > 0 &&
+ <Table style={{width: "100%"}}>
+ <TableHead>
+ <TableRow>
+ <TableCell>File name</TableCell>
+ <TableCell>File size</TableCell>
+ <TableCell>Upload speed</TableCell>
+ <TableCell>Upload progress</TableCell>
+ </TableRow>
+ </TableHead>
+ <TableBody>
+ {files.map(f =>
+ <TableRow key={f.id}>
+ <TableCell>{f.file.name}</TableCell>
+ <TableCell>{formatFileSize(f.file.size)}</TableCell>
+ <TableCell>{formatUploadSpeed(f.prevLoaded, f.loaded, f.prevTime, f.currentTime)}</TableCell>
+ <TableCell>{formatProgress(f.loaded, f.total)}</TableCell>
+ </TableRow>
+ )}
+ </TableBody>
+ </Table>
+ }
</Dropzone>
</Grid>
);
diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts
index cf2d53e..4d75036 100644
--- a/src/services/collection-service/collection-service.ts
+++ b/src/services/collection-service/collection-service.ts
@@ -7,29 +7,47 @@ import { CollectionResource } from "../../models/collection";
import axios, { AxiosInstance } from "axios";
import { KeepService } from "../keep-service/keep-service";
import { FilterBuilder } from "../../common/api/filter-builder";
-import { CollectionFile, CollectionFileType, createCollectionFile } from "../../models/collection-file";
+import { CollectionFile, createCollectionFile } from "../../models/collection-file";
import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser";
import * as _ from "lodash";
import { KeepManifestStream } from "../../models/keep-manifest";
+export type UploadProgress = (fileId: number, loaded: number, total: number, currentTime: number) => void;
+
export class CollectionService extends CommonResourceService<CollectionResource> {
constructor(serverApi: AxiosInstance, private keepService: KeepService) {
super(serverApi, "collections");
}
- uploadFile(keepServiceHost: string, file: File, fileIdx = 0): Promise<CollectionFile> {
- const fd = new FormData();
- fd.append(`file_${fileIdx}`, file);
+ private readFile(file: File): Promise<ArrayBuffer> {
+ return new Promise<ArrayBuffer>(resolve => {
+ const reader = new FileReader();
+ reader.onload = () => {
+ resolve(reader.result as ArrayBuffer);
+ };
- return axios.post<string>(keepServiceHost, fd, {
- onUploadProgress: (e: ProgressEvent) => {
- console.log(`${e.loaded} / ${e.total}`);
- }
- }).then(data => createCollectionFile({
- id: data.data,
- name: file.name,
- size: file.size
- }));
+ reader.readAsArrayBuffer(file);
+ });
+ }
+
+ private uploadFile(keepServiceHost: string, file: File, fileId: number, onProgress?: UploadProgress): Promise<CollectionFile> {
+ return this.readFile(file).then(content => {
+ return axios.post<string>(keepServiceHost, content, {
+ headers: {
+ 'Content-Type': 'text/octet-stream'
+ },
+ onUploadProgress: (e: ProgressEvent) => {
+ if (onProgress) {
+ onProgress(fileId, e.loaded, e.total, Date.now());
+ }
+ console.log(`${e.loaded} / ${e.total}`);
+ }
+ }).then(data => createCollectionFile({
+ id: data.data,
+ name: file.name,
+ size: file.size
+ }));
+ });
}
private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise<CollectionResource> {
@@ -65,9 +83,7 @@ export class CollectionService extends CommonResourceService<CollectionResource>
return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data));
}
- uploadFiles(collectionUuid: string, files: File[]) {
- console.log("Uploading files", files);
-
+ uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress): Promise<CollectionResource | never> {
const filters = FilterBuilder.create()
.addEqual("service_type", "proxy");
@@ -78,14 +94,14 @@ export class CollectionService extends CommonResourceService<CollectionResource>
data.items[0].serviceHost +
":" + data.items[0].servicePort;
- console.log("Servicehost", serviceHost);
+ console.log("serviceHost", serviceHost);
- const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx));
- Promise.all(files$).then(values => {
- this.updateManifest(collectionUuid, values).then(() => {
- console.log("Upload done!");
- });
+ const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx, onProgress));
+ return Promise.all(files$).then(values => {
+ return this.updateManifest(collectionUuid, values);
});
+ } else {
+ return Promise.reject("Missing keep service host");
}
});
}
diff --git a/src/store/collections/creator/collection-creator-action.ts b/src/store/collections/creator/collection-creator-action.ts
index 3afe0e9..023e5be 100644
--- a/src/store/collections/creator/collection-creator-action.ts
+++ b/src/store/collections/creator/collection-creator-action.ts
@@ -8,6 +8,7 @@ import { Dispatch } from "redux";
import { RootState } from "../../store";
import { CollectionResource } from '../../../models/collection';
import { ServiceRepository } from "../../../services/services";
+import { collectionUploaderActions } from "../uploader/collection-uploader-actions";
export const collectionCreateActions = unionize({
OPEN_COLLECTION_CREATOR: ofType<{ ownerUuid: string }>(),
@@ -19,7 +20,7 @@ export const collectionCreateActions = unionize({
value: 'payload'
});
-export const createCollection = (collection: Partial<CollectionResource>) =>
+export const createCollection = (collection: Partial<CollectionResource>, files: File[]) =>
(dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
const { ownerUuid } = getState().collections.creator;
const collectiontData = { ownerUuid, ...collection };
@@ -27,7 +28,14 @@ export const createCollection = (collection: Partial<CollectionResource>) =>
return services.collectionService
.create(collectiontData)
.then(collection => {
- dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
+ dispatch(collectionUploaderActions.START_UPLOAD());
+ services.collectionService.uploadFiles(collection.uuid, files,
+ (fileId, loaded, total, currentTime) => {
+ dispatch(collectionUploaderActions.SET_UPLOAD_PROGRESS({ fileId, loaded, total, currentTime }));
+ })
+ .then(collection => {
+ dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
+ });
return collection;
});
};
diff --git a/src/store/collections/uploader/collection-uploader-actions.ts b/src/store/collections/uploader/collection-uploader-actions.ts
index 0b9aeb9..7c85d74 100644
--- a/src/store/collections/uploader/collection-uploader-actions.ts
+++ b/src/store/collections/uploader/collection-uploader-actions.ts
@@ -4,10 +4,21 @@
import { default as unionize, ofType, UnionOf } from "unionize";
+export interface UploadFile {
+ id: number;
+ file: File;
+ prevLoaded: number;
+ loaded: number;
+ total: number;
+ startTime: number;
+ prevTime: number;
+ currentTime: number;
+}
+
export const collectionUploaderActions = unionize({
SET_UPLOAD_FILES: ofType<File[]>(),
- START_UPLOADING: ofType<{}>(),
- UPDATE_UPLOAD_PROGRESS: ofType<{}>()
+ START_UPLOAD: ofType(),
+ SET_UPLOAD_PROGRESS: ofType<{ fileId: number, loaded: number, total: number, currentTime: number }>()
}, {
tag: 'type',
value: 'payload'
diff --git a/src/store/collections/uploader/collection-uploader-reducer.ts b/src/store/collections/uploader/collection-uploader-reducer.ts
index 05735d6..5b24d2c 100644
--- a/src/store/collections/uploader/collection-uploader-reducer.ts
+++ b/src/store/collections/uploader/collection-uploader-reducer.ts
@@ -2,23 +2,41 @@
//
// SPDX-License-Identifier: AGPL-3.0
-import { CollectionUploaderAction, collectionUploaderActions } from "./collection-uploader-actions";
-import { CollectionUploadFile } from "../../../models/collection-file";
+import { CollectionUploaderAction, collectionUploaderActions, UploadFile } from "./collection-uploader-actions";
+import * as _ from 'lodash';
-export interface CollectionUploaderState {
- files: File[];
-}
+export type CollectionUploaderState = UploadFile[];
-const initialState: CollectionUploaderState = {
- files: []
-};
+const initialState: CollectionUploaderState = [];
export const collectionUploaderReducer = (state: CollectionUploaderState = initialState, action: CollectionUploaderAction) => {
return collectionUploaderActions.match(action, {
- SET_UPLOAD_FILES: (files) => ({
- ...state,
- files
- }),
+ SET_UPLOAD_FILES: files => files.map((f, idx) => ({
+ id: idx,
+ file: f,
+ prevLoaded: 0,
+ loaded: 0,
+ total: 0,
+ startTime: 0,
+ prevTime: 0,
+ currentTime: 0
+ })),
+ START_UPLOAD: () => {
+ const startTime = Date.now();
+ return state.map(f => ({...f, startTime, prevTime: startTime}));
+ },
+ SET_UPLOAD_PROGRESS: ({ fileId, loaded, total, currentTime }) => {
+ const files = _.cloneDeep(state);
+ const f = files.find(f => f.id === fileId);
+ if (f) {
+ f.prevLoaded = f.loaded;
+ f.loaded = loaded;
+ f.total = total;
+ f.prevTime = f.currentTime;
+ f.currentTime = currentTime;
+ }
+ return files;
+ },
default: () => state
});
};
diff --git a/src/views-components/create-collection-dialog/create-collection-dialog.tsx b/src/views-components/create-collection-dialog/create-collection-dialog.tsx
index 8711c5f..3f8cc07 100644
--- a/src/views-components/create-collection-dialog/create-collection-dialog.tsx
+++ b/src/views-components/create-collection-dialog/create-collection-dialog.tsx
@@ -9,11 +9,8 @@ import { SubmissionError } from "redux-form";
import { RootState } from "../../store/store";
import { DialogCollectionCreate } from "../dialog-create/dialog-collection-create";
import { collectionCreateActions, createCollection } from "../../store/collections/creator/collection-creator-action";
-import { dataExplorerActions } from "../../store/data-explorer/data-explorer-action";
-import { PROJECT_PANEL_ID } from "../../views/project-panel/project-panel";
import { snackbarActions } from "../../store/snackbar/snackbar-actions";
-import { ServiceRepository } from "../../services/services";
-import { CollectionResource } from "../../models/collection";
+import { UploadFile } from "../../store/collections/uploader/collection-uploader-actions";
const mapStateToProps = (state: RootState) => ({
open: state.collections.creator.opened
@@ -23,24 +20,21 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({
handleClose: () => {
dispatch(collectionCreateActions.CLOSE_COLLECTION_CREATOR());
},
- onSubmit: (data: { name: string, description: string, files: File[] }) => {
- return dispatch<any>(addCollection(data))
+ onSubmit: (data: { name: string, description: string }, files: UploadFile[]) => {
+ return dispatch<any>(addCollection(data, files.map(f => f.file)))
.catch((e: any) => {
throw new SubmissionError({ name: e.errors.join("").includes("UniqueViolation") ? "Collection with this name already exists." : "" });
});
}
});
-const addCollection = (data: { name: string, description: string, files: File[] }) =>
- (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
- return dispatch<any>(createCollection(data)).then((collection: CollectionResource) => {
+const addCollection = (data: { name: string, description: string }, files: File[]) =>
+ (dispatch: Dispatch) => {
+ return dispatch<any>(createCollection(data, files)).then(() => {
dispatch(snackbarActions.OPEN_SNACKBAR({
message: "Collection has been successfully created.",
hideDuration: 2000
}));
- services.collectionService.uploadFiles(collection.uuid, data.files).then(() => {
- dispatch(dataExplorerActions.REQUEST_ITEMS({ id: PROJECT_PANEL_ID }));
- });
});
};
diff --git a/src/views-components/dialog-create/dialog-collection-create.tsx b/src/views-components/dialog-create/dialog-collection-create.tsx
index a0cbcba..32fc657 100644
--- a/src/views-components/dialog-create/dialog-collection-create.tsx
+++ b/src/views-components/dialog-create/dialog-collection-create.tsx
@@ -16,7 +16,7 @@ import { COLLECTION_NAME_VALIDATION, COLLECTION_DESCRIPTION_VALIDATION } from '.
import { FileUpload } from "../../components/file-upload/file-upload";
import { connect, DispatchProp } from "react-redux";
import { RootState } from "../../store/store";
-import { collectionUploaderActions } from "../../store/collections/uploader/collection-uploader-actions";
+import { collectionUploaderActions, UploadFile } from "../../store/collections/uploader/collection-uploader-actions";
type CssRules = "button" | "lastButton" | "formContainer" | "textField" | "createProgress" | "dialogActions";
@@ -48,12 +48,12 @@ const styles: StyleRulesCallback<CssRules> = theme => ({
interface DialogCollectionCreateProps {
open: boolean;
handleClose: () => void;
- onSubmit: (data: { name: string, description: string, files: File[] }) => void;
+ onSubmit: (data: { name: string, description: string }, files: UploadFile[]) => void;
handleSubmit: any;
submitting: boolean;
invalid: boolean;
pristine: boolean;
- files: File[];
+ files: UploadFile[];
}
interface TextFieldProps {
@@ -66,13 +66,13 @@ interface TextFieldProps {
export const DialogCollectionCreate = compose(
connect((state: RootState) => ({
- files: state.collections.uploader.files
+ files: state.collections.uploader
})),
reduxForm({ form: 'collectionCreateDialog' }),
withStyles(styles))(
class DialogCollectionCreate extends React.Component<DialogCollectionCreateProps & DispatchProp & WithStyles<CssRules>> {
render() {
- const { classes, open, handleClose, handleSubmit, onSubmit, submitting, invalid, pristine } = this.props;
+ const { classes, open, handleClose, handleSubmit, onSubmit, submitting, invalid, pristine, files } = this.props;
return (
<Dialog
@@ -82,7 +82,7 @@ export const DialogCollectionCreate = compose(
maxWidth='sm'
disableBackdropClick={true}
disableEscapeKeyDown={true}>
- <form onSubmit={handleSubmit((data: any) => onSubmit({ ...data, files: this.props.files }))}>
+ <form onSubmit={handleSubmit((data: any) => onSubmit(data, files))}>
<DialogTitle id="form-dialog-title">Create a collection</DialogTitle>
<DialogContent className={classes.formContainer}>
<Field name="name"
@@ -100,7 +100,7 @@ export const DialogCollectionCreate = compose(
className={classes.textField}
label="Description - optional"/>
<FileUpload
- files={this.props.files}
+ files={files}
onDrop={files => this.props.dispatch(collectionUploaderActions.SET_UPLOAD_FILES(files))}/>
</DialogContent>
<DialogActions className={classes.dialogActions}>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list