[ARVADOS-WORKBENCH2] created: 1.1.4-592-g8c259b5
Git user
git at public.curoverse.com
Mon Aug 13 10:33:40 EDT 2018
at 8c259b5122df8254613e8d23b0d860a7b1b37b41 (commit)
commit 8c259b5122df8254613e8d23b0d860a7b1b37b41
Author: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
Date: Mon Aug 13 16:33:23 2018 +0200
Extract response parsing utils from collection service, use webdav to upload files in collection service
Feature #14013
Arvados-DCO-1.1-Signed-off-by: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
diff --git a/src/services/collection-service/collection-service-files-response.ts b/src/services/collection-service/collection-service-files-response.ts
new file mode 100644
index 0000000..4545096
--- /dev/null
+++ b/src/services/collection-service/collection-service-files-response.ts
@@ -0,0 +1,53 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import { createCollectionFilesTree, CollectionDirectory, CollectionFile, CollectionFileType, createCollectionDirectory, createCollectionFile } from "../../models/collection-file";
+import { Tree, mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
+import { getTagValue } from "../../common/xml";
+
+export const parseFilesResponse = (document: Document) => {
+ const files = extractFilesData(document);
+ const tree = createCollectionFilesTree(files);
+ return sortFilesTree(tree);
+};
+
+export const sortFilesTree = (tree: Tree<CollectionDirectory | CollectionFile>) => {
+ return mapTree(node => {
+ const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
+ children.sort((a, b) =>
+ a.value.type !== b.value.type
+ ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
+ : a.value.name.localeCompare(b.value.name)
+ );
+ return { ...node, children: children.map(child => child.id) } as TreeNode<CollectionDirectory | CollectionFile>;
+ })(tree);
+};
+
+export const extractFilesData = (document: Document) => {
+ const collectionUrlPrefix = /\/c=[0-9a-zA-Z\-]*/;
+ return Array
+ .from(document.getElementsByTagName('D:response'))
+ .slice(1) // omit first element which is collection itself
+ .map(element => {
+ const name = getTagValue(element, 'D:displayname', '');
+ const size = parseInt(getTagValue(element, 'D:getcontentlength', '0'), 10);
+ const pathname = getTagValue(element, 'D:href', '');
+ const nameSuffix = `/${name || ''}`;
+ const directory = pathname
+ .replace(collectionUrlPrefix, '')
+ .replace(nameSuffix, '');
+
+ const data = {
+ url: pathname,
+ id: `${directory}/${name}`,
+ name,
+ path: directory,
+ };
+
+ return getTagValue(element, 'D:resourcetype', '')
+ ? createCollectionDirectory(data)
+ : createCollectionFile({ ...data, size });
+
+ });
+};
diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts
index 8cf515c..3b98f75 100644
--- a/src/services/collection-service/collection-service.ts
+++ b/src/services/collection-service/collection-service.ts
@@ -4,40 +4,25 @@
import { CommonResourceService } from "../../common/api/common-resource-service";
import { CollectionResource } from "../../models/collection";
-import axios, { AxiosInstance } from "axios";
-import { KeepService } from "../keep-service/keep-service";
-import { FilterBuilder } from "../../common/api/filter-builder";
-import { CollectionFile, createCollectionFile, createCollectionDirectory, createCollectionFilesTree, CollectionFileType, CollectionDirectory } from "../../models/collection-file";
-import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser";
-import * as _ from "lodash";
-import { KeepManifestStream } from "../../models/keep-manifest";
+import { AxiosInstance } from "axios";
+import { CollectionFile, CollectionDirectory } from "../../models/collection-file";
import { WebDAV } from "../../common/webdav";
import { AuthService } from "../auth-service/auth-service";
-import { mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
-import { getTagValue } from "../../common/xml";
+import { mapTreeValues } from "../../models/tree";
+import { parseFilesResponse } from "./collection-service-files-response";
export type UploadProgress = (fileId: number, loaded: number, total: number, currentTime: number) => void;
export class CollectionService extends CommonResourceService<CollectionResource> {
- constructor(serverApi: AxiosInstance, private keepService: KeepService, private webdavClient: WebDAV, private authService: AuthService) {
+ constructor(serverApi: AxiosInstance, private webdavClient: WebDAV, private authService: AuthService) {
super(serverApi, "collections");
}
async files(uuid: string) {
const request = await this.webdavClient.propfind(`/c=${uuid}`);
if (request.responseXML != null) {
- const files = this.extractFilesData(request.responseXML);
- const tree = createCollectionFilesTree(files);
- const sortedTree = mapTree(node => {
- const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
- children.sort((a, b) =>
- a.value.type !== b.value.type
- ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
- : a.value.name.localeCompare(b.value.name)
- );
- return { ...node, children: children.map(child => child.id) };
- })(tree);
- return sortedTree;
+ const filesTree = parseFilesResponse(request.responseXML);
+ return mapTreeValues(this.extendFileURL)(filesTree);
}
return Promise.reject();
}
@@ -46,120 +31,38 @@ export class CollectionService extends CommonResourceService<CollectionResource>
return this.webdavClient.delete(`/c=${collectionUuid}${filePath}`);
}
- extractFilesData(document: Document) {
- const collectionUrlPrefix = /\/c=[0-9a-zA-Z\-]*/;
- return Array
- .from(document.getElementsByTagName('D:response'))
- .slice(1) // omit first element which is collection itself
- .map(element => {
- const name = getTagValue(element, 'D:displayname', '');
- const size = parseInt(getTagValue(element, 'D:getcontentlength', '0'), 10);
- const pathname = getTagValue(element, 'D:href', '');
- const nameSuffix = `/${name || ''}`;
- const directory = pathname
- .replace(collectionUrlPrefix, '')
- .replace(nameSuffix, '');
- const href = this.webdavClient.defaults.baseURL + pathname + '?api_token=' + this.authService.getApiToken();
-
- const data = {
- url: href,
- id: `${directory}/${name}`,
- name,
- path: directory,
- };
+ async uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress) {
+ for (let idx = 0; idx < files.length; idx++) {
+ await this.uploadFile(collectionUuid, files[idx], idx, onProgress);
+ }
+ }
- return getTagValue(element, 'D:resourcetype', '')
- ? createCollectionDirectory(data)
- : createCollectionFile({ ...data, size });
+ private extendFileURL = (file: CollectionDirectory | CollectionFile) => ({
+ ...file,
+ url: this.webdavClient.defaults.baseURL + file.url + '?api_token=' + this.authService.getApiToken()
+ })
- });
+ private uploadFile(collectionUuid: string, file: File, fileId: number, onProgress: UploadProgress = () => { return; }) {
+ return this.readFile(file).then(content =>
+ this.webdavClient.put(`/c=${collectionUuid}/${file.name}`, content, {
+ headers: {
+ 'Content-Type': 'text/octet-stream'
+ },
+ onUploadProgress: (e: ProgressEvent) => {
+ onProgress(fileId, e.loaded, e.total, Date.now());
+ }
+ })
+ );
}
-
private readFile(file: File): Promise<ArrayBuffer> {
return new Promise<ArrayBuffer>(resolve => {
const reader = new FileReader();
reader.onload = () => {
resolve(reader.result as ArrayBuffer);
};
-
reader.readAsArrayBuffer(file);
});
}
- private uploadFile(keepServiceHost: string, file: File, fileId: number, onProgress?: UploadProgress): Promise<CollectionFile> {
- return this.readFile(file).then(content => {
- return axios.post<string>(keepServiceHost, content, {
- headers: {
- 'Content-Type': 'text/octet-stream'
- },
- onUploadProgress: (e: ProgressEvent) => {
- if (onProgress) {
- onProgress(fileId, e.loaded, e.total, Date.now());
- }
- console.log(`${e.loaded} / ${e.total}`);
- }
- }).then(data => createCollectionFile({
- id: data.data,
- name: file.name,
- size: file.size
- }));
- });
- }
-
- private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise<CollectionResource> {
- const collection = await this.get(collectionUuid);
- const manifest: KeepManifestStream[] = parseKeepManifestText(collection.manifestText);
-
- files.forEach(f => {
- let kms = manifest.find(stream => stream.name === f.path);
- if (!kms) {
- kms = {
- files: [],
- locators: [],
- name: f.path
- };
- manifest.push(kms);
- }
- kms.locators.push(f.id);
- const len = kms.files.length;
- const nextPos = len > 0
- ? parseInt(kms.files[len - 1].position, 10) + kms.files[len - 1].size
- : 0;
- kms.files.push({
- name: f.name,
- position: nextPos.toString(),
- size: f.size
- });
- });
-
- console.log(manifest);
-
- const manifestText = stringifyKeepManifest(manifest);
- const data = { ...collection, manifestText };
- return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data));
- }
-
- uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress): Promise<CollectionResource | never> {
- const filters = FilterBuilder.create()
- .addEqual("service_type", "proxy");
-
- return this.keepService.list({ filters }).then(data => {
- if (data.items && data.items.length > 0) {
- const serviceHost =
- (data.items[0].serviceSslFlag ? "https://" : "http://") +
- data.items[0].serviceHost +
- ":" + data.items[0].servicePort;
-
- console.log("serviceHost", serviceHost);
-
- const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx, onProgress));
- return Promise.all(files$).then(values => {
- return this.updateManifest(collectionUuid, values);
- });
- } else {
- return Promise.reject("Missing keep service host");
- }
- });
- }
}
diff --git a/src/services/services.ts b/src/services/services.ts
index eedd124..9b6495f 100644
--- a/src/services/services.ts
+++ b/src/services/services.ts
@@ -30,7 +30,7 @@ export const createServices = (config: Config) => {
const projectService = new ProjectService(apiClient);
const linkService = new LinkService(apiClient);
const favoriteService = new FavoriteService(linkService, groupsService);
- const collectionService = new CollectionService(apiClient, keepService, webdavClient, authService);
+ const collectionService = new CollectionService(apiClient, webdavClient, authService);
const tagService = new TagService(linkService);
const collectionFilesService = new CollectionFilesService(collectionService);
diff --git a/src/store/collections/creator/collection-creator-action.ts b/src/store/collections/creator/collection-creator-action.ts
index d0a66b4..984a3c9 100644
--- a/src/store/collections/creator/collection-creator-action.ts
+++ b/src/store/collections/creator/collection-creator-action.ts
@@ -17,9 +17,9 @@ export const collectionCreateActions = unionize({
CREATE_COLLECTION: ofType<{}>(),
CREATE_COLLECTION_SUCCESS: ofType<{}>(),
}, {
- tag: 'type',
- value: 'payload'
-});
+ tag: 'type',
+ value: 'payload'
+ });
export const createCollection = (collection: Partial<CollectionResource>, files: File[]) =>
(dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
@@ -34,11 +34,11 @@ export const createCollection = (collection: Partial<CollectionResource>, files:
(fileId, loaded, total, currentTime) => {
dispatch(collectionUploaderActions.SET_UPLOAD_PROGRESS({ fileId, loaded, total, currentTime }));
})
- .then(collection => {
- dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
- dispatch(reset('collectionCreateDialog'));
- dispatch(collectionUploaderActions.CLEAR_UPLOAD());
- });
+ .then(() => {
+ dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
+ dispatch(reset('collectionCreateDialog'));
+ dispatch(collectionUploaderActions.CLEAR_UPLOAD());
+ });
return collection;
});
};
commit a49f059145a3054052e1c79555bdeadf660d1c6a
Author: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
Date: Mon Aug 13 16:31:22 2018 +0200
Add onUploadProgress callback to webdav
Feature #14013
Arvados-DCO-1.1-Signed-off-by: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
diff --git a/src/common/webdav.test.ts b/src/common/webdav.test.ts
index d96465b..e1eefad 100644
--- a/src/common/webdav.test.ts
+++ b/src/common/webdav.test.ts
@@ -41,15 +41,15 @@ describe('WebDAV', () => {
it('PUT', async () => {
const { open, send, load, progress, createRequest } = mockCreateRequest();
- const onProgress = jest.fn();
+ const onUploadProgress = jest.fn();
const webdav = new WebDAV(undefined, createRequest);
- const promise = webdav.put('foo', 'Test data', { onProgress });
+ const promise = webdav.put('foo', 'Test data', { onUploadProgress });
progress();
load();
const request = await promise;
expect(open).toHaveBeenCalledWith('PUT', 'foo');
expect(send).toHaveBeenCalledWith('Test data');
- expect(onProgress).toHaveBeenCalled();
+ expect(onUploadProgress).toHaveBeenCalled();
expect(request).toBeInstanceOf(XMLHttpRequest);
});
diff --git a/src/common/webdav.ts b/src/common/webdav.ts
index 57caebc..27e1f22 100644
--- a/src/common/webdav.ts
+++ b/src/common/webdav.ts
@@ -58,8 +58,8 @@ export class WebDAV {
.keys(headers)
.forEach(key => r.setRequestHeader(key, headers[key]));
- if (config.onProgress) {
- r.addEventListener('progress', config.onProgress);
+ if (config.onUploadProgress) {
+ r.upload.addEventListener('progress', config.onUploadProgress);
}
r.addEventListener('load', () => resolve(r));
@@ -73,7 +73,7 @@ export interface WebDAVRequestConfig {
headers?: {
[key: string]: string;
};
- onProgress?: (event: ProgressEvent) => void;
+ onUploadProgress?: (event: ProgressEvent) => void;
}
interface WebDAVDefaults {
@@ -86,5 +86,5 @@ interface RequestConfig {
url: string;
headers?: { [key: string]: string };
data?: any;
- onProgress?: (event: ProgressEvent) => void;
+ onUploadProgress?: (event: ProgressEvent) => void;
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list