[ARVADOS-WORKBENCH2] created: 1.1.4-592-g8c259b5

Git user git at public.curoverse.com
Mon Aug 13 10:33:40 EDT 2018


        at  8c259b5122df8254613e8d23b0d860a7b1b37b41 (commit)


commit 8c259b5122df8254613e8d23b0d860a7b1b37b41
Author: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
Date:   Mon Aug 13 16:33:23 2018 +0200

    Extract response parsing utils from collection service, use webdav to upload files in collection service
    
    Feature #14013
    
    Arvados-DCO-1.1-Signed-off-by: Michal Klobukowski <michal.klobukowski at contractors.roche.com>

diff --git a/src/services/collection-service/collection-service-files-response.ts b/src/services/collection-service/collection-service-files-response.ts
new file mode 100644
index 0000000..4545096
--- /dev/null
+++ b/src/services/collection-service/collection-service-files-response.ts
@@ -0,0 +1,53 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import { createCollectionFilesTree, CollectionDirectory, CollectionFile, CollectionFileType, createCollectionDirectory, createCollectionFile } from "../../models/collection-file";
+import { Tree, mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
+import { getTagValue } from "../../common/xml";
+
+export const parseFilesResponse = (document: Document) => {
+    const files = extractFilesData(document);
+    const tree = createCollectionFilesTree(files);
+    return sortFilesTree(tree);
+};
+
+export const sortFilesTree = (tree: Tree<CollectionDirectory | CollectionFile>) => {
+    return mapTree(node => {
+        const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
+        children.sort((a, b) =>
+            a.value.type !== b.value.type
+                ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
+                : a.value.name.localeCompare(b.value.name)
+        );
+        return { ...node, children: children.map(child => child.id) } as TreeNode<CollectionDirectory | CollectionFile>;
+    })(tree);
+};
+
+export const extractFilesData = (document: Document) => {
+    const collectionUrlPrefix = /\/c=[0-9a-zA-Z\-]*/;
+    return Array
+        .from(document.getElementsByTagName('D:response'))
+        .slice(1) // omit first element which is collection itself
+        .map(element => {
+            const name = getTagValue(element, 'D:displayname', '');
+            const size = parseInt(getTagValue(element, 'D:getcontentlength', '0'), 10);
+            const pathname = getTagValue(element, 'D:href', '');
+            const nameSuffix = `/${name || ''}`;
+            const directory = pathname
+                .replace(collectionUrlPrefix, '')
+                .replace(nameSuffix, '');
+
+            const data = {
+                url: pathname,
+                id: `${directory}/${name}`,
+                name,
+                path: directory,
+            };
+
+            return getTagValue(element, 'D:resourcetype', '')
+                ? createCollectionDirectory(data)
+                : createCollectionFile({ ...data, size });
+
+        });
+};
diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts
index 8cf515c..3b98f75 100644
--- a/src/services/collection-service/collection-service.ts
+++ b/src/services/collection-service/collection-service.ts
@@ -4,40 +4,25 @@
 
 import { CommonResourceService } from "../../common/api/common-resource-service";
 import { CollectionResource } from "../../models/collection";
-import axios, { AxiosInstance } from "axios";
-import { KeepService } from "../keep-service/keep-service";
-import { FilterBuilder } from "../../common/api/filter-builder";
-import { CollectionFile, createCollectionFile, createCollectionDirectory, createCollectionFilesTree, CollectionFileType, CollectionDirectory } from "../../models/collection-file";
-import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser";
-import * as _ from "lodash";
-import { KeepManifestStream } from "../../models/keep-manifest";
+import { AxiosInstance } from "axios";
+import { CollectionFile, CollectionDirectory } from "../../models/collection-file";
 import { WebDAV } from "../../common/webdav";
 import { AuthService } from "../auth-service/auth-service";
-import { mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
-import { getTagValue } from "../../common/xml";
+import { mapTreeValues } from "../../models/tree";
+import { parseFilesResponse } from "./collection-service-files-response";
 
 export type UploadProgress = (fileId: number, loaded: number, total: number, currentTime: number) => void;
 
 export class CollectionService extends CommonResourceService<CollectionResource> {
-    constructor(serverApi: AxiosInstance, private keepService: KeepService, private webdavClient: WebDAV, private authService: AuthService) {
+    constructor(serverApi: AxiosInstance, private webdavClient: WebDAV, private authService: AuthService) {
         super(serverApi, "collections");
     }
 
     async files(uuid: string) {
         const request = await this.webdavClient.propfind(`/c=${uuid}`);
         if (request.responseXML != null) {
-            const files = this.extractFilesData(request.responseXML);
-            const tree = createCollectionFilesTree(files);
-            const sortedTree = mapTree(node => {
-                const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
-                children.sort((a, b) =>
-                    a.value.type !== b.value.type
-                        ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
-                        : a.value.name.localeCompare(b.value.name)
-                );
-                return { ...node, children: children.map(child => child.id) };
-            })(tree);
-            return sortedTree;
+            const filesTree = parseFilesResponse(request.responseXML);
+            return mapTreeValues(this.extendFileURL)(filesTree);
         }
         return Promise.reject();
     }
@@ -46,120 +31,38 @@ export class CollectionService extends CommonResourceService<CollectionResource>
         return this.webdavClient.delete(`/c=${collectionUuid}${filePath}`);
     }
 
-    extractFilesData(document: Document) {
-        const collectionUrlPrefix = /\/c=[0-9a-zA-Z\-]*/;
-        return Array
-            .from(document.getElementsByTagName('D:response'))
-            .slice(1) // omit first element which is collection itself
-            .map(element => {
-                const name = getTagValue(element, 'D:displayname', '');
-                const size = parseInt(getTagValue(element, 'D:getcontentlength', '0'), 10);
-                const pathname = getTagValue(element, 'D:href', '');
-                const nameSuffix = `/${name || ''}`;
-                const directory = pathname
-                    .replace(collectionUrlPrefix, '')
-                    .replace(nameSuffix, '');
-                const href = this.webdavClient.defaults.baseURL + pathname + '?api_token=' + this.authService.getApiToken();
-
-                const data = {
-                    url: href,
-                    id: `${directory}/${name}`,
-                    name,
-                    path: directory,
-                };
+    async uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress) {
+        for (let idx = 0; idx < files.length; idx++) {
+            await this.uploadFile(collectionUuid, files[idx], idx, onProgress);
+        }
+    }
 
-                return getTagValue(element, 'D:resourcetype', '')
-                    ? createCollectionDirectory(data)
-                    : createCollectionFile({ ...data, size });
+    private extendFileURL = (file: CollectionDirectory | CollectionFile) => ({
+        ...file,
+        url: this.webdavClient.defaults.baseURL + file.url + '?api_token=' + this.authService.getApiToken()
+    })
 
-            });
+    private uploadFile(collectionUuid: string, file: File, fileId: number, onProgress: UploadProgress = () => { return; }) {
+        return this.readFile(file).then(content =>
+            this.webdavClient.put(`/c=${collectionUuid}/${file.name}`, content, {
+                headers: {
+                    'Content-Type': 'text/octet-stream'
+                },
+                onUploadProgress: (e: ProgressEvent) => {
+                    onProgress(fileId, e.loaded, e.total, Date.now());
+                }
+            })
+        );
     }
 
-
     private readFile(file: File): Promise<ArrayBuffer> {
         return new Promise<ArrayBuffer>(resolve => {
             const reader = new FileReader();
             reader.onload = () => {
                 resolve(reader.result as ArrayBuffer);
             };
-
             reader.readAsArrayBuffer(file);
         });
     }
 
-    private uploadFile(keepServiceHost: string, file: File, fileId: number, onProgress?: UploadProgress): Promise<CollectionFile> {
-        return this.readFile(file).then(content => {
-            return axios.post<string>(keepServiceHost, content, {
-                headers: {
-                    'Content-Type': 'text/octet-stream'
-                },
-                onUploadProgress: (e: ProgressEvent) => {
-                    if (onProgress) {
-                        onProgress(fileId, e.loaded, e.total, Date.now());
-                    }
-                    console.log(`${e.loaded} / ${e.total}`);
-                }
-            }).then(data => createCollectionFile({
-                id: data.data,
-                name: file.name,
-                size: file.size
-            }));
-        });
-    }
-
-    private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise<CollectionResource> {
-        const collection = await this.get(collectionUuid);
-        const manifest: KeepManifestStream[] = parseKeepManifestText(collection.manifestText);
-
-        files.forEach(f => {
-            let kms = manifest.find(stream => stream.name === f.path);
-            if (!kms) {
-                kms = {
-                    files: [],
-                    locators: [],
-                    name: f.path
-                };
-                manifest.push(kms);
-            }
-            kms.locators.push(f.id);
-            const len = kms.files.length;
-            const nextPos = len > 0
-                ? parseInt(kms.files[len - 1].position, 10) + kms.files[len - 1].size
-                : 0;
-            kms.files.push({
-                name: f.name,
-                position: nextPos.toString(),
-                size: f.size
-            });
-        });
-
-        console.log(manifest);
-
-        const manifestText = stringifyKeepManifest(manifest);
-        const data = { ...collection, manifestText };
-        return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data));
-    }
-
-    uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress): Promise<CollectionResource | never> {
-        const filters = FilterBuilder.create()
-            .addEqual("service_type", "proxy");
-
-        return this.keepService.list({ filters }).then(data => {
-            if (data.items && data.items.length > 0) {
-                const serviceHost =
-                    (data.items[0].serviceSslFlag ? "https://" : "http://") +
-                    data.items[0].serviceHost +
-                    ":" + data.items[0].servicePort;
-
-                console.log("serviceHost", serviceHost);
-
-                const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx, onProgress));
-                return Promise.all(files$).then(values => {
-                    return this.updateManifest(collectionUuid, values);
-                });
-            } else {
-                return Promise.reject("Missing keep service host");
-            }
-        });
-    }
 }
diff --git a/src/services/services.ts b/src/services/services.ts
index eedd124..9b6495f 100644
--- a/src/services/services.ts
+++ b/src/services/services.ts
@@ -30,7 +30,7 @@ export const createServices = (config: Config) => {
     const projectService = new ProjectService(apiClient);
     const linkService = new LinkService(apiClient);
     const favoriteService = new FavoriteService(linkService, groupsService);
-    const collectionService = new CollectionService(apiClient, keepService, webdavClient, authService);
+    const collectionService = new CollectionService(apiClient, webdavClient, authService);
     const tagService = new TagService(linkService);
     const collectionFilesService = new CollectionFilesService(collectionService);
 
diff --git a/src/store/collections/creator/collection-creator-action.ts b/src/store/collections/creator/collection-creator-action.ts
index d0a66b4..984a3c9 100644
--- a/src/store/collections/creator/collection-creator-action.ts
+++ b/src/store/collections/creator/collection-creator-action.ts
@@ -17,9 +17,9 @@ export const collectionCreateActions = unionize({
     CREATE_COLLECTION: ofType<{}>(),
     CREATE_COLLECTION_SUCCESS: ofType<{}>(),
 }, {
-    tag: 'type',
-    value: 'payload'
-});
+        tag: 'type',
+        value: 'payload'
+    });
 
 export const createCollection = (collection: Partial<CollectionResource>, files: File[]) =>
     (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
@@ -34,11 +34,11 @@ export const createCollection = (collection: Partial<CollectionResource>, files:
                     (fileId, loaded, total, currentTime) => {
                         dispatch(collectionUploaderActions.SET_UPLOAD_PROGRESS({ fileId, loaded, total, currentTime }));
                     })
-                .then(collection => {
-                    dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
-                    dispatch(reset('collectionCreateDialog'));
-                    dispatch(collectionUploaderActions.CLEAR_UPLOAD());
-                });
+                    .then(() => {
+                        dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
+                        dispatch(reset('collectionCreateDialog'));
+                        dispatch(collectionUploaderActions.CLEAR_UPLOAD());
+                    });
                 return collection;
             });
     };

commit a49f059145a3054052e1c79555bdeadf660d1c6a
Author: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
Date:   Mon Aug 13 16:31:22 2018 +0200

    Add onUploadProgress callback to webdav
    
    Feature #14013
    
    Arvados-DCO-1.1-Signed-off-by: Michal Klobukowski <michal.klobukowski at contractors.roche.com>

diff --git a/src/common/webdav.test.ts b/src/common/webdav.test.ts
index d96465b..e1eefad 100644
--- a/src/common/webdav.test.ts
+++ b/src/common/webdav.test.ts
@@ -41,15 +41,15 @@ describe('WebDAV', () => {
 
     it('PUT', async () => {
         const { open, send, load, progress, createRequest } = mockCreateRequest();
-        const onProgress = jest.fn();
+        const onUploadProgress = jest.fn();
         const webdav = new WebDAV(undefined, createRequest);
-        const promise = webdav.put('foo', 'Test data', { onProgress });
+        const promise = webdav.put('foo', 'Test data', { onUploadProgress });
         progress();
         load();
         const request = await promise;
         expect(open).toHaveBeenCalledWith('PUT', 'foo');
         expect(send).toHaveBeenCalledWith('Test data');
-        expect(onProgress).toHaveBeenCalled();
+        expect(onUploadProgress).toHaveBeenCalled();
         expect(request).toBeInstanceOf(XMLHttpRequest);
     });
 
diff --git a/src/common/webdav.ts b/src/common/webdav.ts
index 57caebc..27e1f22 100644
--- a/src/common/webdav.ts
+++ b/src/common/webdav.ts
@@ -58,8 +58,8 @@ export class WebDAV {
                 .keys(headers)
                 .forEach(key => r.setRequestHeader(key, headers[key]));
 
-            if (config.onProgress) {
-                r.addEventListener('progress', config.onProgress);
+            if (config.onUploadProgress) {
+                r.upload.addEventListener('progress', config.onUploadProgress);
             }
 
             r.addEventListener('load', () => resolve(r));
@@ -73,7 +73,7 @@ export interface WebDAVRequestConfig {
     headers?: {
         [key: string]: string;
     };
-    onProgress?: (event: ProgressEvent) => void;
+    onUploadProgress?: (event: ProgressEvent) => void;
 }
 
 interface WebDAVDefaults {
@@ -86,5 +86,5 @@ interface RequestConfig {
     url: string;
     headers?: { [key: string]: string };
     data?: any;
-    onProgress?: (event: ProgressEvent) => void;
+    onUploadProgress?: (event: ProgressEvent) => void;
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list