[ARVADOS-WORKBENCH2] created: 1.4.1-368-g7131c732
Git user
git at public.arvados.org
Mon Jun 15 19:17:03 UTC 2020
at 7131c732cffb1383099b5e1593b3ee4b48635df5 (commit)
commit 7131c732cffb1383099b5e1593b3ee4b48635df5
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Mon Jun 15 16:03:58 2020 -0300
15610: Avoids loading the file list on big collections, offers manual loading.
After the previous performance enhancements, 75% of the time spent to show
the collection's files goes to the WebDAV request + parsing, so to avoid
inadvertently freezing the UI, when the file_count field passes a predefined
value (now 40k files), the user gets the option to manually load the file
listing by clicking on a button.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/src/components/collection-panel-files/collection-panel-files.tsx b/src/components/collection-panel-files/collection-panel-files.tsx
index 3a2d55fb..4d7e9384 100644
--- a/src/components/collection-panel-files/collection-panel-files.tsx
+++ b/src/components/collection-panel-files/collection-panel-files.tsx
@@ -14,12 +14,14 @@ export interface CollectionPanelFilesProps {
items: Array<TreeItem<FileTreeData>>;
isWritable: boolean;
isLoading: boolean;
+ tooManyFiles: boolean;
onUploadDataClick: () => void;
onItemMenuOpen: (event: React.MouseEvent<HTMLElement>, item: TreeItem<FileTreeData>, isWritable: boolean) => void;
onOptionsMenuOpen: (event: React.MouseEvent<HTMLElement>, isWritable: boolean) => void;
onSelectionToggle: (event: React.MouseEvent<HTMLElement>, item: TreeItem<FileTreeData>) => void;
onCollapseToggle: (id: string, status: TreeItemStatus) => void;
onFileClick: (id: string) => void;
+ loadFilesFunc: () => void;
currentItemUuid?: string;
}
@@ -55,7 +57,7 @@ const styles: StyleRulesCallback<CssRules> = theme => ({
export const CollectionPanelFiles =
withStyles(styles)(
({ onItemMenuOpen, onOptionsMenuOpen, onUploadDataClick, classes,
- isWritable, isLoading, ...treeProps }: CollectionPanelFilesProps & WithStyles<CssRules>) =>
+ isWritable, isLoading, tooManyFiles, loadFilesFunc, ...treeProps }: CollectionPanelFilesProps & WithStyles<CssRules>) =>
<Card data-cy='collection-files-panel' className={classes.root}>
<CardHeader
title="Files"
@@ -72,26 +74,33 @@ export const CollectionPanelFiles =
Upload data
</Button>
} />
- <CardHeader
- className={classes.cardSubheader}
- action={
- <Tooltip title="More options" disableFocusListener>
- <IconButton
- data-cy='collection-files-panel-options-btn'
- onClick={(ev) => onOptionsMenuOpen(ev, isWritable)}>
- <CustomizeTableIcon />
- </IconButton>
- </Tooltip>
- } />
- <Grid container justify="space-between">
- <Typography variant="caption" className={classes.nameHeader}>
- Name
- </Typography>
- <Typography variant="caption" className={classes.fileSizeHeader}>
- File size
- </Typography>
- </Grid>
- { isLoading
- ? <div className={classes.centeredLabel}>(loading files...)</div>
- : <FileTree onMenuOpen={(ev, item) => onItemMenuOpen(ev, item, isWritable)} {...treeProps} /> }
+ { tooManyFiles
+ ? <div className={classes.centeredLabel}>
+ File listing may take some time, please click to browse: <Button onClick={loadFilesFunc}><DownloadIcon/>Show files</Button>
+ </div>
+ : <>
+ <CardHeader
+ className={classes.cardSubheader}
+ action={
+ <Tooltip title="More options" disableFocusListener>
+ <IconButton
+ data-cy='collection-files-panel-options-btn'
+ onClick={(ev) => onOptionsMenuOpen(ev, isWritable)}>
+ <CustomizeTableIcon />
+ </IconButton>
+ </Tooltip>
+ } />
+ <Grid container justify="space-between">
+ <Typography variant="caption" className={classes.nameHeader}>
+ Name
+ </Typography>
+ <Typography variant="caption" className={classes.fileSizeHeader}>
+ File size
+ </Typography>
+ </Grid>
+ { isLoading
+ ? <div className={classes.centeredLabel}>(loading files...)</div>
+ : <FileTree onMenuOpen={(ev, item) => onItemMenuOpen(ev, item, isWritable)} {...treeProps} /> }
+ </>
+ }
</Card>);
diff --git a/src/store/collection-panel/collection-panel-action.ts b/src/store/collection-panel/collection-panel-action.ts
index 9922d8b5..35c3c3d3 100644
--- a/src/store/collection-panel/collection-panel-action.ts
+++ b/src/store/collection-panel/collection-panel-action.ts
@@ -3,7 +3,7 @@
// SPDX-License-Identifier: AGPL-3.0
import { Dispatch } from "redux";
-import { loadCollectionFiles } from "./collection-panel-files/collection-panel-files-actions";
+import { loadCollectionFiles, COLLECTION_PANEL_LOAD_FILES_THRESHOLD } from "./collection-panel-files/collection-panel-files-actions";
import { CollectionResource } from '~/models/collection';
import { collectionPanelFilesAction } from "./collection-panel-files/collection-panel-files-actions";
import { createTree } from "~/models/tree";
@@ -21,7 +21,8 @@ import { addProperty, deleteProperty } from "~/lib/resource-properties";
export const collectionPanelActions = unionize({
SET_COLLECTION: ofType<CollectionResource>(),
LOAD_COLLECTION: ofType<{ uuid: string }>(),
- LOAD_COLLECTION_SUCCESS: ofType<{ item: CollectionResource }>()
+ LOAD_COLLECTION_SUCCESS: ofType<{ item: CollectionResource }>(),
+ LOAD_BIG_COLLECTIONS: ofType<boolean>(),
});
export type CollectionPanelAction = UnionOf<typeof collectionPanelActions>;
@@ -36,7 +37,10 @@ export const loadCollectionPanel = (uuid: string) =>
dispatch(loadDetailsPanel(collection.uuid));
dispatch(collectionPanelActions.LOAD_COLLECTION_SUCCESS({ item: collection }));
dispatch(resourcesActions.SET_RESOURCES([collection]));
- dispatch<any>(loadCollectionFiles(collection.uuid));
+ if (collection.fileCount <= COLLECTION_PANEL_LOAD_FILES_THRESHOLD &&
+ !getState().collectionPanel.loadBigCollections) {
+ dispatch<any>(loadCollectionFiles(collection.uuid));
+ }
return collection;
};
diff --git a/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts b/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
index fe93eef2..204d4c0e 100644
--- a/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
+++ b/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
@@ -27,6 +27,7 @@ export const collectionPanelFilesAction = unionize({
export type CollectionPanelFilesAction = UnionOf<typeof collectionPanelFilesAction>;
export const COLLECTION_PANEL_LOAD_FILES = 'collectionPanelLoadFiles';
+export const COLLECTION_PANEL_LOAD_FILES_THRESHOLD = 40000;
export const loadCollectionFiles = (uuid: string) =>
async (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
diff --git a/src/store/collection-panel/collection-panel-reducer.ts b/src/store/collection-panel/collection-panel-reducer.ts
index f09b0198..18590181 100644
--- a/src/store/collection-panel/collection-panel-reducer.ts
+++ b/src/store/collection-panel/collection-panel-reducer.ts
@@ -7,15 +7,22 @@ import { CollectionResource } from "~/models/collection";
export interface CollectionPanelState {
item: CollectionResource | null;
+ loadBigCollections: boolean;
}
const initialState = {
- item: null
+ item: null,
+ loadBigCollections: false,
};
export const collectionPanelReducer = (state: CollectionPanelState = initialState, action: CollectionPanelAction) =>
collectionPanelActions.match(action, {
default: () => state,
- SET_COLLECTION: (item) => ({ ...state, item }),
- LOAD_COLLECTION_SUCCESS: ({ item }) => ({ ...state, item })
+ SET_COLLECTION: (item) => ({
+ ...state,
+ item,
+ loadBigCollections: false,
+ }),
+ LOAD_COLLECTION_SUCCESS: ({ item }) => ({ ...state, item }),
+ LOAD_BIG_COLLECTIONS: (loadBigCollections) => ({ ...state, loadBigCollections}),
});
diff --git a/src/views/collection-panel/collection-panel.tsx b/src/views/collection-panel/collection-panel.tsx
index 27a68541..1cfa48de 100644
--- a/src/views/collection-panel/collection-panel.tsx
+++ b/src/views/collection-panel/collection-panel.tsx
@@ -16,7 +16,7 @@ import { DetailsAttribute } from '~/components/details-attribute/details-attribu
import { CollectionResource } from '~/models/collection';
import { CollectionPanelFiles } from '~/views-components/collection-panel-files/collection-panel-files';
import { CollectionTagForm } from './collection-tag-form';
-import { deleteCollectionTag, navigateToProcess } from '~/store/collection-panel/collection-panel-action';
+import { deleteCollectionTag, navigateToProcess, collectionPanelActions } from '~/store/collection-panel/collection-panel-action';
import { getResource } from '~/store/resources/resources';
import { openContextMenu } from '~/store/context-menu/context-menu-actions';
import { ContextMenuKind } from '~/views-components/context-menu/context-menu';
@@ -29,7 +29,7 @@ import { GroupResource } from '~/models/group';
import { UserResource } from '~/models/user';
import { getUserUuid } from '~/common/getuser';
import { getProgressIndicator } from '~/store/progress-indicator/progress-indicator-reducer';
-import { COLLECTION_PANEL_LOAD_FILES } from '~/store/collection-panel/collection-panel-files/collection-panel-files-actions';
+import { COLLECTION_PANEL_LOAD_FILES, loadCollectionFiles, COLLECTION_PANEL_LOAD_FILES_THRESHOLD } from '~/store/collection-panel/collection-panel-files/collection-panel-files-actions';
type CssRules = 'card' | 'iconHeader' | 'tag' | 'label' | 'value' | 'link' | 'centeredLabel' | 'readOnlyIcon';
@@ -73,6 +73,7 @@ interface CollectionPanelDataProps {
item: CollectionResource;
isWritable: boolean;
isLoadingFiles: boolean;
+ tooManyFiles: boolean;
}
type CollectionPanelProps = CollectionPanelDataProps & DispatchProp
@@ -93,11 +94,12 @@ export const CollectionPanel = withStyles(styles)(
}
const loadingFilesIndicator = getProgressIndicator(COLLECTION_PANEL_LOAD_FILES)(state.progressIndicator);
const isLoadingFiles = loadingFilesIndicator && loadingFilesIndicator!.working || false;
- return { item, isWritable, isLoadingFiles };
+ const tooManyFiles = !state.collectionPanel.loadBigCollections && item && item.fileCount > COLLECTION_PANEL_LOAD_FILES_THRESHOLD || false;
+ return { item, isWritable, isLoadingFiles, tooManyFiles };
})(
class extends React.Component<CollectionPanelProps> {
render() {
- const { classes, item, dispatch, isWritable, isLoadingFiles } = this.props;
+ const { classes, item, dispatch, isWritable, isLoadingFiles, tooManyFiles } = this.props;
return item
? <>
<Card data-cy='collection-info-panel' className={classes.card}>
@@ -188,7 +190,15 @@ export const CollectionPanel = withStyles(styles)(
</CardContent>
</Card>
<div className={classes.card}>
- <CollectionPanelFiles isWritable={isWritable} isLoading={isLoadingFiles} />
+ <CollectionPanelFiles
+ isWritable={isWritable}
+ isLoading={isLoadingFiles}
+ tooManyFiles={tooManyFiles}
+ loadFilesFunc={() => {
+ dispatch(collectionPanelActions.LOAD_BIG_COLLECTIONS(true));
+ dispatch<any>(loadCollectionFiles(this.props.item.uuid));
+ }
+ } />
</div>
</>
: null;
commit 6381c9957d1b936937ebe79a9e4b5c08f74dce16
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Jun 12 19:11:59 2020 -0300
15610: Removes dead code about collection manifest parsing.
File handling is done via WebDAV, I guess the code was written before
we had the service available.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/src/services/collection-files-service/collection-files-service.ts b/src/services/collection-files-service/collection-files-service.ts
deleted file mode 100644
index f8e7de98..00000000
--- a/src/services/collection-files-service/collection-files-service.ts
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-import { CollectionService } from "../collection-service/collection-service";
-import { parseKeepManifestText, stringifyKeepManifest } from "./collection-manifest-parser";
-import { mapManifestToCollectionFilesTree } from "./collection-manifest-mapper";
-
-export class CollectionFilesService {
-
- constructor(private collectionService: CollectionService) { }
-
- getFiles(collectionUuid: string) {
- return this.collectionService
- .get(collectionUuid)
- .then(collection =>
- mapManifestToCollectionFilesTree(
- parseKeepManifestText(
- collection.manifestText
- )
- )
- );
- }
-
- async renameFile(collectionUuid: string, file: { name: string, path: string }, newName: string) {
- const collection = await this.collectionService.get(collectionUuid);
- const manifest = parseKeepManifestText(collection.manifestText);
- const updatedManifest = manifest.map(
- stream => stream.name === file.path
- ? {
- ...stream,
- files: stream.files.map(
- f => f.name === file.name
- ? { ...f, name: newName }
- : f
- )
- }
- : stream
- );
- const manifestText = stringifyKeepManifest(updatedManifest);
- return this.collectionService.update(collectionUuid, { manifestText });
- }
-
- async deleteFile(collectionUuid: string, file: { name: string, path: string }) {
- const collection = await this.collectionService.get(collectionUuid);
- const manifest = parseKeepManifestText(collection.manifestText);
- const updatedManifest = manifest.map(stream =>
- stream.name === file.path
- ? {
- ...stream,
- files: stream.files.filter(f => f.name !== file.name)
- }
- : stream
- );
- const manifestText = stringifyKeepManifest(updatedManifest);
- return this.collectionService.update(collectionUuid, { manifestText });
- }
-}
diff --git a/src/services/collection-files-service/collection-manifest-mapper.test.ts b/src/services/collection-files-service/collection-manifest-mapper.test.ts
deleted file mode 100644
index 698a6bb7..00000000
--- a/src/services/collection-files-service/collection-manifest-mapper.test.ts
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-import { parseKeepManifestText } from "./collection-manifest-parser";
-import { mapManifestToFiles, mapManifestToDirectories, mapManifestToCollectionFilesTree, mapCollectionFilesTreeToManifest } from "./collection-manifest-mapper";
-
-test('mapManifestToFiles', () => {
- const manifestText = `. 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n./c d41d8cd98f00b204e9800998ecf8427e+0 0:0:d`;
- const manifest = parseKeepManifestText(manifestText);
- const files = mapManifestToFiles(manifest);
- expect(files).toEqual([{
- path: '',
- id: '/a',
- name: 'a',
- size: 0,
- type: 'file',
- url: ''
- }, {
- path: '',
- id: '/b',
- name: 'b',
- size: 0,
- type: 'file',
- url: ''
- }, {
- path: '',
- id: '/output.txt',
- name: 'output.txt',
- size: 33,
- type: 'file',
- url: ''
- }, {
- path: '/c',
- id: '/c/d',
- name: 'd',
- size: 0,
- type: 'file',
- url: ''
- },]);
-});
-
-test('mapManifestToDirectories', () => {
- const manifestText = `./c/user/results 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n`;
- const manifest = parseKeepManifestText(manifestText);
- const directories = mapManifestToDirectories(manifest);
- expect(directories).toEqual([{
- path: "",
- id: '/c',
- name: 'c',
- type: 'directory',
- url: ''
- }, {
- path: '/c',
- id: '/c/user',
- name: 'user',
- type: 'directory',
- url: ''
- }, {
- path: '/c/user',
- id: '/c/user/results',
- name: 'results',
- type: 'directory',
- url: ''
- },]);
-});
-
-test('mapCollectionFilesTreeToManifest', () => {
- const manifestText = `. 930625b054ce894ac40596c3f5a0d947+33 0:22:test.txt\n./c/user/results 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n`;
- const tree = mapManifestToCollectionFilesTree(parseKeepManifestText(manifestText));
- const manifest = mapCollectionFilesTreeToManifest(tree);
- expect(manifest).toEqual([{
- name: '',
- locators: [],
- files: [{
- name: 'test.txt',
- position: '',
- size: 22
- },],
- }, {
- name: '/c/user/results',
- locators: [],
- files: [{
- name: 'a',
- position: '',
- size: 0
- }, {
- name: 'b',
- position: '',
- size: 0
- }, {
- name: 'output.txt',
- position: '',
- size: 33
- },],
- },]);
-
-});
\ No newline at end of file
diff --git a/src/services/collection-files-service/collection-manifest-mapper.ts b/src/services/collection-files-service/collection-manifest-mapper.ts
deleted file mode 100644
index 6e64f833..00000000
--- a/src/services/collection-files-service/collection-manifest-mapper.ts
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-import { uniqBy, groupBy } from 'lodash';
-import { KeepManifestStream, KeepManifestStreamFile, KeepManifest } from "~/models/keep-manifest";
-import { TreeNode, setNode, createTree, getNodeDescendantsIds, getNodeValue, TreeNodeStatus } from '~/models/tree';
-import { CollectionFilesTree, CollectionFile, CollectionDirectory, createCollectionDirectory, createCollectionFile, CollectionFileType } from '../../models/collection-file';
-
-export const mapCollectionFilesTreeToManifest = (tree: CollectionFilesTree): KeepManifest => {
- const values = getNodeDescendantsIds('')(tree).map(id => getNodeValue(id)(tree));
- const files = values.filter(value => value && value.type === CollectionFileType.FILE) as CollectionFile[];
- const fileGroups = groupBy(files, file => file.path);
- return Object
- .keys(fileGroups)
- .map(dirName => ({
- name: dirName,
- locators: [],
- files: fileGroups[dirName].map(mapCollectionFile)
- }));
-};
-
-export const mapManifestToCollectionFilesTree = (manifest: KeepManifest): CollectionFilesTree =>
- manifestToCollectionFiles(manifest)
- .map(mapCollectionFileToTreeNode)
- .reduce((tree, node) => setNode(node)(tree), createTree<CollectionFile>());
-
-
-export const mapCollectionFileToTreeNode = (file: CollectionFile): TreeNode<CollectionFile> => ({
- children: [],
- id: file.id,
- parent: file.path,
- value: file,
- active: false,
- selected: false,
- expanded: false,
- status: TreeNodeStatus.INITIAL,
-});
-
-export const manifestToCollectionFiles = (manifest: KeepManifest): Array<CollectionDirectory | CollectionFile> => ([
- ...mapManifestToDirectories(manifest),
- ...mapManifestToFiles(manifest)
-]);
-
-export const mapManifestToDirectories = (manifest: KeepManifest): CollectionDirectory[] =>
- uniqBy(
- manifest
- .map(mapStreamDirectory)
- .map(splitDirectory)
- .reduce((all, splitted) => ([...all, ...splitted]), []),
- directory => directory.id);
-
-export const mapManifestToFiles = (manifest: KeepManifest): CollectionFile[] =>
- manifest
- .map(stream => stream.files.map(mapStreamFile(stream)))
- .reduce((all, current) => ([...all, ...current]), []);
-
-const splitDirectory = (directory: CollectionDirectory): CollectionDirectory[] => {
- return directory.name
- .split('/')
- .slice(1)
- .map(mapPathComponentToDirectory);
-};
-
-const mapPathComponentToDirectory = (component: string, index: number, components: string[]): CollectionDirectory =>
- createCollectionDirectory({
- path: index === 0 ? '' : joinPathComponents(components, index),
- id: joinPathComponents(components, index + 1),
- name: component,
- });
-
-const joinPathComponents = (components: string[], index: number) =>
- `/${components.slice(0, index).join('/')}`;
-
-const mapCollectionFile = (file: CollectionFile): KeepManifestStreamFile => ({
- name: file.name,
- position: '',
- size: file.size
-});
-
-const mapStreamDirectory = (stream: KeepManifestStream): CollectionDirectory =>
- createCollectionDirectory({
- path: '',
- id: stream.name,
- name: stream.name,
- });
-
-const mapStreamFile = (stream: KeepManifestStream) =>
- (file: KeepManifestStreamFile): CollectionFile =>
- createCollectionFile({
- path: stream.name,
- id: `${stream.name}/${file.name}`,
- name: file.name,
- size: file.size,
- });
-
diff --git a/src/services/collection-files-service/collection-manifest-parser.test.ts b/src/services/collection-files-service/collection-manifest-parser.test.ts
deleted file mode 100644
index 09525d86..00000000
--- a/src/services/collection-files-service/collection-manifest-parser.test.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-import { parseKeepManifestText, parseKeepManifestStream, stringifyKeepManifest } from "./collection-manifest-parser";
-
-describe('parseKeepManifestText', () => {
- it('should parse text into streams', () => {
- const manifestText = `. 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n./c d41d8cd98f00b204e9800998ecf8427e+0 0:0:d\n`;
- const manifest = parseKeepManifestText(manifestText);
- expect(manifest[0].name).toBe('');
- expect(manifest[1].name).toBe('/c');
- expect(manifest.length).toBe(2);
- });
-});
-
-describe('parseKeepManifestStream', () => {
- const streamText = './c 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt';
- const stream = parseKeepManifestStream(streamText);
-
- it('should parse stream name', () => {
- expect(stream.name).toBe('/c');
- });
- it('should parse stream locators', () => {
- expect(stream.locators).toEqual(['930625b054ce894ac40596c3f5a0d947+33']);
- });
- it('should parse stream files', () => {
- expect(stream.files).toEqual([
- { name: 'a', position: '0', size: 0 },
- { name: 'b', position: '0', size: 0 },
- { name: 'output.txt', position: '0', size: 33 },
- ]);
- });
-});
-
-test('stringifyKeepManifest', () => {
- const manifestText = `. 930625b054ce894ac40596c3f5a0d947+33 0:22:test.txt\n./c/user/results 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n`;
- const manifest = parseKeepManifestText(manifestText);
- expect(stringifyKeepManifest(manifest)).toEqual(`. 930625b054ce894ac40596c3f5a0d947+33 0:22:test.txt\n./c/user/results 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n`);
-});
\ No newline at end of file
diff --git a/src/services/collection-files-service/collection-manifest-parser.ts b/src/services/collection-files-service/collection-manifest-parser.ts
deleted file mode 100644
index d564f33e..00000000
--- a/src/services/collection-files-service/collection-manifest-parser.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-import { KeepManifestStream, KeepManifestStreamFile, KeepManifest } from "~/models/keep-manifest";
-
-/**
- * Documentation [http://doc.arvados.org/api/storage.html](http://doc.arvados.org/api/storage.html)
- */
-export const parseKeepManifestText: (text: string) => KeepManifestStream[] = (text: string) =>
- text
- .split(/\n/)
- .filter(streamText => streamText.length > 0)
- .map(parseKeepManifestStream);
-
-/**
- * Documentation [http://doc.arvados.org/api/storage.html](http://doc.arvados.org/api/storage.html)
- */
-export const parseKeepManifestStream = (stream: string): KeepManifestStream => {
- const tokens = stream.split(' ');
- return {
- name: streamName(tokens),
- locators: locators(tokens),
- files: files(tokens)
- };
-};
-
-export const stringifyKeepManifest = (manifest: KeepManifest) =>
- manifest.map(stringifyKeepManifestStream).join('');
-
-export const stringifyKeepManifestStream = (stream: KeepManifestStream) =>
- `.${stream.name} ${stream.locators.join(' ')} ${stream.files.map(stringifyFile).join(' ')}\n`;
-
-const FILE_LOCATOR_REGEXP = /^([0-9a-f]{32})\+([0-9]+)(\+[A-Z][-A-Za-z0-9 at _]*)*$/;
-
-const FILE_REGEXP = /([0-9]+):([0-9]+):(.*)/;
-
-const streamName = (tokens: string[]) => tokens[0].slice(1);
-
-const locators = (tokens: string[]) => tokens.filter(isFileLocator);
-
-const files = (tokens: string[]) => tokens.filter(isFile).map(parseFile);
-
-const isFileLocator = (token: string) => FILE_LOCATOR_REGEXP.test(token);
-
-const isFile = (token: string) => FILE_REGEXP.test(token);
-
-const parseFile = (token: string): KeepManifestStreamFile => {
- const match = FILE_REGEXP.exec(token);
- const [position, size, name] = match!.slice(1);
- return { name, position, size: parseInt(size, 10) };
-};
-
-const stringifyFile = (file: KeepManifestStreamFile) =>
- `${file.position}:${file.size}:${file.name}`;
diff --git a/src/services/services.ts b/src/services/services.ts
index af547dec..41dc831e 100644
--- a/src/services/services.ts
+++ b/src/services/services.ts
@@ -12,7 +12,6 @@ import { LinkService } from "./link-service/link-service";
import { FavoriteService } from "./favorite-service/favorite-service";
import { CollectionService } from "./collection-service/collection-service";
import { TagService } from "./tag-service/tag-service";
-import { CollectionFilesService } from "./collection-files-service/collection-files-service";
import { KeepService } from "./keep-service/keep-service";
import { WebDAV } from "~/common/webdav";
import { Config } from "~/common/config";
@@ -81,7 +80,6 @@ export const createServices = (config: Config, actions: ApiActions, useApiClient
const ancestorsService = new AncestorService(groupsService, userService);
const authService = new AuthService(apiClient, config.rootUrl, actions);
const collectionService = new CollectionService(apiClient, webdavClient, authService, actions);
- const collectionFilesService = new CollectionFilesService(collectionService);
const favoriteService = new FavoriteService(linkService, groupsService);
const tagService = new TagService(linkService);
const searchService = new SearchService();
@@ -94,7 +92,6 @@ export const createServices = (config: Config, actions: ApiActions, useApiClient
apiClientAuthorizationService,
authService,
authorizedKeysService,
- collectionFilesService,
collectionService,
containerRequestService,
containerService,
commit 4530e2fcf4cac004af67cecddefc742a83c82b16
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Jun 12 17:54:31 2020 -0300
15610: Shows status indicator while loading collection's file data.
This includes the general app progress indicator, and a '(loading files...)'
text label where the file tree should be rendered.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/src/components/collection-panel-files/collection-panel-files.tsx b/src/components/collection-panel-files/collection-panel-files.tsx
index 48b36be1..3a2d55fb 100644
--- a/src/components/collection-panel-files/collection-panel-files.tsx
+++ b/src/components/collection-panel-files/collection-panel-files.tsx
@@ -13,6 +13,7 @@ import { DownloadIcon } from '~/components/icon/icon';
export interface CollectionPanelFilesProps {
items: Array<TreeItem<FileTreeData>>;
isWritable: boolean;
+ isLoading: boolean;
onUploadDataClick: () => void;
onItemMenuOpen: (event: React.MouseEvent<HTMLElement>, item: TreeItem<FileTreeData>, isWritable: boolean) => void;
onOptionsMenuOpen: (event: React.MouseEvent<HTMLElement>, isWritable: boolean) => void;
@@ -22,7 +23,7 @@ export interface CollectionPanelFilesProps {
currentItemUuid?: string;
}
-type CssRules = 'root' | 'cardSubheader' | 'nameHeader' | 'fileSizeHeader' | 'uploadIcon' | 'button';
+type CssRules = 'root' | 'cardSubheader' | 'nameHeader' | 'fileSizeHeader' | 'uploadIcon' | 'button' | 'centeredLabel';
const styles: StyleRulesCallback<CssRules> = theme => ({
root: {
@@ -44,12 +45,17 @@ const styles: StyleRulesCallback<CssRules> = theme => ({
button: {
marginRight: -theme.spacing.unit,
marginTop: '0px'
- }
+ },
+ centeredLabel: {
+ fontSize: '0.875rem',
+ textAlign: 'center'
+ },
});
export const CollectionPanelFiles =
withStyles(styles)(
- ({ onItemMenuOpen, onOptionsMenuOpen, onUploadDataClick, classes, isWritable, ...treeProps }: CollectionPanelFilesProps & WithStyles<CssRules>) =>
+ ({ onItemMenuOpen, onOptionsMenuOpen, onUploadDataClick, classes,
+ isWritable, isLoading, ...treeProps }: CollectionPanelFilesProps & WithStyles<CssRules>) =>
<Card data-cy='collection-files-panel' className={classes.root}>
<CardHeader
title="Files"
@@ -85,5 +91,7 @@ export const CollectionPanelFiles =
File size
</Typography>
</Grid>
- <FileTree onMenuOpen={(ev, item) => onItemMenuOpen(ev, item, isWritable)} {...treeProps} />
+ { isLoading
+ ? <div className={classes.centeredLabel}>(loading files...)</div>
+ : <FileTree onMenuOpen={(ev, item) => onItemMenuOpen(ev, item, isWritable)} {...treeProps} /> }
</Card>);
diff --git a/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts b/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
index 9d3ae861..fe93eef2 100644
--- a/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
+++ b/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
@@ -14,6 +14,7 @@ import { filterCollectionFilesBySelection } from './collection-panel-files-state
import { startSubmit, stopSubmit, initialize, FormErrors } from 'redux-form';
import { getDialog } from "~/store/dialog/dialog-reducer";
import { getFileFullPath, sortFilesTree } from "~/services/collection-service/collection-service-files-response";
+import { progressIndicatorActions } from "~/store/progress-indicator/progress-indicator-actions";
export const collectionPanelFilesAction = unionize({
SET_COLLECTION_FILES: ofType<CollectionFilesTree>(),
@@ -25,8 +26,11 @@ export const collectionPanelFilesAction = unionize({
export type CollectionPanelFilesAction = UnionOf<typeof collectionPanelFilesAction>;
+export const COLLECTION_PANEL_LOAD_FILES = 'collectionPanelLoadFiles';
+
export const loadCollectionFiles = (uuid: string) =>
async (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
+ dispatch(progressIndicatorActions.START_WORKING(COLLECTION_PANEL_LOAD_FILES));
const files = await services.collectionService.files(uuid);
// Given the array of directories and files, create the appropriate tree nodes,
@@ -35,6 +39,7 @@ export const loadCollectionFiles = (uuid: string) =>
const sorted = sortFilesTree(tree);
const mapped = mapTreeValues(services.collectionService.extendFileURL)(sorted);
dispatch(collectionPanelFilesAction.SET_COLLECTION_FILES(mapped));
+ dispatch(progressIndicatorActions.STOP_WORKING(COLLECTION_PANEL_LOAD_FILES));
};
export const removeCollectionFiles = (filePaths: string[]) =>
diff --git a/src/views/collection-panel/collection-panel.tsx b/src/views/collection-panel/collection-panel.tsx
index 36625387..27a68541 100644
--- a/src/views/collection-panel/collection-panel.tsx
+++ b/src/views/collection-panel/collection-panel.tsx
@@ -28,6 +28,8 @@ import { IllegalNamingWarning } from '~/components/warning/warning';
import { GroupResource } from '~/models/group';
import { UserResource } from '~/models/user';
import { getUserUuid } from '~/common/getuser';
+import { getProgressIndicator } from '~/store/progress-indicator/progress-indicator-reducer';
+import { COLLECTION_PANEL_LOAD_FILES } from '~/store/collection-panel/collection-panel-files/collection-panel-files-actions';
type CssRules = 'card' | 'iconHeader' | 'tag' | 'label' | 'value' | 'link' | 'centeredLabel' | 'readOnlyIcon';
@@ -70,6 +72,7 @@ const styles: StyleRulesCallback<CssRules> = (theme: ArvadosTheme) => ({
interface CollectionPanelDataProps {
item: CollectionResource;
isWritable: boolean;
+ isLoadingFiles: boolean;
}
type CollectionPanelProps = CollectionPanelDataProps & DispatchProp
@@ -88,11 +91,13 @@ export const CollectionPanel = withStyles(styles)(
isWritable = itemOwner.writableBy.indexOf(currentUserUUID || '') >= 0;
}
}
- return { item, isWritable };
+ const loadingFilesIndicator = getProgressIndicator(COLLECTION_PANEL_LOAD_FILES)(state.progressIndicator);
+ const isLoadingFiles = loadingFilesIndicator && loadingFilesIndicator!.working || false;
+ return { item, isWritable, isLoadingFiles };
})(
class extends React.Component<CollectionPanelProps> {
render() {
- const { classes, item, dispatch, isWritable } = this.props;
+ const { classes, item, dispatch, isWritable, isLoadingFiles } = this.props;
return item
? <>
<Card data-cy='collection-info-panel' className={classes.card}>
@@ -183,7 +188,7 @@ export const CollectionPanel = withStyles(styles)(
</CardContent>
</Card>
<div className={classes.card}>
- <CollectionPanelFiles isWritable={isWritable} />
+ <CollectionPanelFiles isWritable={isWritable} isLoading={isLoadingFiles} />
</div>
</>
: null;
commit bcc074c282de71fe63d2ce23127b58f92c90037c
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Jun 12 16:42:45 2020 -0300
15610: Fixes file downloading.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts
index 77f5bf3b..90441a64 100644
--- a/src/services/collection-service/collection-service.ts
+++ b/src/services/collection-service/collection-service.ts
@@ -68,7 +68,7 @@ export class CollectionService extends TrashableResourceService<CollectionResour
const splittedApiToken = apiToken ? apiToken.split('/') : [];
const userApiToken = `/t=${splittedApiToken[2]}/`;
const splittedPrevFileUrl = file.url.split('/');
- const url = `${baseUrl}/${splittedPrevFileUrl[1]}${userApiToken}${splittedPrevFileUrl[2]}`;
+ const url = `${baseUrl}/${splittedPrevFileUrl[1]}${userApiToken}${splittedPrevFileUrl.slice(2).join('/')}`;
return {
...file,
url
commit b3c45e57f0b488e98bb24b342c37724c8ef7bb5f
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Jun 12 15:07:23 2020 -0300
15610: Enhances performance on tree handling.
Large trees (eg: large collection file hierarchies) handling performance boost
by doing in-place data manipulations instead of functional programming style
data copying.
This enables workbench2 to gracefully show mid-sized collections of around
50k items without making the user wait too much for the UI to respond.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/src/models/collection-file.ts b/src/models/collection-file.ts
index 97afcac6..3951d272 100644
--- a/src/models/collection-file.ts
+++ b/src/models/collection-file.ts
@@ -66,7 +66,6 @@ export const createCollectionFilesTree = (data: Array<CollectionDirectory | Coll
selected: false,
expanded: false,
status: TreeNodeStatus.INITIAL
-
})(tree), createTree<CollectionDirectory | CollectionFile>());
};
diff --git a/src/models/tree.ts b/src/models/tree.ts
index de2f7b71..c7713cbc 100644
--- a/src/models/tree.ts
+++ b/src/models/tree.ts
@@ -43,12 +43,13 @@ export const appendSubtree = <T>(id: string, subtree: Tree<T>) => (tree: Tree<T>
)(subtree) as Tree<T>;
export const setNode = <T>(node: TreeNode<T>) => (tree: Tree<T>): Tree<T> => {
- return pipe(
- (tree: Tree<T>) => getNode(node.id)(tree) === node
- ? tree
- : { ...tree, [node.id]: node },
- addChild(node.parent, node.id)
- )(tree);
+ if (tree[node.id] && tree[node.id] === node) { return tree; }
+
+ tree[node.id] = node;
+ if (tree[node.parent]) {
+ tree[node.parent].children = Array.from(new Set([...tree[node.parent].children, node.id]));
+ }
+ return tree;
};
export const getNodeValue = (id: string) => <T>(tree: Tree<T>) => {
@@ -156,7 +157,6 @@ export const toggleNodeSelection = (id: string) => <T>(tree: Tree<T>) => {
toggleAncestorsSelection(id),
toggleDescendantsSelection(id))(tree)
: tree;
-
};
export const selectNode = (id: string) => <T>(tree: Tree<T>) => {
@@ -235,23 +235,3 @@ const getRootNodeChildrenIds = <T>(tree: Tree<T>) =>
Object
.keys(tree)
.filter(id => getNode(id)(tree)!.parent === TREE_ROOT_ID);
-
-
-const addChild = (parentId: string, childId: string) => <T>(tree: Tree<T>): Tree<T> => {
- if (childId === "") {
- return tree;
- }
- const node = getNode(parentId)(tree);
- if (node) {
- const children = node.children.some(id => id === childId)
- ? node.children
- : [...node.children, childId];
-
- const newNode = children === node.children
- ? node
- : { ...node, children };
-
- return setNode(newNode)(tree);
- }
- return tree;
-};
diff --git a/src/services/collection-service/collection-service-files-response.ts b/src/services/collection-service/collection-service-files-response.ts
index 2e726d0b..5e6f7b83 100644
--- a/src/services/collection-service/collection-service-files-response.ts
+++ b/src/services/collection-service/collection-service-files-response.ts
@@ -50,7 +50,6 @@ export const extractFilesData = (document: Document) => {
return getTagValue(element, 'D:resourcetype', '')
? createCollectionDirectory(data)
: createCollectionFile({ ...data, size });
-
});
};
diff --git a/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.ts b/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.ts
index 57961538..08a71759 100644
--- a/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.ts
+++ b/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.ts
@@ -8,23 +8,25 @@ import { createTree, mapTreeValues, getNode, setNode, getNodeAncestorsIds, getNo
import { CollectionFileType } from "~/models/collection-file";
export const collectionPanelFilesReducer = (state: CollectionPanelFilesState = createTree(), action: CollectionPanelFilesAction) => {
+ // Low-level tree handling setNode() func does in-place data modifications
+ // for performance reasons, so we pass a copy of 'state' to avoid side effects.
return collectionPanelFilesAction.match(action, {
SET_COLLECTION_FILES: files =>
- mergeCollectionPanelFilesStates(state, mapTree(mapCollectionFileToCollectionPanelFile)(files)),
+ mergeCollectionPanelFilesStates({...state}, mapTree(mapCollectionFileToCollectionPanelFile)(files)),
TOGGLE_COLLECTION_FILE_COLLAPSE: data =>
- toggleCollapse(data.id)(state),
+ toggleCollapse(data.id)({...state}),
- TOGGLE_COLLECTION_FILE_SELECTION: data => [state]
+ TOGGLE_COLLECTION_FILE_SELECTION: data => [{...state}]
.map(toggleSelected(data.id))
.map(toggleAncestors(data.id))
.map(toggleDescendants(data.id))[0],
SELECT_ALL_COLLECTION_FILES: () =>
- mapTreeValues(v => ({ ...v, selected: true }))(state),
+ mapTreeValues(v => ({ ...v, selected: true }))({...state}),
UNSELECT_ALL_COLLECTION_FILES: () =>
- mapTreeValues(v => ({ ...v, selected: false }))(state),
+ mapTreeValues(v => ({ ...v, selected: false }))({...state}),
default: () => state
}) as CollectionPanelFilesState;
diff --git a/src/views-components/projects-tree-picker/generic-projects-tree-picker.tsx b/src/views-components/projects-tree-picker/generic-projects-tree-picker.tsx
index 8e27d445..07b1ad81 100644
--- a/src/views-components/projects-tree-picker/generic-projects-tree-picker.tsx
+++ b/src/views-components/projects-tree-picker/generic-projects-tree-picker.tsx
@@ -33,7 +33,7 @@ export interface ProjectsTreePickerDataProps {
showSelection?: boolean;
relatedTreePickers?: string[];
disableActivation?: string[];
- loadRootItem: (item: TreeItem<ProjectsTreePickerRootItem>, pickerId: string, includeCollections?: boolean, inlcudeFiles?: boolean) => void;
+ loadRootItem: (item: TreeItem<ProjectsTreePickerRootItem>, pickerId: string, includeCollections?: boolean, includeFiles?: boolean) => void;
}
export type ProjectsTreePickerProps = ProjectsTreePickerDataProps & Partial<PickedTreePickerProps>;
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list