[ARVADOS-WORKBENCH2] updated: 1.1.4-523-ged9321c
Git user
git at public.curoverse.com
Mon Aug 6 06:53:17 EDT 2018
Summary of changes:
src/models/collection-file.ts | 8 ++--
.../collection-manifest-mapper.test.ts | 50 ++++++++++++++++++----
.../collection-manifest-mapper.ts | 33 +++++++++++---
.../collection-manifest-parser.test.ts | 14 ++++--
.../collection-manifest-parser.ts | 13 +++++-
.../collection-panel-files-reducer.test.ts | 22 +++++-----
6 files changed, 104 insertions(+), 36 deletions(-)
via ed9321c52cef8fbb9f29d4cf1c2145fbfa90f297 (commit)
from 0a6a8ab679e84f99eedc17aec46a310ef86172c6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ed9321c52cef8fbb9f29d4cf1c2145fbfa90f297
Author: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
Date: Mon Aug 6 12:52:59 2018 +0200
Add functions for stringifying collection files and keep manifest
Feature #13967
Arvados-DCO-1.1-Signed-off-by: Michal Klobukowski <michal.klobukowski at contractors.roche.com>
diff --git a/src/models/collection-file.ts b/src/models/collection-file.ts
index a140080..7bf6df7 100644
--- a/src/models/collection-file.ts
+++ b/src/models/collection-file.ts
@@ -12,14 +12,14 @@ export enum CollectionFileType {
}
export interface CollectionDirectory {
- parentId: string;
+ path: string;
id: string;
name: string;
type: CollectionFileType.DIRECTORY;
}
export interface CollectionFile {
- parentId: string;
+ path: string;
id: string;
name: string;
size: number;
@@ -29,7 +29,7 @@ export interface CollectionFile {
export const createCollectionDirectory = (data: Partial<CollectionDirectory>): CollectionDirectory => ({
id: '',
name: '',
- parentId: '',
+ path: '',
type: CollectionFileType.DIRECTORY,
...data
});
@@ -37,7 +37,7 @@ export const createCollectionDirectory = (data: Partial<CollectionDirectory>): C
export const createCollectionFile = (data: Partial<CollectionFile>): CollectionFile => ({
id: '',
name: '',
- parentId: '',
+ path: '',
size: 0,
type: CollectionFileType.FILE,
...data
diff --git a/src/services/collection-files-service/collection-manifest-mapper.test.ts b/src/services/collection-files-service/collection-manifest-mapper.test.ts
index ad8f872..e12dc35 100644
--- a/src/services/collection-files-service/collection-manifest-mapper.test.ts
+++ b/src/services/collection-files-service/collection-manifest-mapper.test.ts
@@ -3,32 +3,34 @@
// SPDX-License-Identifier: AGPL-3.0
import { parseKeepManifestText } from "./collection-manifest-parser";
-import { mapManifestToFiles, mapManifestToDirectories } from "./collection-manifest-mapper";
+import { mapManifestToFiles, mapManifestToDirectories, mapManifestToCollectionFilesTree, mapCollectionFilesTreeToManifest } from "./collection-manifest-mapper";
+import { createTree, setNode, TreeNode } from "../../models/tree";
+import { CollectionFilesTree, CollectionDirectory, CollectionFile, CollectionFileType } from "../../models/collection-file";
test('mapManifestToFiles', () => {
const manifestText = `. 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n./c d41d8cd98f00b204e9800998ecf8427e+0 0:0:d`;
const manifest = parseKeepManifestText(manifestText);
const files = mapManifestToFiles(manifest);
expect(files).toEqual([{
- parentId: '',
+ path: '',
id: '/a',
name: 'a',
size: 0,
type: 'file'
}, {
- parentId: '',
+ path: '',
id: '/b',
name: 'b',
size: 0,
type: 'file'
}, {
- parentId: '',
+ path: '',
id: '/output.txt',
name: 'output.txt',
size: 33,
type: 'file'
}, {
- parentId: '/c',
+ path: '/c',
id: '/c/d',
name: 'd',
size: 0,
@@ -41,19 +43,51 @@ test('mapManifestToDirectories', () => {
const manifest = parseKeepManifestText(manifestText);
const directories = mapManifestToDirectories(manifest);
expect(directories).toEqual([{
- parentId: "",
+ path: "",
id: '/c',
name: 'c',
type: 'directory'
}, {
- parentId: '/c',
+ path: '/c',
id: '/c/user',
name: 'user',
type: 'directory'
}, {
- parentId: '/c/user',
+ path: '/c/user',
id: '/c/user/results',
name: 'results',
type: 'directory'
},]);
+});
+
+test('mapCollectionFilesTreeToManifest', () => {
+ const manifestText = `. 930625b054ce894ac40596c3f5a0d947+33 0:22:test.txt\n./c/user/results 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n`;
+ const tree = mapManifestToCollectionFilesTree(parseKeepManifestText(manifestText));
+ const manifest = mapCollectionFilesTreeToManifest(tree);
+ expect(manifest).toEqual([{
+ name: '',
+ locators: [],
+ files: [{
+ name: 'test.txt',
+ position: '',
+ size: 22
+ },],
+ }, {
+ name: '/c/user/results',
+ locators: [],
+ files: [{
+ name: 'a',
+ position: '',
+ size: 0
+ }, {
+ name: 'b',
+ position: '',
+ size: 0
+ }, {
+ name: 'output.txt',
+ position: '',
+ size: 33
+ },],
+ },]);
+
});
\ No newline at end of file
diff --git a/src/services/collection-files-service/collection-manifest-mapper.ts b/src/services/collection-files-service/collection-manifest-mapper.ts
index c2a8ae8..327ee99 100644
--- a/src/services/collection-files-service/collection-manifest-mapper.ts
+++ b/src/services/collection-files-service/collection-manifest-mapper.ts
@@ -2,10 +2,23 @@
//
// SPDX-License-Identifier: AGPL-3.0
-import { uniqBy } from 'lodash';
+import { uniqBy, flow, groupBy } from 'lodash';
import { KeepManifestStream, KeepManifestStreamFile, KeepManifest } from "../../models/keep-manifest";
-import { TreeNode, setNode, createTree } from '../../models/tree';
-import { CollectionFilesTree, CollectionFile, CollectionDirectory, createCollectionDirectory, createCollectionFile } from '../../models/collection-file';
+import { TreeNode, setNode, createTree, getNodeDescendants, getNodeValue, getNode } from '../../models/tree';
+import { CollectionFilesTree, CollectionFile, CollectionDirectory, createCollectionDirectory, createCollectionFile, CollectionFileType } from '../../models/collection-file';
+
+export const mapCollectionFilesTreeToManifest = (tree: CollectionFilesTree): KeepManifest => {
+ const values = getNodeDescendants('')(tree).map(id => getNodeValue(id)(tree));
+ const files = values.filter(value => value && value.type === CollectionFileType.FILE) as CollectionFile[];
+ const fileGroups = groupBy(files, file => file.path);
+ return Object
+ .keys(fileGroups)
+ .map(dirName => ({
+ name: dirName,
+ locators: [],
+ files: fileGroups[dirName].map(mapCollectionFile)
+ }));
+};
export const mapManifestToCollectionFilesTree = (manifest: KeepManifest): CollectionFilesTree =>
manifestToCollectionFiles(manifest)
@@ -16,7 +29,7 @@ export const mapManifestToCollectionFilesTree = (manifest: KeepManifest): Collec
export const mapCollectionFileToTreeNode = (file: CollectionFile): TreeNode<CollectionFile> => ({
children: [],
id: file.id,
- parent: file.parentId,
+ parent: file.path,
value: file
});
@@ -47,7 +60,7 @@ const splitDirectory = (directory: CollectionDirectory): CollectionDirectory[] =
const mapPathComponentToDirectory = (component: string, index: number, components: string[]): CollectionDirectory =>
createCollectionDirectory({
- parentId: index === 0 ? '' : joinPathComponents(components, index),
+ path: index === 0 ? '' : joinPathComponents(components, index),
id: joinPathComponents(components, index + 1),
name: component,
});
@@ -55,9 +68,15 @@ const mapPathComponentToDirectory = (component: string, index: number, component
const joinPathComponents = (components: string[], index: number) =>
`/${components.slice(0, index).join('/')}`;
+const mapCollectionFile = (file: CollectionFile): KeepManifestStreamFile => ({
+ name: file.name,
+ position: '',
+ size: file.size
+});
+
const mapStreamDirectory = (stream: KeepManifestStream): CollectionDirectory =>
createCollectionDirectory({
- parentId: '',
+ path: '',
id: stream.name,
name: stream.name,
});
@@ -65,7 +84,7 @@ const mapStreamDirectory = (stream: KeepManifestStream): CollectionDirectory =>
const mapStreamFile = (stream: KeepManifestStream) =>
(file: KeepManifestStreamFile): CollectionFile =>
createCollectionFile({
- parentId: stream.name,
+ path: stream.name,
id: `${stream.name}/${file.name}`,
name: file.name,
size: file.size,
diff --git a/src/services/collection-files-service/collection-manifest-parser.test.ts b/src/services/collection-files-service/collection-manifest-parser.test.ts
index eddc9c6..09525d8 100644
--- a/src/services/collection-files-service/collection-manifest-parser.test.ts
+++ b/src/services/collection-files-service/collection-manifest-parser.test.ts
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: AGPL-3.0
-import { parseKeepManifestText, parseKeepManifestStream } from "./collection-manifest-parser";
+import { parseKeepManifestText, parseKeepManifestStream, stringifyKeepManifest } from "./collection-manifest-parser";
describe('parseKeepManifestText', () => {
it('should parse text into streams', () => {
@@ -26,9 +26,15 @@ describe('parseKeepManifestStream', () => {
});
it('should parse stream files', () => {
expect(stream.files).toEqual([
- {name: 'a', position: '0', size: 0},
- {name: 'b', position: '0', size: 0},
- {name: 'output.txt', position: '0', size: 33},
+ { name: 'a', position: '0', size: 0 },
+ { name: 'b', position: '0', size: 0 },
+ { name: 'output.txt', position: '0', size: 33 },
]);
});
+});
+
+test('stringifyKeepManifest', () => {
+ const manifestText = `. 930625b054ce894ac40596c3f5a0d947+33 0:22:test.txt\n./c/user/results 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n`;
+ const manifest = parseKeepManifestText(manifestText);
+ expect(stringifyKeepManifest(manifest)).toEqual(`. 930625b054ce894ac40596c3f5a0d947+33 0:22:test.txt\n./c/user/results 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt\n`);
});
\ No newline at end of file
diff --git a/src/services/collection-files-service/collection-manifest-parser.ts b/src/services/collection-files-service/collection-manifest-parser.ts
index df334d4..bed2678 100644
--- a/src/services/collection-files-service/collection-manifest-parser.ts
+++ b/src/services/collection-files-service/collection-manifest-parser.ts
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: AGPL-3.0
-import { KeepManifestStream, KeepManifestStreamFile } from "../../models/keep-manifest";
+import { KeepManifestStream, KeepManifestStreamFile, KeepManifest } from "../../models/keep-manifest";
/**
* Documentation [http://doc.arvados.org/api/storage.html](http://doc.arvados.org/api/storage.html)
@@ -25,6 +25,12 @@ export const parseKeepManifestStream = (stream: string): KeepManifestStream => {
};
};
+export const stringifyKeepManifest = (manifest: KeepManifest) =>
+ manifest.map(stringifyKeepManifestStream).join('');
+
+export const stringifyKeepManifestStream = (stream: KeepManifestStream) =>
+ `.${stream.name} ${stream.locators.join(' ')} ${stream.files.map(stringifyFile).join(' ')}\n`;
+
const FILE_LOCATOR_REGEXP = /^([0-9a-f]{32})\+([0-9]+)(\+[A-Z][-A-Za-z0-9 at _]*)*$/;
const FILE_REGEXP = /([0-9]+):([0-9]+):(.*)/;
@@ -43,4 +49,7 @@ const parseFile = (token: string): KeepManifestStreamFile => {
const match = FILE_REGEXP.exec(token);
const [position, size, name] = match!.slice(1);
return { name, position, size: parseInt(size, 10) };
-};
\ No newline at end of file
+};
+
+const stringifyFile = (file: KeepManifestStreamFile) =>
+ `${file.position}:${file.size}:${file.name}`;
\ No newline at end of file
diff --git a/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.test.ts b/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.test.ts
index 1a6bb7d..94b71ff 100644
--- a/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.test.ts
+++ b/src/store/collection-panel/collection-panel-files/collection-panel-files-reducer.test.ts
@@ -11,21 +11,21 @@ import { CollectionPanelFile, CollectionPanelDirectory } from "./collection-pane
describe('CollectionPanelFilesReducer', () => {
const files: Array<CollectionFile | CollectionDirectory> = [
- createCollectionDirectory({ id: 'Directory 1', name: 'Directory 1', parentId: '' }),
- createCollectionDirectory({ id: 'Directory 2', name: 'Directory 2', parentId: 'Directory 1' }),
- createCollectionDirectory({ id: 'Directory 3', name: 'Directory 3', parentId: '' }),
- createCollectionDirectory({ id: 'Directory 4', name: 'Directory 4', parentId: 'Directory 3' }),
- createCollectionFile({ id: 'file1.txt', name: 'file1.txt', parentId: 'Directory 2' }),
- createCollectionFile({ id: 'file2.txt', name: 'file2.txt', parentId: 'Directory 2' }),
- createCollectionFile({ id: 'file3.txt', name: 'file3.txt', parentId: 'Directory 3' }),
- createCollectionFile({ id: 'file4.txt', name: 'file4.txt', parentId: 'Directory 3' }),
- createCollectionFile({ id: 'file5.txt', name: 'file5.txt', parentId: 'Directory 4' }),
+ createCollectionDirectory({ id: 'Directory 1', name: 'Directory 1', path: '' }),
+ createCollectionDirectory({ id: 'Directory 2', name: 'Directory 2', path: 'Directory 1' }),
+ createCollectionDirectory({ id: 'Directory 3', name: 'Directory 3', path: '' }),
+ createCollectionDirectory({ id: 'Directory 4', name: 'Directory 4', path: 'Directory 3' }),
+ createCollectionFile({ id: 'file1.txt', name: 'file1.txt', path: 'Directory 2' }),
+ createCollectionFile({ id: 'file2.txt', name: 'file2.txt', path: 'Directory 2' }),
+ createCollectionFile({ id: 'file3.txt', name: 'file3.txt', path: 'Directory 3' }),
+ createCollectionFile({ id: 'file4.txt', name: 'file4.txt', path: 'Directory 3' }),
+ createCollectionFile({ id: 'file5.txt', name: 'file5.txt', path: 'Directory 4' }),
];
const collectionFilesTree = files.reduce((tree, file) => setNode({
children: [],
id: file.id,
- parent: file.parentId,
+ parent: file.path,
value: file
})(tree), createTree<CollectionFile | CollectionDirectory>());
@@ -35,7 +35,7 @@ describe('CollectionPanelFilesReducer', () => {
it('SET_COLLECTION_FILES', () => {
expect(getNodeValue('Directory 1')(collectionPanelFilesTree)).toEqual({
- ...createCollectionDirectory({ id: 'Directory 1', name: 'Directory 1', parentId: '' }),
+ ...createCollectionDirectory({ id: 'Directory 1', name: 'Directory 1', path: '' }),
collapsed: true,
selected: false
});
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list