[arvados-workbench2] updated: 2.6.3-42-gabc9ef71
git repository hosting
git at public.arvados.org
Wed Aug 2 22:02:28 UTC 2023
Summary of changes:
cypress/integration/process.spec.js | 28 +++++---
src/services/log-service/log-service.ts | 39 ++++++-----
.../process-logs-panel-actions.ts | 75 +++++++++++++---------
src/store/processes/process.ts | 4 ++
src/views/process-panel/process-log-card.tsx | 4 +-
tsconfig.json | 1 +
6 files changed, 89 insertions(+), 62 deletions(-)
via abc9ef71a341f72c061c6fc8b0276fcf9937c411 (commit)
via d53a08ea0cc9a9af1a3071969d41165e8a6ff5d5 (commit)
via 05acb61d6369592b987fa827cac3e5774d1a7b1e (commit)
via 75eb080fec293c283ce934c5c82cfeee85bee4c1 (commit)
via 33b725526a0a16c6006d1ceaa89039d263024bc1 (commit)
via 41779cd087575a8216450e8dd27b20ef0fb13dd5 (commit)
from 5430c336b96cbb7c20bffa1cbdb8cffea32fb460 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit abc9ef71a341f72c061c6fc8b0276fcf9937c411
Author: Stephen Smith <stephen at curii.com>
Date: Wed Aug 2 17:48:36 2023 -0400
20219: Pass through promise rejections from getLogFileContents
* No longer returns undefined for failed log fragments
* initProcessLogsPanel added catch to handle errors, on failure it shows a
toast and initializes the store empty, allowing polling to run and retry if
the container is running
* Add note to pollProcessLogs error console.log that the promise rejection is
ignored currently
* loadContainerLogFileContents now handles errors from getLogFileContents
* Switched from Promise.all to Promise.allSettled, which allows some promises
to fail and still return a result
* Replace undefined fragment & subfragment filtering with top level promise
status filtering, subfragments can no longer have some failures due to
sub-promises using .all which rejects on any failure
* Add es2020 to typescript lib array for Promise.allSettled
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/src/services/log-service/log-service.ts b/src/services/log-service/log-service.ts
index 4ba02bef..03d3c01e 100644
--- a/src/services/log-service/log-service.ts
+++ b/src/services/log-service/log-service.ts
@@ -31,24 +31,20 @@ export class LogService extends CommonResourceService<LogResource> {
return Promise.reject();
}
- async getLogFileContents(containerRequestUuid: string, fileRecord: CollectionFile, startByte: number, endByte: number): Promise<LogFragment | undefined> {
- try {
- const request = await this.apiWebdavClient.get(
- `container_requests/${containerRequestUuid}/log/${fileRecord.name}`,
- {headers: {Range: `bytes=${startByte}-${endByte}`}}
- );
- const logFileType = logFileToLogType(fileRecord);
+ async getLogFileContents(containerRequestUuid: string, fileRecord: CollectionFile, startByte: number, endByte: number): Promise<LogFragment> {
+ const request = await this.apiWebdavClient.get(
+ `container_requests/${containerRequestUuid}/log/${fileRecord.name}`,
+ {headers: {Range: `bytes=${startByte}-${endByte}`}}
+ );
+ const logFileType = logFileToLogType(fileRecord);
- if (request.responseText && logFileType) {
- return {
- logType: logFileType,
- contents: request.responseText.split(/\r?\n/),
- };
- } else {
- return undefined;
- }
- } catch(e) {
- return undefined;
+ if (request.responseText && logFileType) {
+ return {
+ logType: logFileType,
+ contents: request.responseText.split(/\r?\n/),
+ };
+ } else {
+ return Promise.reject();
}
}
}
diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts
index af30e270..9da7d1b9 100644
--- a/src/store/process-logs-panel/process-logs-panel-actions.ts
+++ b/src/store/process-logs-panel/process-logs-panel-actions.ts
@@ -39,21 +39,28 @@ export const setProcessLogsPanelFilter = (filter: string) =>
export const initProcessLogsPanel = (processUuid: string) =>
async (dispatch: Dispatch, getState: () => RootState, { logService }: ServiceRepository) => {
- dispatch(processLogsPanelActions.RESET_PROCESS_LOGS_PANEL());
- const process = getProcess(processUuid)(getState().resources);
- if (process?.containerRequest?.uuid) {
- // Get log file size info
- const logFiles = await loadContainerLogFileList(process.containerRequest.uuid, logService);
+ try {
+ dispatch(processLogsPanelActions.RESET_PROCESS_LOGS_PANEL());
+ const process = getProcess(processUuid)(getState().resources);
+ if (process?.containerRequest?.uuid) {
+ // Get log file size info
+ const logFiles = await loadContainerLogFileList(process.containerRequest.uuid, logService);
- // Populate lastbyte 0 for each file
- const filesWithProgress = logFiles.map((file) => ({file, lastByte: 0}));
+ // Populate lastbyte 0 for each file
+ const filesWithProgress = logFiles.map((file) => ({file, lastByte: 0}));
- // Fetch array of LogFragments
- const logLines = await loadContainerLogFileContents(filesWithProgress, logService, process);
+ // Fetch array of LogFragments
+ const logLines = await loadContainerLogFileContents(filesWithProgress, logService, process);
- // Populate initial state with filters
- const initialState = createInitialLogPanelState(logFiles, logLines);
+ // Populate initial state with filters
+ const initialState = createInitialLogPanelState(logFiles, logLines);
+ dispatch(processLogsPanelActions.INIT_PROCESS_LOGS_PANEL(initialState));
+ }
+ } catch(e) {
+ // On error, populate empty state to allow polling to start
+ const initialState = createInitialLogPanelState([], []);
dispatch(processLogsPanelActions.INIT_PROCESS_LOGS_PANEL(initialState));
+ dispatch(snackbarActions.OPEN_SNACKBAR({ message: 'Could not load process logs', hideDuration: 2000, kind: SnackbarKind.ERROR }));
}
};
@@ -94,6 +101,7 @@ export const pollProcessLogs = (processUuid: string) =>
}
return Promise.resolve();
} catch (e) {
+ // Remove log when polling error is handled in some way instead of being ignored
console.log("Polling process logs failed");
return Promise.reject();
}
@@ -120,7 +128,7 @@ const loadContainerLogFileList = async (containerUuid: string, logService: LogSe
* @returns LogFragment[] containing a single LogFragment corresponding to each input file
*/
const loadContainerLogFileContents = async (logFilesWithProgress: FileWithProgress[], logService: LogService, process: Process) => (
- (await Promise.all(logFilesWithProgress.filter(({file}) => file.size > 0).map(({file, lastByte}) => {
+ (await Promise.allSettled(logFilesWithProgress.filter(({file}) => file.size > 0).map(({file, lastByte}) => {
const requestSize = file.size - lastByte;
if (requestSize > maxLogFetchSize) {
const chunkSize = Math.floor(maxLogFetchSize / 2);
@@ -128,31 +136,38 @@ const loadContainerLogFileContents = async (logFilesWithProgress: FileWithProgre
return Promise.all([
logService.getLogFileContents(process.containerRequest.uuid, file, lastByte, firstChunkEnd),
logService.getLogFileContents(process.containerRequest.uuid, file, file.size-chunkSize, file.size-1)
- ] as Promise<(LogFragment | undefined)>[]);
+ ] as Promise<(LogFragment)>[]);
} else {
return Promise.all([logService.getLogFileContents(process.containerRequest.uuid, file, lastByte, file.size-1)]);
}
- }))).filter((logResponseSet) => ( // Top level filter ensures array of LogFragments is not empty and contains 1 or more fragments containing log lines
- logResponseSet.length && logResponseSet.some(logFragment => logFragment && logFragment.contents.length)
- )).map((logResponseSet)=> {
- // Remove fragments from subarrays that are undefined or have no lines
- const responseSet = logResponseSet.filter((logFragment): logFragment is LogFragment => (
- !!logFragment && logFragment.contents.length > 0
- ));
-
+ })).then((res) => {
+ if (res.length && res.every(promise => (promise.status === 'rejected'))) {
+ // Since allSettled does not pass promise rejection we throw an
+ // error if every request failed
+ return Promise.reject("Failed to load logs");
+ }
+ return res.filter((one): one is PromiseFulfilledResult<LogFragment[]> => (
+ // Filter out log files with rejected promises
+ // (Promise.all rejects on any failure)
+ one.status === 'fulfilled' &&
+ // Filter out files where any fragment is empty
+ // (prevent incorrect snipline generation or an un-resumable situation)
+ !!one.value.every(logFragment => logFragment.contents.length)
+ )).map(one => one.value)
+ })).map((logResponseSet)=> {
// For any multi fragment response set, modify the last line of non-final chunks to include a line break and snip line
// Don't add snip line as a separate line so that sorting won't reorder it
- for (let i = 1; i < responseSet.length; i++) {
- const fragment = responseSet[i-1];
+ for (let i = 1; i < logResponseSet.length; i++) {
+ const fragment = logResponseSet[i-1];
const lastLineIndex = fragment.contents.length-1;
const lastLineContents = fragment.contents[lastLineIndex];
const newLastLine = `${lastLineContents}\n${SNIPLINE}`;
- responseSet[i-1].contents[lastLineIndex] = newLastLine;
+ logResponseSet[i-1].contents[lastLineIndex] = newLastLine;
}
// Merge LogFragment Array (representing multiple log line arrays) into single LogLine[] / LogFragment
- return responseSet.reduce((acc, curr: LogFragment) => ({
+ return logResponseSet.reduce((acc, curr: LogFragment) => ({
logType: curr.logType,
contents: [...(acc.contents || []), ...curr.contents]
}), {} as LogFragment);
diff --git a/tsconfig.json b/tsconfig.json
index 7bce4022..08f7108e 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -5,6 +5,7 @@
"target": "es5",
"lib": [
"es6",
+ "es2020",
"dom"
],
"sourceMap": true,
commit d53a08ea0cc9a9af1a3071969d41165e8a6ff5d5
Author: Stephen Smith <stephen at curii.com>
Date: Tue Aug 1 21:47:53 2023 -0400
20219: Start / stop log polling when container is running
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/cypress/integration/process.spec.js b/cypress/integration/process.spec.js
index 00a6631b..b41a443e 100644
--- a/cypress/integration/process.spec.js
+++ b/cypress/integration/process.spec.js
@@ -2,6 +2,8 @@
//
// SPDX-License-Identifier: AGPL-3.0
+import { ContainerState } from 'models/container';
+
describe('Process tests', function() {
let activeUser;
let adminUser;
@@ -415,6 +417,12 @@ describe('Process tests', function() {
describe('Logs panel', function() {
it('shows live process logs', function() {
+ cy.intercept({method: 'GET', url: '**/arvados/v1/containers/*'}, (req) => {
+ req.reply((res) => {
+ res.body.state = ContainerState.RUNNING;
+ });
+ });
+
const crName = 'test_container_request';
createContainerRequest(
activeUser,
diff --git a/src/store/processes/process.ts b/src/store/processes/process.ts
index e6219498..526629cd 100644
--- a/src/store/processes/process.ts
+++ b/src/store/processes/process.ts
@@ -185,6 +185,10 @@ export const getProcessStatus = ({ containerRequest, container }: Process): Proc
}
};
+export const isProcessRunning = ({ container }: Process): boolean => (
+ container?.state === ContainerState.RUNNING
+);
+
export const isProcessRunnable = ({ containerRequest }: Process): boolean => (
containerRequest.state === ContainerRequestState.UNCOMMITTED
);
diff --git a/src/views/process-panel/process-log-card.tsx b/src/views/process-panel/process-log-card.tsx
index bcd4b240..4fd8f234 100644
--- a/src/views/process-panel/process-log-card.tsx
+++ b/src/views/process-panel/process-log-card.tsx
@@ -29,7 +29,7 @@ import {
WordWrapOffIcon,
WordWrapOnIcon,
} from 'components/icon/icon';
-import { Process } from 'store/processes/process';
+import { Process, isProcessRunning } from 'store/processes/process';
import { MPVPanelProps } from 'components/multi-panel-view/multi-panel-view';
import {
FilterOption,
@@ -105,7 +105,7 @@ export const ProcessLogsCard = withStyles(styles)(
useAsyncInterval(() => (
pollProcessLogs(process.containerRequest.uuid)
- ), 2000);
+ ), isProcessRunning(process) ? 2000 : null);
return <Grid item className={classes.root} xs={12}>
<Card className={classes.card}>
commit 05acb61d6369592b987fa827cac3e5774d1a7b1e
Author: Stephen Smith <stephen at curii.com>
Date: Tue Aug 1 21:30:28 2023 -0400
20219: Add console log to process log polling failure since promise rejection
is currently unhandled
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts
index 851d13ef..af30e270 100644
--- a/src/store/process-logs-panel/process-logs-panel-actions.ts
+++ b/src/store/process-logs-panel/process-logs-panel-actions.ts
@@ -94,6 +94,7 @@ export const pollProcessLogs = (processUuid: string) =>
}
return Promise.resolve();
} catch (e) {
+ console.log("Polling process logs failed");
return Promise.reject();
}
};
commit 75eb080fec293c283ce934c5c82cfeee85bee4c1
Author: Stephen Smith <stephen at curii.com>
Date: Tue Aug 1 21:26:37 2023 -0400
20219: Rearrange non-sortable logs to top of merged all logs view
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/cypress/integration/process.spec.js b/cypress/integration/process.spec.js
index 2be9c5c3..00a6631b 100644
--- a/cypress/integration/process.spec.js
+++ b/cypress/integration/process.spec.js
@@ -576,24 +576,24 @@ describe('Process tests', function() {
// Switch to All logs
cy.get('[data-cy=process-logs-filter]').click();
cy.get('body').contains('li', 'All logs').click();
- // Verify sorted logs
+ // Verify non-sorted lines were preserved
cy.get('[data-cy=process-logs] pre')
- .eq(0).should('contain', '2023-07-18T20:14:48.128642814Z first');
+ .eq(0).should('contain', '3: nodeinfo 1');
cy.get('[data-cy=process-logs] pre')
- .eq(1).should('contain', '2023-07-18T20:14:48.528642814Z second');
+ .eq(1).should('contain', '2: nodeinfo 2');
cy.get('[data-cy=process-logs] pre')
- .eq(2).should('contain', '2023-07-18T20:14:49.128642814Z third');
- // Verify non-sorted lines were preserved
+ .eq(2).should('contain', '1: nodeinfo 3');
cy.get('[data-cy=process-logs] pre')
- .eq(3).should('contain', '3: nodeinfo 1');
+ .eq(3).should('contain', '2: nodeinfo 4');
cy.get('[data-cy=process-logs] pre')
- .eq(4).should('contain', '2: nodeinfo 2');
+ .eq(4).should('contain', '3: nodeinfo 5');
+ // Verify sorted logs
cy.get('[data-cy=process-logs] pre')
- .eq(5).should('contain', '1: nodeinfo 3');
+ .eq(5).should('contain', '2023-07-18T20:14:48.128642814Z first');
cy.get('[data-cy=process-logs] pre')
- .eq(6).should('contain', '2: nodeinfo 4');
+ .eq(6).should('contain', '2023-07-18T20:14:48.528642814Z second');
cy.get('[data-cy=process-logs] pre')
- .eq(7).should('contain', '3: nodeinfo 5');
+ .eq(7).should('contain', '2023-07-18T20:14:49.128642814Z third');
});
});
});
diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts
index 23aeb96a..851d13ef 100644
--- a/src/store/process-logs-panel/process-logs-panel-actions.ts
+++ b/src/store/process-logs-panel/process-logs-panel-actions.ts
@@ -237,7 +237,7 @@ const mergeSortLogFragments = (logFragments: LogFragment[]): string[] => {
.filter((fragment) => (NON_SORTED_LOG_TYPES.includes(fragment.logType)))
.sort((a, b) => (a.logType.localeCompare(b.logType))));
- return [...sortableLines.sort(sortLogLines), ...nonSortableLines]
+ return [...nonSortableLines, ...sortableLines.sort(sortLogLines)]
};
const sortLogLines = (a: string, b: string) => {
commit 33b725526a0a16c6006d1ceaa89039d263024bc1
Author: Stephen Smith <stephen at curii.com>
Date: Tue Aug 1 21:20:49 2023 -0400
20219: Perform log file root path filtering in listLogFiles
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/src/services/log-service/log-service.ts b/src/services/log-service/log-service.ts
index f2424715..4ba02bef 100644
--- a/src/services/log-service/log-service.ts
+++ b/src/services/log-service/log-service.ts
@@ -23,7 +23,10 @@ export class LogService extends CommonResourceService<LogResource> {
async listLogFiles(containerRequestUuid: string) {
const request = await this.apiWebdavClient.propfind(`container_requests/${containerRequestUuid}/log`);
if (request.responseXML != null) {
- return extractFilesData(request.responseXML);
+ return extractFilesData(request.responseXML)
+ .filter((file) => (
+ file.path === `/arvados/v1/container_requests/${containerRequestUuid}/log`
+ ));
}
return Promise.reject();
}
diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts
index ea082cd9..23aeb96a 100644
--- a/src/store/process-logs-panel/process-logs-panel-actions.ts
+++ b/src/store/process-logs-panel/process-logs-panel-actions.ts
@@ -104,7 +104,6 @@ const loadContainerLogFileList = async (containerUuid: string, logService: LogSe
// Filter only root directory files matching log event types which have bytes
return logCollectionContents.filter((file): file is CollectionFile => (
file.type === CollectionFileType.FILE &&
- file.path === `/arvados/v1/container_requests/${containerUuid}/log` &&
PROCESS_PANEL_LOG_EVENT_TYPES.indexOf(logFileToLogType(file)) > -1 &&
file.size > 0
));
commit 41779cd087575a8216450e8dd27b20ef0fb13dd5
Author: Stephen Smith <stephen at curii.com>
Date: Tue Aug 1 21:19:09 2023 -0400
20219: Adjust logFileToLogType argument type to only CollectionFile and adjust
loadContainerLogFileList filter type hint to satisfy type
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/src/services/log-service/log-service.ts b/src/services/log-service/log-service.ts
index b96d8223..f2424715 100644
--- a/src/services/log-service/log-service.ts
+++ b/src/services/log-service/log-service.ts
@@ -8,7 +8,7 @@ import { CommonResourceService } from "services/common-service/common-resource-s
import { ApiActions } from "services/api/api-actions";
import { WebDAV } from "common/webdav";
import { extractFilesData } from "services/collection-service/collection-service-files-response";
-import { CollectionDirectory, CollectionFile } from "models/collection-file";
+import { CollectionFile } from "models/collection-file";
export type LogFragment = {
logType: LogEventType;
@@ -50,4 +50,4 @@ export class LogService extends CommonResourceService<LogResource> {
}
}
-export const logFileToLogType = (file: CollectionFile | CollectionDirectory) => (file.name.replace(/\.(txt|json)$/, '') as LogEventType);
+export const logFileToLogType = (file: CollectionFile) => (file.name.replace(/\.(txt|json)$/, '') as LogEventType);
diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts
index 328ca4da..ea082cd9 100644
--- a/src/store/process-logs-panel/process-logs-panel-actions.ts
+++ b/src/store/process-logs-panel/process-logs-panel-actions.ts
@@ -102,12 +102,12 @@ const loadContainerLogFileList = async (containerUuid: string, logService: LogSe
const logCollectionContents = await logService.listLogFiles(containerUuid);
// Filter only root directory files matching log event types which have bytes
- return logCollectionContents.filter((file) => (
+ return logCollectionContents.filter((file): file is CollectionFile => (
+ file.type === CollectionFileType.FILE &&
file.path === `/arvados/v1/container_requests/${containerUuid}/log` &&
PROCESS_PANEL_LOG_EVENT_TYPES.indexOf(logFileToLogType(file)) > -1 &&
- file.type === CollectionFileType.FILE &&
file.size > 0
- )) as CollectionFile[];
+ ));
};
/**
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list