[arvados] created: 2.7.0-5597-g095e176632
git repository hosting
git at public.arvados.org
Mon Dec 11 17:25:31 UTC 2023
at 095e176632bbf81d28a239742a1ecce12404bd2d (commit)
commit 095e176632bbf81d28a239742a1ecce12404bd2d
Merge: 19c37b6272 b774da617d
Author: Stephen Smith <stephen at curii.com>
Date: Mon Dec 11 12:24:21 2023 -0500
21204: Merge branch '21204-stable-log-sort' from arvados-workbench2.git
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --cc services/workbench2/Makefile
index 7e70f3ec9d,0000000000..0d402a63c1
mode 100644,000000..100644
--- a/services/workbench2/Makefile
+++ b/services/workbench2/Makefile
@@@ -1,187 -1,0 +1,190 @@@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Use bash, and run all lines in each recipe as one shell command
+SHELL := /bin/bash
+.ONESHELL:
+
+GOPATH?=~/go
+APP_NAME?=arvados-workbench2
+
+# VERSION uses all the above to produce X.Y.Z.timestamp
+# something in the lines of 1.2.0.20180612145021, this will be the package version
+# it can be overwritten when invoking make as in make packages VERSION=1.2.0
+VERSION?=$(shell ./version-at-commit.sh HEAD)
+# We don't use BUILD_NUMBER at the moment, but it needs to be defined
+BUILD_NUMBER?=0
+GIT_COMMIT?=$(shell git rev-parse --short HEAD)
+
+# ITERATION is the package iteration, intended for manual change if anything non-code related
+# changes in the package. (i.e. example config files externally added
+ITERATION?=1
+
+TARGETS?=centos7 rocky8 debian10 debian11 ubuntu1804 ubuntu2004
+
+DESCRIPTION=Arvados Workbench2 - Arvados is a free and open source platform for big data science.
+MAINTAINER=Arvados Package Maintainers <packaging at arvados.org>
+
+# DEST_DIR will have the build package copied.
+DEST_DIR=/var/www/$(APP_NAME)/workbench2/
+
+# Debian package file
+DEB_FILE=$(APP_NAME)_$(VERSION)-$(ITERATION)_amd64.deb
+
+# redHat package file
+RPM_FILE=$(APP_NAME)-$(VERSION)-$(ITERATION).x86_64.rpm
+
+GOPATH=$(shell go env GOPATH)
+export WORKSPACE?=$(shell pwd)
+
+ARVADOS_DIRECTORY?=$(shell env -C $(WORKSPACE) git rev-parse --show-toplevel)
+
+.PHONY: help clean* yarn-install test build packages packages-with-version integration-tests-in-docker
+
+help:
+ @echo >&2
+ @echo >&2 "There is no default make target here. Did you mean 'make test'?"
+ @echo >&2
+ @echo >&2 "More info:"
+ @echo >&2 " Installing --> http://doc.arvados.org/install"
+ @echo >&2 " Developing/contributing --> https://dev.arvados.org"
+ @echo >&2 " Project home --> https://arvados.org"
+ @echo >&2
+ @false
+
+clean-deb:
+ rm -f $(WORKSPACE)/*.deb
+
+clean-rpm:
+ rm -f $(WORKSPACE)/*.rpm
+
+clean-node-modules:
+ rm -rf $(WORKSPACE)/node_modules
+
+clean: clean-rpm clean-deb clean-node-modules
+
+arvados-server-install: check-arvados-directory
+ cd $(ARVADOS_DIRECTORY)
+ go mod download
+ cd cmd/arvados-server
+ echo GOPATH is $(GOPATH)
+ GOFLAGS=-buildvcs=false go install
+ cd -
+ ls -l $(GOPATH)/bin/arvados-server
+ $(GOPATH)/bin/arvados-server install -type test
+
+yarn-install:
+ yarn install
+
+unit-tests: yarn-install
+ yarn test --no-watchAll --bail --ci
+
+integration-tests: yarn-install check-arvados-directory
+ yarn run cypress install
+ $(WORKSPACE)/tools/run-integration-tests.sh -a $(ARVADOS_DIRECTORY)
+
+integration-tests-in-docker: workbench2-build-image check-arvados-directory
+ docker run -ti --rm \
+ --env ARVADOS_DIRECTORY=/usr/src/arvados \
+ --env GIT_DISCOVERY_ACROSS_FILESYSTEM=1 \
+ -v $(WORKSPACE):/usr/src/arvados/services/workbench2 \
+ -v $(ARVADOS_DIRECTORY):/usr/src/arvados \
+ -w /usr/src/arvados/services/workbench2 \
+ workbench2-build \
+ make arvados-server-install integration-tests
+
+unit-tests-in-docker: workbench2-build-image check-arvados-directory
+ docker run -ti --rm \
+ --env ARVADOS_DIRECTORY=/usr/src/arvados \
+ --env GIT_DISCOVERY_ACROSS_FILESYSTEM=1 \
+ -v $(WORKSPACE):/usr/src/arvados/services/workbench2 \
+ -v $(ARVADOS_DIRECTORY):/usr/src/arvados \
+ -w /usr/src/arvados/services/workbench2 \
+ workbench2-build \
+ make arvados-server-install unit-tests
+
+tests-in-docker: workbench2-build-image check-arvados-directory
+ docker run -ti --rm \
+ --env ARVADOS_DIRECTORY=/usr/src/arvados \
+ --env GIT_DISCOVERY_ACROSS_FILESYSTEM=1 \
+ --env ci="${ci}" \
+ -v $(WORKSPACE):/usr/src/arvados/services/workbench2 \
+ -v$(ARVADOS_DIRECTORY):/usr/src/arvados \
+ -w /usr/src/arvados/services/workbench2 \
+ workbench2-build \
+ make test
+
+test: unit-tests integration-tests
+
+build: yarn-install
+ VERSION=$(VERSION) BUILD_NUMBER=$(BUILD_NUMBER) GIT_COMMIT=$(GIT_COMMIT) yarn build
+
+$(DEB_FILE): build
+ fpm \
+ -s dir \
+ -t deb \
+ -n "$(APP_NAME)" \
+ -v "$(VERSION)" \
+ --iteration "$(ITERATION)" \
+ --vendor="The Arvados Authors" \
+ --maintainer="$(MAINTAINER)" \
+ --url="https://arvados.org" \
+ --license="GNU Affero General Public License, version 3.0" \
+ --description="$(DESCRIPTION)" \
+ --config-files="etc/arvados/$(APP_NAME)/workbench2.example.json" \
+ $(WORKSPACE)/build/=$(DEST_DIR) \
+ etc/arvados/workbench2/workbench2.example.json=/etc/arvados/$(APP_NAME)/workbench2.example.json
+
+$(RPM_FILE): build
+ fpm \
+ -s dir \
+ -t rpm \
+ -n "$(APP_NAME)" \
+ -v "$(VERSION)" \
+ --iteration "$(ITERATION)" \
+ --vendor="The Arvados Authors" \
+ --maintainer="$(MAINTAINER)" \
+ --url="https://arvados.org" \
+ --license="GNU Affero General Public License, version 3.0" \
+ --description="$(DESCRIPTION)" \
+ --config-files="etc/arvados/$(APP_NAME)/workbench2.example.json" \
+ $(WORKSPACE)/build/=$(DEST_DIR) \
+ etc/arvados/workbench2/workbench2.example.json=/etc/arvados/$(APP_NAME)/workbench2.example.json
+
+copy: $(DEB_FILE) $(RPM_FILE)
+ for target in $(TARGETS); do \
+ mkdir -p "packages/$$target" && \
+ case "$$target" in \
+ centos*|rocky*) cp -p "$(RPM_FILE)" "packages/$$target" ;; \
+ debian*|ubuntu*) cp -p "$(DEB_FILE)" "packages/$$target" ;; \
+ *) echo "Unknown copy target $$target"; exit 1 ;; \
+ esac ; \
+ done ; \
+ rm -f "$(DEB_FILE)" "$(RPM_FILE)"
+
+# use FPM to create DEB and RPM
+packages: copy
+
+check-arvados-directory:
+ @if test "${ARVADOS_DIRECTORY}" == "unset"; then echo "the environment variable ARVADOS_DIRECTORY must be set to the path of an arvados git checkout"; exit 1; fi
+ @if ! test -d "${ARVADOS_DIRECTORY}"; then echo "the environment variable ARVADOS_DIRECTORY does not point at a directory"; exit 1; fi
+
+packages-in-docker: check-arvados-directory workbench2-build-image
+ docker run -t --rm --env ci="true" \
+ --env ARVADOS_DIRECTORY=/tmp/arvados \
+ --env APP_NAME=${APP_NAME} \
++ --env VERSION="${VERSION}" \
+ --env ITERATION=${ITERATION} \
+ --env TARGETS="${TARGETS}" \
++ --env MAINTAINER="${MAINTAINER}" \
++ --env DESCRIPTION="${DESCRIPTION}" \
+ --env GIT_DISCOVERY_ACROSS_FILESYSTEM=1 \
+ -w "/tmp/workbench2" \
+ -v ${WORKSPACE}:/tmp/workbench2 \
+ -v ${ARVADOS_DIRECTORY}:/tmp/arvados \
+ workbench2-build:latest \
+ sh -c 'git config --global --add safe.directory /tmp/workbench2 && make packages'
+
+workbench2-build-image:
+ (cd docker && docker build -t workbench2-build .)
commit b774da617da87999f68863b90542dd41d9ba0bd3
Author: Stephen Smith <stephen at curii.com>
Date: Mon Dec 11 10:00:43 2023 -0500
21204: Add test for stable log line sort
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/cypress/integration/process.spec.js b/cypress/integration/process.spec.js
index 438acbf14d..38d9794b18 100644
--- a/cypress/integration/process.spec.js
+++ b/cypress/integration/process.spec.js
@@ -556,6 +556,53 @@ describe("Process tests", function () {
});
});
+ it("preserves original ordering of lines within the same log type", function () {
+ const crName = "test_container_request";
+ createContainerRequest(activeUser, crName, "arvados/jobs", ["echo", "hello world"], false, "Committed").then(function (containerRequest) {
+ cy.appendLog(adminUser.token, containerRequest.uuid, "stdout.txt", [
+ // Should come first
+ "2023-07-18T20:14:46.000000000Z A out 1",
+ // Comes fourth in a contiguous block
+ "2023-07-18T20:14:48.128642814Z A out 2",
+ "2023-07-18T20:14:48.128642814Z X out 3",
+ "2023-07-18T20:14:48.128642814Z A out 4",
+ ]).as("stdout");
+
+ cy.appendLog(adminUser.token, containerRequest.uuid, "stderr.txt", [
+ // Comes second
+ "2023-07-18T20:14:47.000000000Z Z err 1",
+ // Comes third in a contiguous block
+ "2023-07-18T20:14:48.128642814Z B err 2",
+ "2023-07-18T20:14:48.128642814Z C err 3",
+ "2023-07-18T20:14:48.128642814Z Y err 4",
+ "2023-07-18T20:14:48.128642814Z Z err 5",
+ "2023-07-18T20:14:48.128642814Z A err 6",
+ ]).as("stderr");
+
+ cy.loginAs(activeUser);
+ cy.goToPath(`/processes/${containerRequest.uuid}`);
+ cy.get("[data-cy=process-details]").should("contain", crName);
+ cy.get("[data-cy=process-logs]").should("contain", "No logs yet");
+
+ cy.getAll("@stdout", "@stderr").then(() => {
+ // Switch to All logs
+ cy.get("[data-cy=process-logs-filter]").click();
+ cy.get("body").contains("li", "All logs").click();
+ // Verify sorted logs
+ cy.get("[data-cy=process-logs] pre").eq(0).should("contain", "2023-07-18T20:14:46.000000000Z A out 1");
+ cy.get("[data-cy=process-logs] pre").eq(1).should("contain", "2023-07-18T20:14:47.000000000Z Z err 1");
+ cy.get("[data-cy=process-logs] pre").eq(2).should("contain", "2023-07-18T20:14:48.128642814Z B err 2");
+ cy.get("[data-cy=process-logs] pre").eq(3).should("contain", "2023-07-18T20:14:48.128642814Z C err 3");
+ cy.get("[data-cy=process-logs] pre").eq(4).should("contain", "2023-07-18T20:14:48.128642814Z Y err 4");
+ cy.get("[data-cy=process-logs] pre").eq(5).should("contain", "2023-07-18T20:14:48.128642814Z Z err 5");
+ cy.get("[data-cy=process-logs] pre").eq(6).should("contain", "2023-07-18T20:14:48.128642814Z A err 6");
+ cy.get("[data-cy=process-logs] pre").eq(7).should("contain", "2023-07-18T20:14:48.128642814Z A out 2");
+ cy.get("[data-cy=process-logs] pre").eq(8).should("contain", "2023-07-18T20:14:48.128642814Z X out 3");
+ cy.get("[data-cy=process-logs] pre").eq(9).should("contain", "2023-07-18T20:14:48.128642814Z A out 4");
+ });
+ });
+ });
+
it("correctly generates sniplines", function () {
const SNIPLINE = `================ ✀ ================ ✀ ========= Some log(s) were skipped ========= ✀ ================ ✀ ================`;
const crName = "test_container_request";
commit 66897bc5cef798a8bfc2a3426568914dcaf15759
Author: Stephen Smith <stephen at curii.com>
Date: Mon Dec 11 09:59:59 2023 -0500
21204: Change log sorter to preserve ordering of same-timestamped lines of a
single type
Arvados-DCO-1.1-Signed-off-by: Stephen Smith <stephen at curii.com>
diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts
index 4e52431eeb..88b56a2c32 100644
--- a/src/store/process-logs-panel/process-logs-panel-actions.ts
+++ b/src/store/process-logs-panel/process-logs-panel-actions.ts
@@ -33,6 +33,12 @@ type FileWithProgress = {
lastByte: number;
}
+type SortableLine = {
+ logType: LogEventType,
+ timestamp: string;
+ contents: string;
+}
+
export type ProcessLogsPanelAction = UnionOf<typeof processLogsPanelActions>;
export const setProcessLogsPanelFilter = (filter: string) =>
@@ -257,18 +263,61 @@ const mergeMultilineLoglines = (logFragments: LogFragment[]) => (
* @returns string[] of merged and sorted log lines
*/
const mergeSortLogFragments = (logFragments: LogFragment[]): string[] => {
- const sortableLines = fragmentsToLines(logFragments
- .filter((fragment) => (!NON_SORTED_LOG_TYPES.includes(fragment.logType))));
+ const sortableFragments = logFragments
+ .filter((fragment) => (!NON_SORTED_LOG_TYPES.includes(fragment.logType)));
const nonSortableLines = fragmentsToLines(logFragments
.filter((fragment) => (NON_SORTED_LOG_TYPES.includes(fragment.logType)))
.sort((a, b) => (a.logType.localeCompare(b.logType))));
- return [...nonSortableLines, ...sortableLines.sort(sortLogLines)]
+ return [...nonSortableLines, ...sortLogFragments(sortableFragments)];
};
-const sortLogLines = (a: string, b: string) => {
- return a.localeCompare(b);
+/**
+ * Performs merge and sort of input log fragment lines
+ * @param logFragments set of sortable log fragments to be merged and sorted
+ * @returns A string array containing all lines, sorted by timestamp and
+ * preserving line ordering and type grouping when timestamps match
+ */
+const sortLogFragments = (logFragments: LogFragment[]): string[] => {
+ const linesWithType: SortableLine[] = logFragments
+ // Map each logFragment into an array of SortableLine
+ .map((fragment: LogFragment): SortableLine[] => (
+ fragment.contents.map((singleLine: string) => {
+ const timestampMatch = singleLine.match(LOG_TIMESTAMP_PATTERN);
+ const timestamp = timestampMatch && timestampMatch[0] ? timestampMatch[0] : "";
+ return {
+ logType: fragment.logType,
+ timestamp: timestamp,
+ contents: singleLine,
+ };
+ })
+ // Merge each array of SortableLine into single array
+ )).reduce((acc: SortableLine[], lines: SortableLine[]) => (
+ [...acc, ...lines]
+ ), [] as SortableLine[]);
+
+ return linesWithType
+ .sort(sortableLineSortFunc)
+ .map(lineWithType => lineWithType.contents);
+};
+
+/**
+ * Sort func to sort lines
+ * Preserves original ordering of lines from the same source
+ * Stably orders lines of differing type but same timestamp
+ * (produces a block of same-timestamped lines of one type before a block
+ * of same timestamped lines of another type for readability)
+ * Sorts all other lines by contents (ie by timestamp)
+ */
+const sortableLineSortFunc = (a: SortableLine, b: SortableLine) => {
+ if (a.logType === b.logType) {
+ return 0;
+ } else if (a.timestamp === b.timestamp) {
+ return a.logType.localeCompare(b.logType);
+ } else {
+ return a.contents.localeCompare(b.contents);
+ }
};
const fragmentsToLines = (fragments: LogFragment[]): string[] => (
commit 2c7f44ed77a3df21088ec608f6a5d58dd7f65518
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Dec 7 14:14:51 2023 -0500
Fix coreui version dependency, refs #21089
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/package.json b/package.json
index d21d067540..abb204907b 100644
--- a/package.json
+++ b/package.json
@@ -3,8 +3,8 @@
"version": "0.1.0",
"private": true,
"dependencies": {
- "@coreui/coreui": "next",
- "@coreui/react": "next",
+ "@coreui/coreui": "^4.3.2",
+ "@coreui/react": "^4.11.0",
"@date-io/date-fns": "1",
"@fortawesome/fontawesome-svg-core": "1.2.28",
"@fortawesome/free-solid-svg-icons": "5.13.0",
diff --git a/yarn.lock b/yarn.lock
index ef7b9d31eb..2e0c4f2a1e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1646,23 +1646,25 @@ __metadata:
languageName: node
linkType: hard
-"@coreui/coreui at npm:next":
- version: 5.0.0-alpha.3
- resolution: "@coreui/coreui at npm:5.0.0-alpha.3"
+"@coreui/coreui at npm:^4.3.2":
+ version: 4.3.2
+ resolution: "@coreui/coreui at npm:4.3.2"
+ dependencies:
+ postcss-combine-duplicated-selectors: ^10.0.3
peerDependencies:
- "@popperjs/core": ^2.11.8
- checksum: 2363ad6be775c6a895a49126a5b9062ffa9ebd0bea6dfb835c1300cd122fb1cf18d85fe647a9c08a3a384caa871e761d8ffb28ea45c7872cb2b034df6527da20
+ "@popperjs/core": ^2.11.6
+ checksum: 88fc70f4f681bb796e1d81ca8472a3d36bfcf92866fc7c6810ead850bc371c99bca123a94abb0fafdf2935972d130005cd62b485406631cfd9abd8f38e14be15
languageName: node
linkType: hard
-"@coreui/react at npm:next":
- version: 5.0.0-alpha.3
- resolution: "@coreui/react at npm:5.0.0-alpha.3"
+"@coreui/react at npm:^4.11.0":
+ version: 4.11.0
+ resolution: "@coreui/react at npm:4.11.0"
peerDependencies:
- "@coreui/coreui": ^5.0.0-alpha.2
+ "@coreui/coreui": 4.3.0
react: ">=17"
react-dom: ">=17"
- checksum: efd333cc346307219dcf7fe183eed65305b12e71984bcb940d80a55509d7b92523082e37045bfcb8c4b334920ca185128a9f72f3e8bec69d15cad889cbeda4b4
+ checksum: 75c9394125e41e24fb5855b82cba93c9abeea080f9ee5bcc063ff2e581318b85c5bbef6f2c5300f5fd7a3450743488daa29b4baee6feabec38a009a452876a88
languageName: node
linkType: hard
@@ -3820,8 +3822,8 @@ __metadata:
version: 0.0.0-use.local
resolution: "arvados-workbench-2 at workspace:."
dependencies:
- "@coreui/coreui": next
- "@coreui/react": next
+ "@coreui/coreui": ^4.3.2
+ "@coreui/react": ^4.11.0
"@date-io/date-fns": 1
"@fortawesome/fontawesome-svg-core": 1.2.28
"@fortawesome/free-solid-svg-icons": 5.13.0
@@ -14138,6 +14140,17 @@ __metadata:
languageName: node
linkType: hard
+"postcss-combine-duplicated-selectors at npm:^10.0.3":
+ version: 10.0.3
+ resolution: "postcss-combine-duplicated-selectors at npm:10.0.3"
+ dependencies:
+ postcss-selector-parser: ^6.0.4
+ peerDependencies:
+ postcss: ^8.1.0
+ checksum: 45c3dff41d0cddb510752ed92fe8c7fc66e5cf88f4988314655419d3ecdf1dc66f484a25ee73f4f292da5da851a0fdba0ec4d59bdedeee935d05b26d31d997ed
+ languageName: node
+ linkType: hard
+
"postcss-convert-values at npm:^4.0.1":
version: 4.0.1
resolution: "postcss-convert-values at npm:4.0.1"
@@ -14782,6 +14795,16 @@ __metadata:
languageName: node
linkType: hard
+"postcss-selector-parser at npm:^6.0.4":
+ version: 6.0.13
+ resolution: "postcss-selector-parser at npm:6.0.13"
+ dependencies:
+ cssesc: ^3.0.0
+ util-deprecate: ^1.0.2
+ checksum: f89163338a1ce3b8ece8e9055cd5a3165e79a15e1c408e18de5ad8f87796b61ec2d48a2902d179ae0c4b5de10fccd3a325a4e660596549b040bc5ad1b465f096
+ languageName: node
+ linkType: hard
+
"postcss-svgo at npm:^4.0.3":
version: 4.0.3
resolution: "postcss-svgo at npm:4.0.3"
commit e06a65d3deb59887fed473e27c3eebd8aefa5090
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Dec 7 12:53:50 2023 -0500
Pass in VERSION, MAINTAINER and DESCRIPTION, refs #21089
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/Makefile b/Makefile
index 0220adadc2..c7a9cbfb8e 100644
--- a/Makefile
+++ b/Makefile
@@ -151,8 +151,11 @@ packages-in-docker: check-arvados-directory workbench2-build-image
docker run --env ci="true" \
--env ARVADOS_DIRECTORY=/tmp/arvados \
--env APP_NAME=${APP_NAME} \
+ --env VERSION="${VERSION}" \
--env ITERATION=${ITERATION} \
--env TARGETS="${TARGETS}" \
+ --env MAINTAINER="${MAINTAINER}" \
+ --env DESCRIPTION="${DESCRIPTION}" \
-w="/tmp/workbench2" \
-t -v ${WORKSPACE}:/tmp/workbench2 \
-v ${ARVADOS_DIRECTORY}:/tmp/arvados workbench2-build:latest \
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list