[ARVADOS] created: 2.1.0-1975-g926c011fb
Git user
git at public.arvados.org
Mon Feb 28 22:08:20 UTC 2022
at 926c011fb4f7a4d7722b88a19afed51c5d4bd1c4 (commit)
commit 926c011fb4f7a4d7722b88a19afed51c5d4bd1c4
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Feb 28 17:06:11 2022 -0500
18656: Update cwltool version and tests for CUDA extension tweaks
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/.licenseignore b/.licenseignore
index e3289aa7c..97ce38af9 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -87,4 +87,5 @@ sdk/python/tests/fed-migrate/*.cwl
sdk/python/tests/fed-migrate/*.cwlex
doc/install/*.xlsx
sdk/cwl/tests/wf/hello.txt
-sdk/cwl/tests/wf/indir1/hello2.txt
\ No newline at end of file
+sdk/cwl/tests/wf/indir1/hello2.txt
+sdk/cwl/tests/chipseq/data/Genomes/*
\ No newline at end of file
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index d5efa31a0..6e2d4f1d9 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -359,13 +359,29 @@ $graph:
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index 4a6b6947f..0e81347d7 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -302,13 +302,29 @@ $graph:
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index e95b6543f..e9f70bf1c 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -304,13 +304,29 @@ $graph:
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 753c2c250..2a5ff3a13 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -295,9 +295,9 @@ class ArvadosContainer(JobBase):
cuda_req, _ = self.get_requirement("http://commonwl.org/cwltool#CUDARequirement")
if cuda_req:
runtime_constraints["cuda"] = {
- "device_count": cuda_req.get("deviceCountMin", 1),
+ "device_count": resources.get("cudaDeviceCount", 1),
"driver_version": cuda_req["cudaVersionMin"],
- "hardware_capability": cuda_req["cudaComputeCapabilityMin"]
+ "hardware_capability": aslist(cuda_req["cudaComputeCapability"])[0]
}
if self.timelimit is not None and self.timelimit > 0:
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index e126d170b..c885ebd4b 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -36,7 +36,7 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20220217222804',
+ 'cwltool==3.1.20220224085855',
'schema-salad==8.2.20211116214159',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 72774daba..bfffe6eac 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -1053,68 +1053,90 @@ class TestContainer(unittest.TestCase):
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- tool = cmap({
- "inputs": [],
- "outputs": [],
- "baseCommand": "nvidia-smi",
- "arguments": [],
- "id": "",
- "cwlVersion": "v1.2",
- "class": "CommandLineTool",
- "requirements": [
- {
+ test_cwl_req = [{
"class": "http://commonwl.org/cwltool#CUDARequirement",
"cudaVersionMin": "11.0",
- "cudaComputeCapabilityMin": "9.0",
- }
- ]
- })
+ "cudaComputeCapability": "9.0",
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": "9.0",
+ "cudaDeviceCountMin": 2
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": ["4.0", "5.0"],
+ "cudaDeviceCountMin": 2
+ }]
+
+ test_arv_req = [{
+ 'device_count': 1,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "4.0"
+ }]
+
+ for test_case in range(0, len(test_cwl_req)):
- loadingContext, runtimeContext = self.helper(runner, True)
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "nvidia-smi",
+ "arguments": [],
+ "id": "",
+ "cwlVersion": "v1.2",
+ "class": "CommandLineTool",
+ "requirements": [test_cwl_req[test_case]]
+ })
- arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
- arvtool.formatgraph = None
+ loadingContext, runtimeContext = self.helper(runner, True)
- for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
- j.run(runtimeContext)
- runner.api.container_requests().create.assert_called_with(
- body=JsonDiffMatcher({
- 'environment': {
- 'HOME': '/var/spool/cwl',
- 'TMPDIR': '/tmp'
- },
- 'name': 'test_run_True',
- 'runtime_constraints': {
- 'vcpus': 1,
- 'ram': 268435456,
- 'cuda': {
- 'device_count': 1,
- 'driver_version': "11.0",
- 'hardware_capability': "9.0"
- }
- },
- 'use_existing': True,
- 'priority': 500,
- 'mounts': {
- '/tmp': {'kind': 'tmp',
- "capacity": 1073741824
- },
- '/var/spool/cwl': {'kind': 'tmp',
- "capacity": 1073741824 }
- },
- 'state': 'Committed',
- 'output_name': 'Output for step test_run_True',
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
- 'output_path': '/var/spool/cwl',
- 'output_ttl': 0,
- 'container_image': '99999999999999999999999999999993+99',
- 'command': ['nvidia-smi'],
- 'cwd': '/var/spool/cwl',
- 'scheduling_parameters': {},
- 'properties': {},
- 'secret_mounts': {},
- 'output_storage_classes': ["default"]
- }))
+ arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 268435456,
+ 'cuda': test_arv_req[test_case]
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['nvidia-smi'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }))
# The test passes no builder.resources
commit 2f61343f1eb92962c938100f980fbeab93525a56
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Feb 28 16:40:40 2022 -0500
18656: Add missing test files
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz b/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz
new file mode 100644
index 000000000..e69de29bb
diff --git a/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz b/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz
new file mode 100644
index 000000000..e69de29bb
diff --git a/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz b/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz
new file mode 100644
index 000000000..e69de29bb
diff --git a/sdk/cwl/tests/chipseq/chip-seq-single.json b/sdk/cwl/tests/chipseq/chip-seq-single.json
new file mode 100644
index 000000000..758390e8b
--- /dev/null
+++ b/sdk/cwl/tests/chipseq/chip-seq-single.json
@@ -0,0 +1,99 @@
+{
+ "referenceGenomeSequence": {
+ "class": "File",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "referenceGenomeSequenceDrosophila": {
+ "class": "File",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "blacklistBed": {
+ "class": "File",
+ "location": "data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieHumanReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieDrosophilaReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "sampleName": "LED054_0p03nMR1.0",
+ "inputFastq1": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastq2": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastqUMI": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz",
+ "secondaryFiles": []
+ }
+}
+
diff --git a/sdk/cwl/tests/chipseq/cwl-packed.json b/sdk/cwl/tests/chipseq/cwl-packed.json
new file mode 100644
index 000000000..8921bcbe5
--- /dev/null
+++ b/sdk/cwl/tests/chipseq/cwl-packed.json
@@ -0,0 +1,94 @@
+{
+ "$graph": [
+ {
+ "class": "Workflow",
+ "id": "#main",
+ "doc": "Pipeline that is applied on single ChIP-seq samples.\n\nStarts with QC on the reads and trimming (for adapters and based on quality)\n\nAligns to human genome and adds UMI\n\nAligns to Drosophila genome and counts the number of reads.\n\nAfter the alignment to human genome the files are filtered for duplicates, multimappers and alignments in black listed regions",
+ "label": "ChIP-Seq (single sample)",
+ "inputs": [
+ {
+ "id": "#inputFastq1",
+ "type": "File",
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1726.25
+ },
+ {
+ "id": "#blacklistBed",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 746.4744873046875,
+ "https://www.sevenbridges.com/y": 1903.265625
+ },
+ {
+ "id": "#referenceGenomeSequence",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai",
+ "^.dict"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fasta, fa",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1405.203125
+ },
+ {
+ "id": "#sampleName",
+ "type": "string",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1191.171875
+ },
+ {
+ "id": "#inputFastq2",
+ "type": [
+ "null",
+ "File"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1619.234375
+ },
+ {
+ "id": "#inputFastqUMI",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1512.21875
+ },
+ {
+ "id": "#BowtieHumanReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1519.21875
+ },
+ {
+ "id": "#BowtieDrosophilaReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1626.234375
+ },
+ {
+ "id": "#referenceGenomeSequenceDrosophila",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai"
+ ],
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1298.1875
+ }
+ ],
+ "outputs": [
+ ],
+ "steps": [
+ {
+ "id": "#step1",
+ "in": {
+ "inp": "#inputFastq1"
+ },
+ "out": [],
+ "run": "../cat.cwl"
+ }
+ ],
+ "requirements": [
+ ]
+ },
+ ],
+ "cwlVersion": "v1.0"
+}
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed b/sdk/cwl/tests/chipseq/data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed
new file mode 100644
index 000000000..e69de29bb
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/genome.fa
new file mode 100644
index 000000000..e69de29bb
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa
new file mode 100644
index 000000000..e69de29bb
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/genome.fa
new file mode 100644
index 000000000..e69de29bb
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa
new file mode 100644
index 000000000..e69de29bb
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list