diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 76f1fc488..f411dea58 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -1,5 +1,6 @@ """Support for executing Docker format containers using Singularity {2,3}.x or Apptainer 1.x.""" +import json import logging import os import os.path @@ -7,9 +8,10 @@ import shutil import sys from collections.abc import Callable, MutableMapping -from subprocess import check_call, check_output # nosec +from subprocess import check_call, check_output, run # nosec from typing import cast +from mypy_extensions import mypyc_attr from schema_salad.sourceline import SourceLine from spython.main import Client from spython.main.parse.parsers.docker import DockerParser @@ -145,6 +147,30 @@ def _normalize_sif_id(string: str) -> str: return string.replace("/", "_") + ".sif" +@mypyc_attr(allow_interpreted_subclasses=True) +def _inspect_singularity_image(path: str) -> bool: + """Inspect singularity image to be sure it is not an empty directory.""" + cmd = [ + "singularity", + "inspect", + "--json", + path, + ] + try: + result = run(cmd, capture_output=True, text=True) # nosec + except Exception: + return False + + if result.returncode == 0: + try: + output = json.loads(result.stdout) + except json.JSONDecodeError: + return False + if output.get("data", {}).get("attributes", {}): + return True + return False + + class SingularityCommandLineJob(ContainerCommandLineJob): def __init__( self, @@ -229,6 +255,16 @@ def get_image( ) found = True elif "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement: + # looking for local singularity sandbox image and handle it as a local image + if os.path.isdir(dockerRequirement["dockerPull"]) and _inspect_singularity_image( + dockerRequirement["dockerPull"] + ): + dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"] + _logger.info( + "Using local Singularity sandbox image found in %s", + dockerRequirement["dockerImageId"], + ) + return True match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"]) img_name = _normalize_image_id(dockerRequirement["dockerPull"]) candidates.append(img_name) @@ -243,6 +279,15 @@ def get_image( elif "dockerImageId" in dockerRequirement: if os.path.isfile(dockerRequirement["dockerImageId"]): found = True + # handling local singularity sandbox image + elif os.path.isdir(dockerRequirement["dockerImageId"]) and _inspect_singularity_image( + dockerRequirement["dockerImageId"] + ): + _logger.info( + "Using local Singularity sandbox image found in %s", + dockerRequirement["dockerImageId"], + ) + return True candidates.append(dockerRequirement["dockerImageId"]) candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"])) if is_version_3_or_newer(): diff --git a/tests/sing_local_sandbox_img_id_test.cwl b/tests/sing_local_sandbox_img_id_test.cwl new file mode 100755 index 000000000..9c44a0cc5 --- /dev/null +++ b/tests/sing_local_sandbox_img_id_test.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerImageId: container_repo/alpine + +inputs: + message: string + +outputs: [] + +baseCommand: echo diff --git a/tests/sing_local_sandbox_test.cwl b/tests/sing_local_sandbox_test.cwl new file mode 100755 index 000000000..64d6f6b1c --- /dev/null +++ b/tests/sing_local_sandbox_test.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: container_repo/alpine + +inputs: + message: string + +outputs: [] + +baseCommand: echo diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 1139dfbc7..ad94e4281 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,11 +1,17 @@ """Tests to find local Singularity image.""" +import json import shutil +import subprocess +from collections.abc import Callable from pathlib import Path +from typing import Any import pytest +from mypy_extensions import KwArg, VarArg from cwltool.main import main +from cwltool.singularity import _inspect_singularity_image from .util import ( get_data, @@ -159,3 +165,124 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None: ] ) assert result_code1 == 0 + + +@needs_singularity +def test_singularity_local_sandbox_image(tmp_path: Path) -> None: + workdir = tmp_path / "working_dir" + workdir.mkdir() + with working_directory(workdir): + # build a sandbox image + container_path = workdir / "container_repo" + container_path.mkdir() + cmd = [ + "singularity", + "build", + "--sandbox", + str(container_path / "alpine"), + "docker://alpine:latest", + ] + + build = subprocess.run(cmd, capture_output=True, text=True) + if build.returncode == 0: + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + "--disable-pull", + get_data("tests/sing_local_sandbox_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0 + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + "--disable-pull", + get_data("tests/sing_local_sandbox_img_id_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0 + else: + pytest.skip(f"Failed to build the singularity image: {build.stderr}") + + +@needs_singularity +def test_singularity_inspect_image(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Test inspect a real image works.""" + workdir = tmp_path / "working_dir" + workdir.mkdir() + repo_path = workdir / "container_repo" + image_path = repo_path / "alpine" + + # test image exists + repo_path.mkdir() + cmd = [ + "singularity", + "build", + "--sandbox", + str(image_path), + "docker://alpine:latest", + ] + build = subprocess.run(cmd, capture_output=True, text=True) + if build.returncode == 0: + # Verify the path is a correct container image + res_inspect = _inspect_singularity_image(str(image_path)) + assert res_inspect is True + else: + pytest.skip(f"singularity sandbox image build didn't worked: {build.stderr}") + + +class _DummyResult: # noqa: B903 + def __init__(self, rc: int, out: str) -> None: + self.returncode = rc + self.stdout = out + + +def _make_run_result( + returncode: int, stdout: str +) -> Callable[[VarArg(Any), KwArg(Any)], _DummyResult]: + """Mock subprocess.run returning returncode and stdout.""" + + def _runner(*args: Any, **kwargs: Any) -> _DummyResult: + return _DummyResult(returncode, stdout) + + return _runner + + +def test_json_decode_error_branch(monkeypatch: pytest.MonkeyPatch) -> None: + """Test json can't decode inspect result.""" + monkeypatch.setattr("cwltool.singularity.run", _make_run_result(0, "not-a-json")) + + def _raise_json_error(s: str) -> None: + # construct and raise an actual JSONDecodeError + raise json.JSONDecodeError("Expecting value", s, 0) + + monkeypatch.setattr("json.loads", _raise_json_error) + + assert _inspect_singularity_image("/tmp/image") is False + + +def test_singularity_sandbox_image_not_exists() -> None: + image_path = "/tmp/not_existing/image" + res_inspect = _inspect_singularity_image(image_path) + assert res_inspect is False + + +def test_singularity_sandbox_not_an_image(tmp_path: Path) -> None: + image_path = tmp_path / "image" + image_path.mkdir() + res_inspect = _inspect_singularity_image(str(image_path)) + assert res_inspect is False + + +def test_inspect_image_wrong_sb_call(monkeypatch: pytest.MonkeyPatch) -> None: + + def mock_failed_subprocess(*args: Any, **kwargs: Any) -> None: + raise subprocess.CalledProcessError(returncode=1, cmd=args[0]) + + monkeypatch.setattr("cwltool.singularity.run", mock_failed_subprocess) + res_inspect = _inspect_singularity_image("/tmp/container_repo/alpine") + assert res_inspect is False diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 18a588cf8..168ce28a1 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -285,6 +285,99 @@ def test_dockerfile_singularity_build(monkeypatch: pytest.MonkeyPatch, tmp_path: shutil.rmtree(subdir) +@needs_singularity +def test_singularity_get_image_from_sandbox( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Test that SingularityCommandLineJob.get_image correctly handle sandbox image.""" + + (tmp_path / "out").mkdir(exist_ok=True) + tmp_outdir_prefix = tmp_path / "out" + tmp_outdir_prefix.mkdir(exist_ok=True) + (tmp_path / "tmp").mkdir(exist_ok=True) + tmpdir_prefix = str(tmp_path / "tmp") + runtime_context = RuntimeContext( + {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None} + ) + builder = Builder( + {}, + [], + [], + {}, + schema.Names(), + [], + [], + {}, + None, + None, + StdFsAccess, + StdFsAccess(""), + None, + 0.1, + True, + False, + False, + "no_listing", + runtime_context.get_outdir(), + runtime_context.get_tmpdir(), + runtime_context.get_stagedir(), + INTERNAL_VERSION, + "singularity", + ) + + workdir = tmp_path / "working_dir" + workdir.mkdir() + repo_path = workdir / "container_repo" + repo_path.mkdir() + image_path = repo_path / "alpine" + image_path.mkdir() + + # directory exists but is not an image + monkeypatch.setattr( + "cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: False + ) + req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerPull"].startswith("docker://") + assert res is False + + # directory exists and is an image: + monkeypatch.setattr( + "cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: True + ) + req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerImageId"] == str(image_path) + assert res + + # test that dockerImageId is set and image exists: + req = {"class": "DockerRequirement", "dockerImageId": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerImageId"] == str(image_path) + assert res + + def test_docker_tmpdir_prefix(tmp_path: Path) -> None: """Test that DockerCommandLineJob respects temp directory directives.""" (tmp_path / "3").mkdir()