From 07b393226d02a3c437fa233e59237d7044209d0e Mon Sep 17 00:00:00 2001 From: "Tristan F.-R." Date: Mon, 26 May 2025 10:01:07 -0700 Subject: [PATCH 1/6] feat: omicsintegrator2 offline support [I mainly used this issue as an excuse to experiment with a git patch workflow over the forking method that seems to be present in SPRAS] Forces OI2 to work offline by disabling the unused networkx graph as interactive HTML feature (we already have workflows in SPRAS that enable this). The network error was coming from `output_networkx_graph_as_interactive_html` dynamically fetching jQuery to insert it as an inline script. This could have also been fixed by providing a `local` folder with all of the scripts, but putting the several hundred-kilobyte scripts on this repository did not seem worth it for an unused feature. --- .../0001-disable-graph-exports.patch | 26 +++++++++++++++++++ docker-wrappers/OmicsIntegrator2/Dockerfile | 9 +++++++ .../OmicsIntegrator2/environment.yml | 3 ++- spras/containers.py | 6 +++-- spras/omicsintegrator2.py | 3 ++- 5 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 docker-wrappers/OmicsIntegrator2/0001-disable-graph-exports.patch diff --git a/docker-wrappers/OmicsIntegrator2/0001-disable-graph-exports.patch b/docker-wrappers/OmicsIntegrator2/0001-disable-graph-exports.patch new file mode 100644 index 000000000..b482261b4 --- /dev/null +++ b/docker-wrappers/OmicsIntegrator2/0001-disable-graph-exports.patch @@ -0,0 +1,26 @@ +From 5dc0e69fa3d1049ae8e1d8f51859335910245ad7 Mon Sep 17 00:00:00 2001 +From: "Tristan F.-R." +Date: Mon, 26 May 2025 09:52:34 -0700 +Subject: [PATCH] fix: disable graph exports + +this allows OI2 to work offline; plus, SPRAS already has graph visualizers. +--- + src/__main__.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/__main__.py b/src/__main__.py +index 49ef402..b7afbca 100644 +--- a/src/__main__.py ++++ b/src/__main__.py +@@ -80,7 +80,7 @@ def main(): + forest, augmented_forest = graph.output_forest_as_networkx(vertex_indices, edge_indices) + + #oi.output_networkx_graph_as_graphml_for_cytoscape(augmented_forest, args.output_dir) +- oi.output_networkx_graph_as_interactive_html(augmented_forest, args.output_dir, args.filename+'.html') ++ # oi.output_networkx_graph_as_interactive_html(augmented_forest, args.output_dir, args.filename+'.html') + augmented_forest_df = oi.get_networkx_graph_as_dataframe_of_edges(augmented_forest) + output_dataframe_to_tsv(augmented_forest_df, args.output_dir, args.filename+'.tsv') + +-- +2.47.0 + diff --git a/docker-wrappers/OmicsIntegrator2/Dockerfile b/docker-wrappers/OmicsIntegrator2/Dockerfile index e0df2d2db..67ceb23a2 100644 --- a/docker-wrappers/OmicsIntegrator2/Dockerfile +++ b/docker-wrappers/OmicsIntegrator2/Dockerfile @@ -2,5 +2,14 @@ # https://github.com/fraenkel-lab/OmicsIntegrator2 FROM continuumio/miniconda3:4.9.2 +COPY 0001-disable-graph-exports.patch . + +RUN git clone https://github.com/agitter/OmicsIntegrator2 && \ + cd OmicsIntegrator2 && \ + git reset --hard 568f170eae388e42e923c478ac9f3308b487760b && \ + git config user.email "email@example.com" && \ + git config user.name "Non-existent User" && \ + git am /0001-disable-graph-exports.patch + COPY environment.yml . RUN conda env update --name base --file environment.yml --prune diff --git a/docker-wrappers/OmicsIntegrator2/environment.yml b/docker-wrappers/OmicsIntegrator2/environment.yml index e1183aaca..00e29f60f 100644 --- a/docker-wrappers/OmicsIntegrator2/environment.yml +++ b/docker-wrappers/OmicsIntegrator2/environment.yml @@ -18,4 +18,5 @@ dependencies: - pcst_fast==1.0.7 - goenrich==1.7.0 - axial==0.1.10 - - git+https://github.com/agitter/OmicsIntegrator2@568f170eae388e42e923c478ac9f3308b487760b + - git+file:///OmicsIntegrator2 + \ No newline at end of file diff --git a/spras/containers.py b/spras/containers.py index 12b658ad9..5fdde10fa 100644 --- a/spras/containers.py +++ b/spras/containers.py @@ -125,7 +125,7 @@ def prepare_dsub_cmd(flags: dict): # run_container_singularity assumes a single string # Follow docker-py's naming conventions (https://docker-py.readthedocs.io/en/stable/containers.html) # Technically the argument is an image, not a container, but we use container here. -def run_container(framework: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, environment: str = 'SPRAS=True'): +def run_container(framework: str, container_suffix: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, environment: str = 'SPRAS=True', network_disabled=False): """ Runs a command in the container using Singularity or Docker @param framework: singularity or docker @@ -134,6 +134,7 @@ def run_container(framework: str, container_suffix: str, command: List[str], vol @param volumes: a list of volumes to mount where each item is a (source, destination) tuple @param working_dir: the working directory in the container @param environment: environment variables to set in the container + @param network_disabled: Disables the network on the container. Only works for docker for now. This acts as a 'runtime assertion' that a container works w/o networking. @return: output from Singularity execute or Docker run """ normalized_framework = framework.casefold() @@ -150,7 +151,7 @@ def run_container(framework: str, container_suffix: str, command: List[str], vol # TODO any issue with creating a new client each time inside this function? -def run_container_docker(container: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, environment: str = 'SPRAS=True'): +def run_container_docker(container: str, command: List[str], volumes: List[Tuple[PurePath, PurePath]], working_dir: str, environment: str = 'SPRAS=True', network_disabled=False): """ Runs a command in the container using Docker. Attempts to automatically correct file owner and group for new files created by the container, setting them to the @@ -189,6 +190,7 @@ def run_container_docker(container: str, command: List[str], volumes: List[Tuple stderr=True, volumes=bind_paths, working_dir=working_dir, + network_disabled=network_disabled, environment=[environment]).decode('utf-8') # TODO does this cleanup need to still run even if there was an error in the above run command? diff --git a/spras/omicsintegrator2.py b/spras/omicsintegrator2.py index 21950ae3d..ea9513b23 100644 --- a/spras/omicsintegrator2.py +++ b/spras/omicsintegrator2.py @@ -125,7 +125,8 @@ def run(edges=None, prizes=None, output_file=None, w=None, b=None, g=None, noise container_suffix, command, volumes, - work_dir) + work_dir, + network_disabled=True) print(out) # TODO do we want to retain other output files? From 8aca9830e4cbff9f66b22d4c20ce41bc377b6f3d Mon Sep 17 00:00:00 2001 From: "Tristan F.-R." Date: Mon, 26 May 2025 17:02:37 +0000 Subject: [PATCH 2/6] style: rm whitespace in environment.yml --- docker-wrappers/OmicsIntegrator2/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-wrappers/OmicsIntegrator2/environment.yml b/docker-wrappers/OmicsIntegrator2/environment.yml index 00e29f60f..cf44c880a 100644 --- a/docker-wrappers/OmicsIntegrator2/environment.yml +++ b/docker-wrappers/OmicsIntegrator2/environment.yml @@ -19,4 +19,3 @@ dependencies: - goenrich==1.7.0 - axial==0.1.10 - git+file:///OmicsIntegrator2 - \ No newline at end of file From 7bd9c3d8dce8b993e4288bbf41b143fef8293ee3 Mon Sep 17 00:00:00 2001 From: "Tristan F.-R." Date: Mon, 16 Jun 2025 11:43:56 -0700 Subject: [PATCH 3/6] style: fmt --- .../OmicsIntegrator2/environment.yml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docker-wrappers/OmicsIntegrator2/environment.yml b/docker-wrappers/OmicsIntegrator2/environment.yml index cf44c880a..09e8d1b93 100644 --- a/docker-wrappers/OmicsIntegrator2/environment.yml +++ b/docker-wrappers/OmicsIntegrator2/environment.yml @@ -1,21 +1,21 @@ name: oi2 dependencies: -- git-lfs=2.13 -- numpy=1.19 -- pandas=0.23.4 -- scikit-learn=0.19 -- networkx=2.1 -- jinja2=2.9 -- python-louvain=0.11 -- scipy=1.1 -- python=3.6 -- pip=21.0 -- patsy=0.5 -- requests=2.25 -- statsmodels=0.12 -- pybind11=2.5 -- pip: - - pcst_fast==1.0.7 - - goenrich==1.7.0 - - axial==0.1.10 - - git+file:///OmicsIntegrator2 + - git-lfs=2.13 + - numpy=1.19 + - pandas=0.23.4 + - scikit-learn=0.19 + - networkx=2.1 + - jinja2=2.9 + - python-louvain=0.11 + - scipy=1.1 + - python=3.6 + - pip=21.0 + - patsy=0.5 + - requests=2.25 + - statsmodels=0.12 + - pybind11=2.5 + - pip: + - pcst_fast==1.0.7 + - goenrich==1.7.0 + - axial==0.1.10 + - git+file:///OmicsIntegrator2 From 01f5dcd5ad7603eff7be6bc185a72b02e1e41a69 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Fri, 15 Aug 2025 04:10:42 -0700 Subject: [PATCH 4/6] docs: patching algorithms --- docs/contributing/patching.rst | 23 +++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 24 insertions(+) create mode 100644 docs/contributing/patching.rst diff --git a/docs/contributing/patching.rst b/docs/contributing/patching.rst new file mode 100644 index 000000000..8c1ead0c4 --- /dev/null +++ b/docs/contributing/patching.rst @@ -0,0 +1,23 @@ +Patching Algorithms +=================== + +Some wrapped algorithms require extra fixes inside their code. For permissively licensed algorithms, +we use ``.patch`` files generated from ``git format-patch``. + +To create patch files using ``git format-patch`` (assuming your wrapped algorithm is in a git repository): + +#. Clone the repository locally. +#. Commit the changes you want to make (with good commit messages and descriptions). + + * Distinct changes should be made in different commits to make patch files easy to read + * For removing code, we prefer to comment out code instead of removing it, to make potential stacktraces less confusing for end users. + +#. Run ``git format-patch HEAD~[N]`` where ``N`` is the number of commits you made. + +To use ``.patch`` files in ``Dockerfiles``, we create a fake user for ``git`` and apply the patch files using ``git am``: + +.. code:: shell + + git config user.email "email@example.com" && \ + git config user.name "Non-existent User" && \ + git am /0001-my-patch.patch diff --git a/docs/index.rst b/docs/index.rst index d12af3157..19e30b0b7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -55,6 +55,7 @@ methods (PRMs) to omics data. contributing/index contributing/maintain + contributing/patching Indices and tables ================== From be107422f0509e131fc71013b5513c9b20979c29 Mon Sep 17 00:00:00 2001 From: "Tristan F." Date: Fri, 15 Aug 2025 12:24:02 +0000 Subject: [PATCH 5/6] docs: drop && / from docker --- docs/contributing/patching.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing/patching.rst b/docs/contributing/patching.rst index 8c1ead0c4..7dc617e73 100644 --- a/docs/contributing/patching.rst +++ b/docs/contributing/patching.rst @@ -18,6 +18,6 @@ To use ``.patch`` files in ``Dockerfiles``, we create a fake user for ``git`` an .. code:: shell - git config user.email "email@example.com" && \ - git config user.name "Non-existent User" && \ + git config user.email "email@example.com" + git config user.name "Non-existent User" git am /0001-my-patch.patch From 4fd96f57e8b027e6cf0da527e9f39a07dde081d9 Mon Sep 17 00:00:00 2001 From: "Tristan F.-R." Date: Fri, 3 Oct 2025 21:40:04 +0000 Subject: [PATCH 6/6] docs: bump oi2 --- docker-wrappers/OmicsIntegrator2/README.md | 1 + spras/omicsintegrator2.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-wrappers/OmicsIntegrator2/README.md b/docker-wrappers/OmicsIntegrator2/README.md index c7c7d11f9..881ce2d1f 100644 --- a/docker-wrappers/OmicsIntegrator2/README.md +++ b/docker-wrappers/OmicsIntegrator2/README.md @@ -25,6 +25,7 @@ The Docker wrapper can be tested with `pytest`. ## Versions: - v1: Created a named conda environment in the container and used `ENTRYPOINT` to execute commands inside that environment. Not compatible with Singularity. - v2: Used the environment file to update the base conda environment so the `ENTRYPOINT` command was no longer needed. Compatible with Singularity. +- v3: Patch to work offline by never running `output_networkx_graph_as_interactive_html` ([#226](https://github.com/Reed-CompBio/spras/pull/226)) ## TODO - Attribute https://github.com/fraenkel-lab/OmicsIntegrator2 diff --git a/spras/omicsintegrator2.py b/spras/omicsintegrator2.py index 160caba93..8b97fa2d1 100644 --- a/spras/omicsintegrator2.py +++ b/spras/omicsintegrator2.py @@ -128,14 +128,14 @@ def run(edges=None, prizes=None, output_file=None, w=None, b=None, g=None, noise if seed is not None: command.extend(['--seed', str(seed)]) - container_suffix = "omics-integrator-2:v2" + container_suffix = "omics-integrator-2:v3" run_container_and_log('Omics Integrator 2', container_framework, container_suffix, command, volumes, work_dir, - out_dir + out_dir, network_disabled=True) # TODO do we want to retain other output files?