diff --git a/.DS_Store b/.DS_Store index fb3ddd8..ec02740 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/README.md b/README.md index b2746c4..b6203f3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,172 @@ -# NAISR -NAISR: A 3D Neural Additive Model for Interpretable Shape Representation +# NAISR: A 3D Neural Additive Model for Interpretable Shape Representation +Pytorch implementation for our `NAISR` paper
+[NAISR: A 3D Neural Additive Model for Interpretable Shape Representation](https://arxiv.org/abs/2303.09234), ICLR 2024 **Spotlight**.
+Yining Jiao, Carlton Zdanski, Julia Kimbell, Andrew Prince, Cameron Worden, Samuel Kirse, Christopher Rutter, Benjamin Shields, William Dunn, Jisan Mahmud, Marc Niethammer.
+UNC-Chapel Hill -Codes on the way... +#### [Paper](https://arxiv.org/abs/2303.09234) | [Project Page](https://uncbiag.github.io/NAISR/) | [Colab Demos](https://colab.research.google.com/drive/1OudGynEydIXpAgfA9lvi5d0L86kK-uwg?usp=sharing) + + +
+ +Please cite as: +``` +@inproceedings{ +jiao2024naisr, +title={\texttt{NAISR}: A 3D Neural Additive Model for Interpretable Shape Representation}, +author={Yining Jiao and Carlton Jude ZDANSKI and Julia S Kimbell and Andrew Prince and Cameron P Worden and Samuel Kirse and Christopher Rutter and Benjamin Shields and William Alexander Dunn and Jisan Mahmud and Marc Niethammer}, +booktitle={The Twelfth International Conference on Learning Representations}, +year={2024}, +url={https://openreview.net/forum?id=wg8NPfeMF9} +} +``` + +>[!NOTE] +>For the three datasets used in `NAISR` paper, we provided this [Colab demo](https://colab.research.google.com/drive/1OudGynEydIXpAgfA9lvi5d0L86kK-uwg) to investigate the shapes learned with `NAISR`.
+We also provide the instructions on this page to apply `NAISR` to your customized shape analysis questions. + + +## Installation +The code is tested with ``python=3.9``, ``torch=2.1.0``, ``torchvision=0.15.2``. +``` +git clone https://github.com/uncbiag/NAISR +cd NAISR +``` +Now, create a new conda environment and install required packages accordingly. +``` +conda create -n naisr python=3.9 +conda activate naisr +pip install -r requirements.txt +``` + + +## Data and Model Weights +We train and test our method on three datasets: Starman, ADNI Hippocampus, and Pediatric Airways. + +| Dataset | Description | Dataset Link | Model Link | +|-----------|----------------------------------------------|:------------------------------------:|:------------------------------------:| +|Starman | simulated 2D Starman shapes | [simulation code)][StarmanData] | [weights][StarmanModel] | +|ADNI Hippocampus | 1632 hippocampus from ADNI | [official site][ADNIData] | [weights][ADNIModel] | +|Pedeatric Airway | 357 pediatric airway shapes | NA | [weights][AirwayModel] | + + + +[ADNIData]: https://ida.loni.usc.edu/login.jsp?project=ADNI +[StarmanData]: + +[ADNIModel]: https://github.com/uncbiag/NAISR/releases/download/naisr_weights_v0/naisr_weights_adni.pth +[StarmanModel]: https://github.com/uncbiag/NAISR/releases/download/naisr_weights_v0/naisr_weights_starman.pth +[AirwayModel]: https://github.com/uncbiag/NAISR/releases/download/naisr_weights_v0/naisr_weights_pediatric_airway.pth + + +## Visualization of Existing Shape Space +To get the shape space like the following figures for the three datasets used in the paper, one just need to apply the model weights with the following command, + +First, download the `NAISR model weights`. These weights will be automatically saved to the ``checkpoints`` folder. +``` +python download.py +``` + +Run interactive GUI with the downloaded weights. The ``assets`` contains images for demo. +``` +python evolution_shapematrix.py -examples/hippocampus/naigsr_0920_base.json +``` + + + + +## Customize +### Data Preprocessing + +#### Alignment with Rigid Transformation +The shapes to explore need to be registered with a rigid transformation (translation + rotation). +If paired point clouds are available, we recommend to use ; otherwise, we recommend to use ICP to register the point clouds. +In our case, we use airway landmarks to learn the rigid transformation; and ICP algorithms to register the point clouds of the hippocampi. + +#### SDF Extraction + +### +```json + { + "Description" : [ "This experiment learns a shape representation for starman dataset." ], + "Device":0, + "DataSource": {"train": "/home/jyn/NAISR/examples/starman/2dshape_train_with_temp.csv", + "test": "/home/jyn/NAISR/examples/starman/2dshape_test_with_temp.csv"}, + "Split": null, + "Network": "DeepNAIGSR", + "NumEpochs": 300, + "LoggingRoot": "/playpen-raid/jyn/NAISR/log", + "ExperimentName": "DeepNAIGSR_STARMAN3D_0222_256_base", + + "EpochsTilCkpt": 10, + "StepsTilSummary": 1000, + "UseLBFGS": false, + "DoublePrecision": false, + "CheckpointPath": "", + "CodeLength": 256, + + "AdditionalSnapshots" : [ 50, 100, 200, 300, 400, 500 ], + "LearningRateSchedule" : [ + { + "Type": "Step", + "Initial": 0.00005, + "Interval": 1000, + "Factor": 0.5 + }, + { + "Type": "Step", + "Initial": 0.001, + "Interval": 1000, + "Factor": 0.5 + }], + "SamplesPerScene" : 750, + "BatchSize": 64, + "DataLoaderThreads": 4, + "ClampingDistance": 1, + + "Articulation": true, + "NumAtcParts": 1, + "TrainWithParts": false, + "Class": "starman", + "Attributes": ["cov_1", "cov_2"], + "TemplateAttributes": {"cov_1": 0, "cov_2": 0}, + "Backbone": "siren", + "PosEnc": false, + "InFeatures": 2, + "HiddenFeatures": 256, + "HidenLayers": 6, + "OutFeatures": 1, + "Loss": { + "whether_sdf": true, + "whether_normal_constraint": true, + "whether_inter_constraint": true, + "whether_eikonal": true, + "whether_code_regularization": true} +``` +### Training + +### Testing + +### Application - Visualization of Your Shape Space + + + + +Instructions on the way... + + + + + +## Cite this work +``` +@inproceedings{ +jiao2024naisr, +title={\texttt{NAISR}: A 3D Neural Additive Model for Interpretable Shape Representation}, +author={Yining Jiao and Carlton Jude ZDANSKI and Julia S Kimbell and Andrew Prince and Cameron P Worden and Samuel Kirse and Christopher Rutter and Benjamin Shields and William Alexander Dunn and Jisan Mahmud and Marc Niethammer}, +booktitle={The Twelfth International Conference on Learning Representations}, +year={2024}, +url={https://openreview.net/forum?id=wg8NPfeMF9} +} +``` diff --git a/docs/NAISR_icon.png b/docs/NAISR_icon.png deleted file mode 100644 index 5288aa1..0000000 Binary files a/docs/NAISR_icon.png and /dev/null differ diff --git a/docs/NAISR_icon.svg b/docs/NAISR_icon.svg new file mode 100644 index 0000000..081677d --- /dev/null +++ b/docs/NAISR_icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/NAISR poster (2).png b/docs/NAISR_poster.png similarity index 100% rename from docs/NAISR poster (2).png rename to docs/NAISR_poster.png diff --git a/docs/index.html b/docs/index.html index 45dd3c3..eda1e60 100644 --- a/docs/index.html +++ b/docs/index.html @@ -6,6 +6,7 @@ content="Deformable Neural Radiance Fields creates free-viewpoint portraits (nerfies) from casually captured videos."> + NAISR: A 3D Neural Additive Model for Interpretable Shape Representation @@ -104,6 +105,16 @@

NAISR: A 3D Neural Additive Mo + + + + + + Colab + + + diff --git a/figs/.DS_Store b/figs/.DS_Store deleted file mode 100755 index fe4a0e4..0000000 Binary files a/figs/.DS_Store and /dev/null differ diff --git a/figs/Manifold.png b/figs/Manifold.png deleted file mode 100644 index 0da95a1..0000000 Binary files a/figs/Manifold.png and /dev/null differ diff --git a/figs/NAISR_fig.png b/figs/NAISR_fig.png deleted file mode 100644 index ef36f6a..0000000 Binary files a/figs/NAISR_fig.png and /dev/null differ diff --git a/figs/NAISRoverviewwide.pdf b/figs/NAISRoverviewwide.pdf deleted file mode 100644 index 933bbf5..0000000 Binary files a/figs/NAISRoverviewwide.pdf and /dev/null differ diff --git a/figs/NAISRoverviewwide.svg b/figs/NAISRoverviewwide.svg deleted file mode 100644 index 871b49c..0000000 --- a/figs/NAISRoverviewwide.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/figs/Presentation2.pptx b/figs/Presentation2.pptx deleted file mode 100644 index e02a576..0000000 Binary files a/figs/Presentation2.pptx and /dev/null differ diff --git a/figs/dataset.svg b/figs/dataset.svg deleted file mode 100644 index 3243737..0000000 --- a/figs/dataset.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/figs/manifold.pdf b/figs/manifold.pdf deleted file mode 100644 index 92535bc..0000000 Binary files a/figs/manifold.pdf and /dev/null differ diff --git a/figs/naisr.png b/figs/naisr.png deleted file mode 100644 index c9ba714..0000000 Binary files a/figs/naisr.png and /dev/null differ diff --git a/figs/overview.svg b/figs/overview.svg deleted file mode 100644 index 9a75afd..0000000 --- a/figs/overview.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/figs/recon.pptx b/figs/recon.pptx deleted file mode 100644 index 7476e62..0000000 Binary files a/figs/recon.pptx and /dev/null differ diff --git a/figs/reconstruction.pdf b/figs/reconstruction.pdf deleted file mode 100644 index eff87e1..0000000 Binary files a/figs/reconstruction.pdf and /dev/null differ diff --git a/figs/reconstruction.png b/figs/reconstruction.png deleted file mode 100644 index 35336ff..0000000 Binary files a/figs/reconstruction.png and /dev/null differ diff --git a/figs/reconstructions.pdf b/figs/reconstructions.pdf deleted file mode 100644 index 9661a96..0000000 Binary files a/figs/reconstructions.pdf and /dev/null differ diff --git a/figs/summary_test_metrics_transport_volume.svg b/figs/summary_test_metrics_transport_volume.svg deleted file mode 100644 index 85f7912..0000000 --- a/figs/summary_test_metrics_transport_volume.svg +++ /dev/null @@ -1,1091 +0,0 @@ - - - - - - - - 2023-02-27T14:58:34.743484 - image/svg+xml - - - Matplotlib v3.5.1, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/figs/vec.pptx b/figs/vec.pptx deleted file mode 100644 index e339cf0..0000000 Binary files a/figs/vec.pptx and /dev/null differ diff --git a/figs/vec_field.pdf b/figs/vec_field.pdf deleted file mode 100644 index 2c112e9..0000000 Binary files a/figs/vec_field.pdf and /dev/null differ diff --git a/figs/weight.png b/figs/weight.png deleted file mode 100644 index 83cb2cb..0000000 Binary files a/figs/weight.png and /dev/null differ